mm: lru_lock splitting
Subsystems (11)
v3 cover mm: lru_lock splitting
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving
This patchset is based on next-20120210
git: https://github.com/koct9i/linux/commits/lruvec-v3
---
Konstantin Khlebnikov (21):
memcg: unify inactive_ratio calculation
memcg: make mm_match_cgroup() hirarchical
memcg: fix page_referencies cgroup filter on global reclaim
memcg: use vm_swappiness from target memory cgroup
mm: rename lruvec->lists into lruvec->pages_lru
mm: lruvec linking functions
mm: add lruvec->pages_count
mm: unify inactive_list_is_low()
mm: add lruvec->reclaim_stat
mm: kill struct mem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec into update_page_reclaim_stat()
mm: push lruvecs from pagevec_lru_move_fn() to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: split zone->lru_lock
mm: zone lru vectors interleaving
include/linux/huge_mm.h | 3 include/linux/memcontrol.h | 75 ------ include/linux/mm.h | 66 +++++ include/linux/mm_inline.h | 19 +- include/linux/mmzone.h | 39 ++- include/linux/swap.h | 6 mm/Kconfig | 16 + mm/compaction.c | 31 +-- mm/huge_memory.c | 14 + mm/internal.h | 204 +++++++++++++++++ mm/ksm.c | 2 mm/memcontrol.c | 343 +++++++++++----------------- mm/migrate.c | 2 mm/page_alloc.c | 70 +----- mm/rmap.c | 2 mm/swap.c | 217 ++++++++++-------- mm/vmscan.c | 534 ++++++++++++++++++++++++-------------------- mm/vmstat.c | 6 18 files changed, 932 insertions(+), 717 deletions(-)
Signature · 8 lines
Signature -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/ Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 1/21 memcg: unify inactive_ratio calculation
This patch removes precalculated zone->inactive_ratio.
Now it always calculated in inactive_anon_is_low() from current lru sizes. After that we can merge memcg and non-memcg cases and drop duplicated code.
--- include/linux/memcontrol.h | 16 -------- include/linux/mmzone.h | 7 ---- mm/memcontrol.c | 38 ------------------- mm/page_alloc.c | 44 ---------------------- mm/vmscan.c | 88 ++++++++++++++++++++++++++++---------------- mm/vmstat.c | 6 +-- 6 files changed, 58 insertions(+), 141 deletions(-)
include/linux/memcontrol.h +0 -16
@@ -113,10 +113,6 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);/* * For memory reclaim. */ -int mem_cgroup_inactive_anon_is_low (struct mem_cgroup *memcg, -struct zone *zone); -int mem_cgroup_inactive_file_is_low (struct mem_cgroup *memcg, -struct zone *zone);int mem_cgroup_select_victim_node (struct mem_cgroup *memcg);unsigned long mem_cgroup_zone_nr_lru_pages (struct mem_cgroup *memcg,int nid,int zid,unsigned int lrumask);
@@ -319,18 +315,6 @@ static inline bool mem_cgroup_disabled(void)return true; } -static inline int -mem_cgroup_inactive_anon_is_low (struct mem_cgroup *memcg,struct zone *zone) -{ -return 1 ; -} - -static inline int -mem_cgroup_inactive_file_is_low (struct mem_cgroup *memcg,struct zone *zone) -{ -return 1 ; -} -static inline unsigned long mem_cgroup_zone_nr_lru_pages (struct mem_cgroup *memcg,int nid,int zid,unsigned int lru_mask)
include/linux/mmzone.h +0 -7
@@ -382,13 +382,6 @@ struct zone {/* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS ]; -/* -* The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on -* this zone's LRU. Maintained by the pageout code. -*/ -unsigned int inactive_ratio; - -ZONE_PADDING (_pad2_)/* Rarely used or read-mostly fields */
mm/memcontrol.c +0 -38
@@ -1157,44 +1157,6 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)return ret; } -int mem_cgroup_inactive_anon_is_low (struct mem_cgroup *memcg,struct zone *zone) -{ -unsigned long inactive_ratio; -int nid =zone_to_nid (zone); -int zid =zone_idx (zone); -unsigned long inactive; -unsigned long active; -unsigned long gb; - - inactive =mem_cgroup_zone_nr_lru_pages (memcg, nid, zid, -BIT (LRU_INACTIVE_ANON )); - active =mem_cgroup_zone_nr_lru_pages (memcg, nid, zid, -BIT (LRU_ACTIVE_ANON )); - - gb = (inactive + active) >> (30 -PAGE_SHIFT ); -if (gb) - inactive_ratio =int_sqrt (10 * gb); -else - inactive_ratio =1 ; - -return inactive * inactive_ratio < active; -} - -int mem_cgroup_inactive_file_is_low (struct mem_cgroup *memcg,struct zone *zone) -{ -unsigned long active; -unsigned long inactive; -int zid =zone_idx (zone); -int nid =zone_to_nid (zone); - - inactive =mem_cgroup_zone_nr_lru_pages (memcg, nid, zid, -BIT (LRU_INACTIVE_FILE )); - active =mem_cgroup_zone_nr_lru_pages (memcg, nid, zid, -BIT (LRU_ACTIVE_FILE )); - -return (active > inactive); -} -struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat (struct mem_cgroup *memcg,struct zone *zone) {
mm/page_alloc.c +0 -44
56 lines not on this page · raw message
mm/vmscan.c +56 -32
123 lines not on this page · raw message
mm/vmstat.c +2 -4
@@ -1017,11 +1017,9 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat, }seq_printf (m,"\n all_unreclaimable: %u" -"\n start_pfn: %lu" -"\n inactive_ratio: %u" , +"\n start_pfn: %lu" , zone->all_unreclaimable, - zone->zone_start_pfn, - zone->inactive_ratio); + zone->zone_start_pfn);seq_putc (m,'\n' ); }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 01/21] memcg: unify inactive_ratio calculation
Quoted 5 lines · On Thu, 23 Feb 2012 17:51:41 +0400 Konstantin Khlebnikov…
This patch removes precalculated
zone->inactive_ratio.
Now it always calculated ininactive_anon_is_low()from current lru sizes. After that we can merge memcg and non-memcg cases and drop duplicated code.Signed-off-by Konstantin Khlebnikov <[email protected]>
seems good to me.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 2/21 memcg: make mm_match_cgroup() hirarchical
In reply to Konstantin KhlebnikovCheck mm-owner cgroup membership hierarchically.
--- include/linux/memcontrol.h | 11 ++--------- mm/memcontrol.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-)
include/linux/memcontrol.h +2 -9
@@ -87,15 +87,8 @@ extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);extern struct mem_cgroup *parent_mem_cgroup (struct mem_cgroup *memcg);extern struct mem_cgroup *mem_cgroup_from_cont (struct cgroup *cont); -static inline -int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) -{ -struct mem_cgroup *memcg; -rcu_read_lock (); - memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); -rcu_read_unlock (); -return cgroup == memcg; -} +extern int mm_match_cgroup (const struct mm_struct *mm, +const struct mem_cgroup *cgroup);extern struct cgroup_subsys_state *mem_cgroup_css (struct mem_cgroup *memcg);
mm/memcontrol.c +20 -0
@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup, css); } +/** +* mm_match_cgroup - cgroup hierarchy mm membership test +* @mm mm_struct to test +* @cgroup target cgroup +* +* Returns true if mm belong this cgroup or any its child in hierarchy +*/ +int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) +{ +struct mem_cgroup *memcg; + +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); +while (memcg != cgroup && memcg && memcg->use_hierarchy) + memcg =parent_mem_cgroup (memcg); +rcu_read_unlock (); + +return cgroup == memcg; +} +struct mem_cgroup *try_get_mem_cgroup_from_mm (struct mm_struct *mm) {struct mem_cgroup *memcg =NULL ;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Johannes Weiner
maintainerRe: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical
Quoted 1 line · Konstantin Khlebnikov
Check mm-owner cgroup membership hierarchically.
I think this one cat just beat up this other cat in front of my window, yelling something about money and missing product. Anyway, I already forgot why we want this patch. Could you describe that in the changelog, please?
Quoted 13 lines
mm/memcontrol.c +9
-0@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup, css); } +/** +* mm_match_cgroup - cgroup hierarchy mm membership test +* @mm mm_struct to test +* @cgroup target cgroup +* +* Returns true if mm belong this cgroup or any its child in hierarchy +*/ +int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) +{
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 842
Quoted 10 lines
mm/memcontrol.c +10
-0+struct mem_cgroup *memcg; + +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); +while (memcg != cgroup && memcg && memcg->use_hierarchy) + memcg =parent_mem_cgroup (memcg); +rcu_read_unlock (); + +return cgroup == memcg; +}
Please don't duplicate mem_cgroup_same_or_subtree()'s functionality in a worse way. The hierarchy information is kept in a stack such that ancestry can be detected in linear time, check out css_is_ancestor().
If you don't want to nest rcu_read_lock(), you could push the rcu_read_lock() from css_is_ancestor() into its sole user and provide a __mem_cgroup_is_ancestor() that assumes rcu already read-locked.
No?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical
Quoted 7 lines · Johannes Weiner
Quoted 1 line · Konstantin Khlebnikov
Check mm-owner cgroup membership hierarchically.
I think this one cat just beat up this other cat in front of my window, yelling something about money and missing product. Anyway, I already forgot why we want this patch. Could you describe that in the changelog, please?
Yeah, sorry for lack of comment.
This test is used in rmap walker at checling page referencies in reclaimer. Memory cgroup shrinker want to skip all referencies outside of cgroup hierarchy which is currently under reclaim.
Actually this patch does not important for this set and can be dropped without problems, it does not share any context with other patches. Next patch is more important because it fixes global reclaimer and required for further cleanups.
Quoted 20 lines
Quoted 20 lines
mm/memcontrol.c +15
-0@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup, css); } +/** +* mm_match_cgroup - cgroup hierarchy mm membership test +* @mm mm_struct to test +* @cgroup target cgroup +* +* Returns true if mm belong this cgroup or any its child in hierarchy +*/ +int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) +{ +struct mem_cgroup *memcg; + +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); +while (memcg != cgroup&& memcg&& memcg->use_hierarchy) + memcg =parent_mem_cgroup (memcg);
Quoted 8 lines
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 842
Quoted 4 lines
mm/memcontrol.c +4
-0+rcu_read_unlock (); + +return cgroup == memcg; +}Please don't duplicate mem_cgroup_same_or_subtree()'s functionality in a worse way. The hierarchy information is kept in a stack such that ancestry can be detected in linear time, check out
css_is_ancestor().
Ok, there will be something like that:
+bool mm_match_cgroup(const struct mm_struct *mm,
+ const struct mem_cgroup *cgroup)
+{
+ struct mem_cgroup *memcg;
+ bool ret;
+
+ rcu_read_lock(); + memcg = mem_cgroup_from_task(rcu_dereference((mm)->owner)); + ret = memcg && mem_cgroup_same_or_subtree(cgroup, memcg); + rcu_read_unlock();
+
+ return ret;
+}
+
Quoted 6 lines
If you don't want to nest
rcu_read_lock(), you could push thercu_read_lock()fromcss_is_ancestor()into its sole user and provide a__mem_cgroup_is_ancestor()that assumes rcu already read-locked.No?
It is not a problem.
looks like mem_cgroup_same_or_subtree() check something different, because it does not check ->use_hierarchy flag on tested cgroup, only on target cgroup.
Or just all this hierarchical stuff is out of sync in different parts of code. For example memcg_get_hierarchical_limit() start from deepest cgroup and go upper while ->use_hierarchy is set.
Quoted 7 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email:<a href=mailto:"[email protected]"> [email protected]</a>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Johannes Weiner
maintainerRe: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical
Quoted 36 lines · Konstantin Khlebnikov
Quoted 20 lines · Johannes Weiner
Quoted 19 lines · Konstantin Khlebnikov
mm/memcontrol.c +15
-0@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup, css); } +/** +* mm_match_cgroup - cgroup hierarchy mm membership test +* @mm mm_struct to test +* @cgroup target cgroup +* +* Returns true if mm belong this cgroup or any its child in hierarchy +*/ +int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) +{ +struct mem_cgroup *memcg; + +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); +while (memcg != cgroup&& memcg&& memcg->use_hierarchy) + memcg =parent_mem_cgroup (memcg);Quoted 8 lines
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 842
Quoted 4 lines
mm/memcontrol.c +4
-0+rcu_read_unlock (); + +return cgroup == memcg; +}Please don't duplicate mem_cgroup_same_or_subtree()'s functionality in a worse way. The hierarchy information is kept in a stack such that ancestry can be detected in linear time, check out
css_is_ancestor().Ok, there will be something like that:
mm/memcontrol.c +4
-0+bool mm_match_cgroup (const struct mm_struct *mm, +const struct mem_cgroup *cgroup) +{ +struct mem_cgroup *memcg;
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 843
Quoted 10 lines
mm/memcontrol.c +10
-0+bool ret; + +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); + ret = memcg &&mem_cgroup_same_or_subtree (cgroup, memcg); +rcu_read_unlock (); + +return ret; +} +
It would be unfortunate to nest rcu_read_lock(), but I think this looks good otherwise.
Quoted 10 lines
Quoted 5 lines
If you don't want to nest
rcu_read_lock(), you could push thercu_read_lock()fromcss_is_ancestor()into its sole user and provide a__mem_cgroup_is_ancestor()that assumes rcu already read-locked.No?
It is not a problem.
looks like
mem_cgroup_same_or_subtree()check something different, because it does not check ->use_hierarchy flag on tested cgroup, only on target cgroup.
If a memcg has hierarchy enabled, any memcg that turns out to be its child is guaranteed to have hierarchy enabled.
Quoted 3 lines
Or just all this hierarchical stuff is out of sync in different parts of code. For example
memcg_get_hierarchical_limit()start from deepest cgroup and go upper while ->use_hierarchy is set.
This one really has to walk up and find the smallest applicable limit, there is no way around looking at every single level.
But checking for ancestry can and has been optimized.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical
In reply to Konstantin KhlebnikovQuoted 3 lines · On Thu, 23 Feb 2012 17:51:46 +0400 Konstantin Khlebnikov…
Check mm-owner cgroup membership hierarchically.
Signed-off-by Konstantin Khlebnikov <[email protected]>
Ack. but ... see below.
Quoted 32 lines
--- include/linux/memcontrol.h | 11 ++--------- mm/memcontrol.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-)include/linux/memcontrol.h +2
-9@@ -87,15 +87,8 @@ extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);extern struct mem_cgroup *parent_mem_cgroup (struct mem_cgroup *memcg);extern struct mem_cgroup *mem_cgroup_from_cont (struct cgroup *cont); -static inline -int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) -{ -struct mem_cgroup *memcg; -rcu_read_lock (); - memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); -rcu_read_unlock (); -return cgroup == memcg; -} +extern int mm_match_cgroup (const struct mm_struct *mm, +const struct mem_cgroup *cgroup);extern struct cgroup_subsys_state *mem_cgroup_css (struct mem_cgroup *memcg);mm/memcontrol.c +0
-0
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 829
Quoted 9 lines
mm/memcontrol.c +6
-0@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup, css); } +/** +* mm_match_cgroup - cgroup hierarchy mm membership test +* @mm mm_struct to test +* @cgroup target cgroup +* +* Returns true if mm belong this cgroup or any its child in hierarchy
belongs to ?
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 832
Quoted 3 lines
mm/memcontrol.c +3
-0+*/ +int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) +{
Please use "memcg" for representing "memory cgroup" (other function's arguments uses "memcg")
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 833
Quoted 1 line
mm/memcontrol.c +1
-0+struct mem_cgroup *memcg;
So, rename this as *cur_memcg or some.
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 838
Quoted 5 lines
mm/memcontrol.c +5
-0+ +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); +while (memcg != cgroup && memcg && memcg->use_hierarchy) + memcg =parent_mem_cgroup (memcg);
IIUC, parent_mem_cgroup() checks mem->res.parent. mem->res.parent is set only when parent->use_hierarchy == true. Then,
while (memcg != cgroup) memcg = parent_mem_cgroup(memcg);
will be enough.
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 02/21] memcg: make mm_match_cgroup() hirarchical
Quoted 73 lines · KAMEZAWA Hiroyuki
Quoted 3 lines · On Thu, 23 Feb 2012 17:51:46 +0400 Konstantin…
Check mm-owner cgroup membership hierarchically.
Signed-off-by Konstantin Khlebnikov<[email protected]>Ack. but ... see below.
Quoted 32 lines
--- include/linux/memcontrol.h | 11 ++--------- mm/memcontrol.c | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 9 deletions(-)include/linux/memcontrol.h +2
-9@@ -87,15 +87,8 @@ extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm);extern struct mem_cgroup *parent_mem_cgroup (struct mem_cgroup *memcg);extern struct mem_cgroup *mem_cgroup_from_cont (struct cgroup *cont); -static inline -int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) -{ -struct mem_cgroup *memcg; -rcu_read_lock (); - memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); -rcu_read_unlock (); -return cgroup == memcg; -} +extern int mm_match_cgroup (const struct mm_struct *mm, +const struct mem_cgroup *cgroup);extern struct cgroup_subsys_state *mem_cgroup_css (struct mem_cgroup *memcg);mm/memcontrol.c +0
-0mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 829
Quoted 9 lines
mm/memcontrol.c +6
-0@@ -821,6 +821,26 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) struct mem_cgroup, css); } +/** +* mm_match_cgroup - cgroup hierarchy mm membership test +* @mm mm_struct to test +* @cgroup target cgroup +* +* Returns true if mm belong this cgroup or any its child in hierarchy belongs to ?
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 832
Quoted 3 lines
mm/memcontrol.c +3
-0+*/ +int mm_match_cgroup (const struct mm_struct *mm,const struct mem_cgroup *cgroup) +{Please use "memcg" for representing "memory cgroup" (other function's arguments uses "memcg")
mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 833
Quoted 1 line
mm/memcontrol.c +1
-0+struct mem_cgroup *memcg;So, rename this as
*cur_memcgor some.mm/memcontrol.c struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) · line 838
Quoted 5 lines
mm/memcontrol.c +5
-0+ +rcu_read_lock (); + memcg =mem_cgroup_from_task (rcu_dereference ((mm)->owner)); +while (memcg != cgroup&& memcg&& memcg->use_hierarchy) + memcg =parent_mem_cgroup (memcg);IIUC,
parent_mem_cgroup()checksmem->res.parent.mem->res.parentis set only whenparent->use_hierarchy== true. Then,
Quoted 4 lines
while (memcg != cgroup) memcg = parent_mem_cgroup(memcg);will be enough.
Here will be mem_cgroup_same_or_subtree(), see reply from Johannes Weiner.
Quoted 4 lines
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 3/21 memcg: fix page_referencies cgroup filter on global reclaim
In reply to Konstantin KhlebnikovGlobal memory reclaimer shouldn't skip any page referencies.
This patch pass sc->target_mem_cgroup into page_referenced().
On global memory reclaim it always NULL, so we will account all. Cgroup reclaimer will account only referencies from target cgroup and its childs.
--- mm/vmscan.c | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-)
mm/vmscan.c +5 -4
@@ -701,13 +701,13 @@ enum page_references { };static enum page_references page_check_references (struct page *page, -struct mem_cgroup_zone *mz,struct scan_control *sc) {int referenced_ptes, referenced_page;unsigned long vm_flags; - referenced_ptes =page_referenced (page,1 , mz->mem_cgroup, &vm_flags); + referenced_ptes =page_referenced (page,1 , + sc->target_mem_cgroup, &vm_flags); referenced_page =TestClearPageReferenced (page);/* Lumpy reclaim - ignore references */
@@ -828,7 +828,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, } } - references =page_check_references (page, mz, sc); + references =page_check_references (page, sc);switch (references) {case PAGEREF_ACTIVATE : goto activate_locked;
@@ -1735,7 +1735,8 @@ static void shrink_active_list(unsigned long nr_to_scan,continue ; } -if (page_referenced (page,0 , mz->mem_cgroup, &vm_flags)) { +if (page_referenced (page,0 , + sc->target_mem_cgroup, &vm_flags)) { nr_rotated +=hpage_nr_pages (page);/* * Identify referenced, file-backed active pages and
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 03/21] memcg: fix page_referencies cgroup filter on global reclaim
Quoted 7 lines · On Thu, 23 Feb 2012 17:51:51 +0400 Konstantin Khlebnikov…
Global memory reclaimer shouldn't skip any page referencies.
This patch pass
sc->target_mem_cgroupintopage_referenced().
On global memory reclaim it always NULL, so we will account all. Cgroup reclaimer will account only referencies from target cgroup and its childs.Signed-off-by Konstantin Khlebnikov <[email protected]>
seems nice to me.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 4/21 memcg: use vm_swappiness from target memory cgroup
In reply to Konstantin KhlebnikovUse vm_swappiness from memory cgroup which is triggered this memory reclaim. This is more reasonable and allows to kill one argument.
--- mm/vmscan.c | 9 ++++----- 1 files changed, 4 insertions(+), 5 deletions(-)
mm/vmscan.c +4 -5
@@ -1903,12 +1903,11 @@ static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,return shrink_inactive_list (nr_to_scan, mz, sc, priority, file); } -static int vmscan_swappiness (struct mem_cgroup_zone *mz, -struct scan_control *sc) +static int vmscan_swappiness (struct scan_control *sc) {if (global_reclaim (sc))return vm_swappiness; -return mem_cgroup_swappiness (mz->mem_cgroup); +return mem_cgroup_swappiness (sc->target_mem_cgroup); }/*
@@ -1976,8 +1975,8 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,* With swappiness at 100, anonymous and file have the same priority. * This scanning priority is essentially the inverse of IO cost. */ - anon_prio =vmscan_swappiness (mz, sc); - file_prio =200 -vmscan_swappiness (mz, sc); + anon_prio =vmscan_swappiness (sc); + file_prio =200 -vmscan_swappiness (sc);/* * OK, so we have swap space and a fair amount of page cache
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 04/21] memcg: use vm_swappiness from target memory cgroup
Quoted 4 lines · On Thu, 23 Feb 2012 17:51:56 +0400 Konstantin Khlebnikov…
Use
vm_swappinessfrom memory cgroup which is triggered this memory reclaim. This is more reasonable and allows to kill one argument.Signed-off-by Konstantin Khlebnikov <[email protected]>
Hmm... it may be better to disallow to have different swappiness in a hierarchy..
Quoted 1 line
From me,
But I wonder other guys may have different idea.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 5/21 mm: rename lruvec->lists into lruvec->pages_lru
In reply to Konstantin KhlebnikovThis is much more unique and grep-friendly name.
--- include/linux/mm_inline.h | 2 +- include/linux/mmzone.h | 2 +- mm/memcontrol.c | 6 +++--- mm/page_alloc.c | 2 +- mm/swap.c | 4 ++-- mm/vmscan.c | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-)
include/linux/mm_inline.h +1 -1
@@ -27,7 +27,7 @@ add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)struct lruvec *lruvec; lruvec =mem_cgroup_lru_add_list (zone, page, lru); -list_add (&page->lru, &lruvec->lists[lru]); +list_add (&page->lru, &lruvec->pages_lru[lru]);__mod_zone_page_state (zone,NR_LRU_BASE + lru,hpage_nr_pages (page)); }
include/linux/mmzone.h +1 -1
@@ -160,7 +160,7 @@ static inline int is_unevictable_lru(enum lru_list lru) }struct lruvec { -struct list_head lists[NR_LRU_LISTS ]; +struct list_head pages_lru[NR_LRU_LISTS ]; };/* Mask used at gathering information at once (see memcontrol.c) */
mm/memcontrol.c +3 -3
@@ -1050,7 +1050,7 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,* the lruvec for the given @zone and the memcg @page is charged to. * * The callsite is then responsible for physically linking the page to -* the returned lruvec->lists[@lru]. +* the returned lruvec->pages_lru[@lru]. */ struct lruvec *mem_cgroup_lru_add_list (struct zone *zone,struct page *page,enum lru_list lru)
@@ -3592,7 +3592,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, zone = &NODE_DATA (node)->node_zones[zid]; mz =mem_cgroup_zoneinfo (memcg, node, zid); - list = &mz->lruvec.lists[lru]; + list = &mz->lruvec.pages_lru[lru]; loop = mz->lru_size[lru];/* give some margin against EBUSY etc...*/
@@ -4716,7 +4716,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)for (zone =0 ; zone <MAX_NR_ZONES ; zone++) { mz = &pn->zoneinfo[zone];for_each_lru (lru) -INIT_LIST_HEAD (&mz->lruvec.lists[lru]); +INIT_LIST_HEAD (&mz->lruvec.pages_lru[lru]); mz->usage_in_excess =0 ; mz->on_tree = false; mz->memcg = memcg;
mm/page_alloc.c +1 -1
@@ -4363,7 +4363,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,zone_pcp_init (zone);for_each_lru (lru) -INIT_LIST_HEAD (&zone->lruvec.lists[lru]); +INIT_LIST_HEAD (&zone->lruvec.pages_lru[lru]); zone->reclaim_stat.recent_rotated[0 ] =0 ; zone->reclaim_stat.recent_rotated[1 ] =0 ; zone->reclaim_stat.recent_scanned[0 ] =0 ;
mm/swap.c +2 -2
@@ -238,7 +238,7 @@ static void pagevec_move_tail_fn(struct page *page, void *arg) lruvec =mem_cgroup_lru_move_lists (page_zone (page), page, lru, lru); -list_move_tail (&page->lru, &lruvec->lists[lru]); +list_move_tail (&page->lru, &lruvec->pages_lru[lru]); (*pgmoved)++; } }
@@ -482,7 +482,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)* We moves tha page into tail of inactive. */ lruvec =mem_cgroup_lru_move_lists (zone, page, lru, lru); -list_move_tail (&page->lru, &lruvec->lists[lru]); +list_move_tail (&page->lru, &lruvec->pages_lru[lru]);__count_vm_event (PGROTATED ); }
mm/vmscan.c +3 -3
@@ -1164,7 +1164,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, lru +=LRU_ACTIVE ;if (file) lru +=LRU_FILE ; - src = &lruvec->lists[lru]; + src = &lruvec->pages_lru[lru];for (scan =0 ; scan < nr_to_scan && !list_empty (src); scan++) {struct page *page;
@@ -1663,7 +1663,7 @@ static void move_active_pages_to_lru(struct zone *zone,SetPageLRU (page); lruvec =mem_cgroup_lru_add_list (zone, page, lru); -list_move (&page->lru, &lruvec->lists[lru]); +list_move (&page->lru, &lruvec->pages_lru[lru]); pgmoved +=hpage_nr_pages (page);if (put_page_testzero (page)) {
@@ -3592,7 +3592,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)__dec_zone_state (zone,NR_UNEVICTABLE ); lruvec =mem_cgroup_lru_move_lists (zone, page,LRU_UNEVICTABLE , lru); -list_move (&page->lru, &lruvec->lists[lru]); +list_move (&page->lru, &lruvec->pages_lru[lru]);__inc_zone_state (zone,NR_INACTIVE_ANON + lru); pgrescued++; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 05/21] mm: rename lruvec->lists into lruvec->pages_lru
Quoted 3 lines · On Thu, 23 Feb 2012 17:52:00 +0400 Konstantin Khlebnikov…
This is much more unique and grep-friendly name.
Signed-off-by Konstantin Khlebnikov <[email protected]>
I worries this kind of change can cause many hunks and make merging difficult.. But this seems not very destructive..
I have no strong opinions to this naming. How other mm developpers think ?
I personally think making this kind of changes in the head of patch set tend do make it difficult to merge full sets of patche series.
Thanks,
-Kame
Quoted 134 lines
--- include/linux/mm_inline.h | 2 +- include/linux/mmzone.h | 2 +- mm/memcontrol.c | 6 +++--- mm/page_alloc.c | 2 +- mm/swap.c | 4 ++-- mm/vmscan.c | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-)include/linux/mm_inline.h +1
-1@@ -27,7 +27,7 @@ add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)struct lruvec *lruvec; lruvec =mem_cgroup_lru_add_list (zone, page, lru); -list_add (&page->lru, &lruvec->lists[lru]); +list_add (&page->lru, &lruvec->pages_lru[lru]);__mod_zone_page_state (zone,NR_LRU_BASE + lru,hpage_nr_pages (page)); }include/linux/mmzone.h +1
-1@@ -160,7 +160,7 @@ static inline int is_unevictable_lru(enum lru_list lru) }struct lruvec { -struct list_head lists[NR_LRU_LISTS ]; +struct list_head pages_lru[NR_LRU_LISTS ]; };/* Mask used at gathering information at once (see memcontrol.c) */ mm/memcontrol.c +3
-3@@ -1050,7 +1050,7 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,* the lruvec for the given @zone and the memcg @page is charged to. * * The callsite is then responsible for physically linking the page to -* the returned lruvec->lists[@lru]. +* the returned lruvec->pages_lru[@lru]. */ struct lruvec *mem_cgroup_lru_add_list (struct zone *zone,struct page *page,enum lru_list lru)@@ -3592,7 +3592,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, zone = &NODE_DATA (node)->node_zones[zid]; mz =mem_cgroup_zoneinfo (memcg, node, zid); - list = &mz->lruvec.lists[lru]; + list = &mz->lruvec.pages_lru[lru]; loop = mz->lru_size[lru];/* give some margin against EBUSY etc...*/ @@ -4716,7 +4716,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)for (zone =0 ; zone <MAX_NR_ZONES ; zone++) { mz = &pn->zoneinfo[zone];for_each_lru (lru) -INIT_LIST_HEAD (&mz->lruvec.lists[lru]); +INIT_LIST_HEAD (&mz->lruvec.pages_lru[lru]); mz->usage_in_excess =0 ; mz->on_tree = false; mz->memcg = memcg;mm/page_alloc.c +1
-1@@ -4363,7 +4363,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,zone_pcp_init (zone);for_each_lru (lru) -INIT_LIST_HEAD (&zone->lruvec.lists[lru]); +INIT_LIST_HEAD (&zone->lruvec.pages_lru[lru]); zone->reclaim_stat.recent_rotated[0 ] =0 ; zone->reclaim_stat.recent_rotated[1 ] =0 ; zone->reclaim_stat.recent_scanned[0 ] =0 ;mm/swap.c +2
-2@@ -238,7 +238,7 @@ static void pagevec_move_tail_fn(struct page *page, void *arg) lruvec =mem_cgroup_lru_move_lists (page_zone (page), page, lru, lru); -list_move_tail (&page->lru, &lruvec->lists[lru]); +list_move_tail (&page->lru, &lruvec->pages_lru[lru]); (*pgmoved)++; } }@@ -482,7 +482,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)* We moves tha page into tail of inactive. */ lruvec =mem_cgroup_lru_move_lists (zone, page, lru, lru); -list_move_tail (&page->lru, &lruvec->lists[lru]); +list_move_tail (&page->lru, &lruvec->pages_lru[lru]);__count_vm_event (PGROTATED ); }mm/vmscan.c +3
-4@@ -1164,7 +1164,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, lru +=LRU_ACTIVE ;if (file) lru +=LRU_FILE ; - src = &lruvec->lists[lru]; + src = &lruvec->pages_lru[lru];for (scan =0 ; scan < nr_to_scan && !list_empty (src); scan++) {struct page *page;@@ -1663,7 +1663,7 @@ static void move_active_pages_to_lru(struct zone *zone,SetPageLRU (page); lruvec =mem_cgroup_lru_add_list (zone, page, lru); -list_move (&page->lru, &lruvec->lists[lru]); +list_move (&page->lru, &lruvec->pages_lru[lru]); pgmoved +=hpage_nr_pages (page);if (put_page_testzero (page)) {@@ -3592,7 +3592,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)__dec_zone_state (zone,NR_UNEVICTABLE ); lruvec =mem_cgroup_lru_move_lists (zone, page,LRU_UNEVICTABLE , lru); -list_move (&page->lru, &lruvec->lists[lru]); +list_move (&page->lru, &lruvec->pages_lru[lru]);__inc_zone_state (zone,NR_INACTIVE_ANON + lru); pgrescued++; } --
Quoted 5 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 05/21] mm: rename lruvec->lists into lruvec->pages_lru
Quoted 10 lines · KAMEZAWA Hiroyuki
Quoted 3 lines · On Thu, 23 Feb 2012 17:52:00 +0400 Konstantin…
This is much more unique and grep-friendly name.
Signed-off-by Konstantin Khlebnikov<[email protected]>I worries this kind of change can cause many hunks and make merging difficult.. But this seems not very destructive..
Quoted 6 lines
Reviewed-by KAMEZAWA Hiroyuki<[email protected]>I have no strong opinions to this naming. How other mm developpers think ?
I personally think making this kind of changes in the head of patch set tend do make it difficult to merge full sets of patche series.
This rename allows to easily highlight lru-lists manipulations, because some of them are in non-trivial places.
Quoted 137 lines
Thanks,
-KameQuoted 133 lines
--- include/linux/mm_inline.h | 2 +- include/linux/mmzone.h | 2 +- mm/memcontrol.c | 6 +++--- mm/page_alloc.c | 2 +- mm/swap.c | 4 ++-- mm/vmscan.c | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-)include/linux/mm_inline.h +1
-1@@ -27,7 +27,7 @@ add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list lru)struct lruvec *lruvec; lruvec =mem_cgroup_lru_add_list (zone, page, lru); -list_add (&page->lru,&lruvec->lists[lru]); +list_add (&page->lru,&lruvec->pages_lru[lru]);__mod_zone_page_state (zone,NR_LRU_BASE + lru,hpage_nr_pages (page)); }include/linux/mmzone.h +1
-1@@ -160,7 +160,7 @@ static inline int is_unevictable_lru(enum lru_list lru) }struct lruvec { -struct list_head lists[NR_LRU_LISTS ]; +struct list_head pages_lru[NR_LRU_LISTS ]; };/* Mask used at gathering information at once (see memcontrol.c) */ mm/memcontrol.c +3
-3@@ -1050,7 +1050,7 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,* the lruvec for the given @zone and the memcg @page is charged to. * * The callsite is then responsible for physically linking the page to -* the returned lruvec->lists[@lru]. +* the returned lruvec->pages_lru[@lru]. */ struct lruvec *mem_cgroup_lru_add_list (struct zone *zone,struct page *page,enum lru_list lru)@@ -3592,7 +3592,7 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg, zone =&NODE_DATA (node)->node_zones[zid]; mz =mem_cgroup_zoneinfo (memcg, node, zid); - list =&mz->lruvec.lists[lru]; + list =&mz->lruvec.pages_lru[lru]; loop = mz->lru_size[lru];/* give some margin against EBUSY etc...*/ @@ -4716,7 +4716,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)for (zone =0 ; zone<MAX_NR_ZONES ; zone++) { mz =&pn->zoneinfo[zone];for_each_lru (lru) -INIT_LIST_HEAD (&mz->lruvec.lists[lru]); +INIT_LIST_HEAD (&mz->lruvec.pages_lru[lru]); mz->usage_in_excess =0 ; mz->on_tree = false; mz->memcg = memcg;mm/page_alloc.c +1
-1@@ -4363,7 +4363,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,zone_pcp_init (zone);for_each_lru (lru) -INIT_LIST_HEAD (&zone->lruvec.lists[lru]); +INIT_LIST_HEAD (&zone->lruvec.pages_lru[lru]); zone->reclaim_stat.recent_rotated[0 ] =0 ; zone->reclaim_stat.recent_rotated[1 ] =0 ; zone->reclaim_stat.recent_scanned[0 ] =0 ;mm/swap.c +2
-2@@ -238,7 +238,7 @@ static void pagevec_move_tail_fn(struct page *page, void *arg) lruvec =mem_cgroup_lru_move_lists (page_zone (page), page, lru, lru); -list_move_tail (&page->lru,&lruvec->lists[lru]); +list_move_tail (&page->lru,&lruvec->pages_lru[lru]); (*pgmoved)++; } }@@ -482,7 +482,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)* We moves tha page into tail of inactive. */ lruvec =mem_cgroup_lru_move_lists (zone, page, lru, lru); -list_move_tail (&page->lru,&lruvec->lists[lru]); +list_move_tail (&page->lru,&lruvec->pages_lru[lru]);__count_vm_event (PGROTATED ); }mm/vmscan.c +3
-3@@ -1164,7 +1164,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, lru +=LRU_ACTIVE ;if (file) lru +=LRU_FILE ; - src =&lruvec->lists[lru]; + src =&lruvec->pages_lru[lru];for (scan =0 ; scan< nr_to_scan&& !list_empty (src); scan++) {struct page *page;@@ -1663,7 +1663,7 @@ static void move_active_pages_to_lru(struct zone *zone,SetPageLRU (page); lruvec =mem_cgroup_lru_add_list (zone, page, lru); -list_move (&page->lru,&lruvec->lists[lru]); +list_move (&page->lru,&lruvec->pages_lru[lru]); pgmoved +=hpage_nr_pages (page);if (put_page_testzero (page)) {@@ -3592,7 +3592,7 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)__dec_zone_state (zone,NR_UNEVICTABLE ); lruvec =mem_cgroup_lru_move_lists (zone, page,LRU_UNEVICTABLE , lru); -list_move (&page->lru,&lruvec->lists[lru]); +list_move (&page->lru,&lruvec->pages_lru[lru]);__inc_zone_state (zone,NR_INACTIVE_ANON + lru); pgrescued++; }
Quoted 7 lines
Quoted 6 lines
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 6/21 mm: lruvec linking functions
In reply to Konstantin KhlebnikovThis patch adds links from page to its lruvec and from lruvec to its zone and node. If CONFIG_CGROUP_MEM_RES_CTLR=n they just page_zone() and container_of().
--- include/linux/mm.h | 37 +++++++++++++++++++++++++++++++++++++ include/linux/mmzone.h | 12 ++++++++---- mm/internal.h | 1 + mm/memcontrol.c | 27 ++++++++++++++++++++++++--- mm/page_alloc.c | 17 ++++++++++++++--- 5 files changed, 84 insertions(+), 10 deletions(-)
include/linux/mm.h +37 -0
@@ -728,6 +728,43 @@ static inline void set_page_links(struct page *page, enum zone_type zone,#endif } +#ifdef CONFIG_CGROUP_MEM_RES_CTLR + +/* Multiple lruvecs in zone */ + +extern struct lruvec *page_lruvec (struct page *page); + +static inline struct zone *lruvec_zone (struct lruvec *lruvec) +{ +return lruvec->zone; +} + +static inline struct pglist_data *lruvec_node (struct lruvec *lruvec) +{ +return lruvec->node; +} + +#else /* CONFIG_CGROUP_MEM_RES_CTLR */ + +/* Single lruvec in zone */ + +static inline struct lruvec *page_lruvec (struct page *page) +{ +return &page_zone (page)->lruvec; +} + +static inline struct zone *lruvec_zone (struct lruvec *lruvec) +{ + return container_of(lruvec, struct zone, lruvec); +} + +static inline struct pglist_data *lruvec_node (struct lruvec *lruvec) +{ +return lruvec_zone (lruvec)->zone_pgdat; +} + +#endif /* CONFIG_CGROUP_MEM_RES_CTLR */ +/* * Some inline functions in vmstat.h depend on page_zone() */
include/linux/mmzone.h +8 -4
@@ -159,10 +159,6 @@ static inline int is_unevictable_lru(enum lru_list lru)return (lru ==LRU_UNEVICTABLE ); } -struct lruvec { -struct list_head pages_lru[NR_LRU_LISTS ]; -}; -/* Mask used at gathering information at once (see memcontrol.c) */ #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
@@ -300,6 +296,14 @@ struct zone_reclaim_stat {unsigned long recent_scanned[2 ]; }; +struct lruvec { +struct list_head pages_lru[NR_LRU_LISTS ]; +#ifdef CONFIG_CGROUP_MEM_RES_CTLR +struct zone *zone; +struct pglist_data *node; +#endif +}; +struct zone {/* Fields commonly accessed by the page allocator */
mm/internal.h +1 -0
@@ -100,6 +100,7 @@ extern void prep_compound_page(struct page *page, unsigned long order);extern bool is_free_buddy_page (struct page *page);#endif +extern void init_zone_lruvec (struct zone *zone,struct lruvec *lruvec);/* * function for dealing with page's order in buddy system.
mm/memcontrol.c +24 -3
@@ -1026,6 +1026,28 @@ struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,return &mz->lruvec; } +/** +* page_lruvec - get the lruvec there this page is located +* @page: the struct page pointer with stable reference +* +* page_cgroup->mem_cgroup pointer validity guaranteed by caller. +* +* Returns pointer to struct lruvec. +*/ +struct lruvec *page_lruvec (struct page *page) +{ +struct mem_cgroup_per_zone *mz; +struct page_cgroup *pc; + +if (mem_cgroup_disabled ()) +return &page_zone (page)->lruvec; + + pc =lookup_page_cgroup (page); + mz =mem_cgroup_zoneinfo (pc->mem_cgroup, +page_to_nid (page),page_zonenum (page)); +return &mz->lruvec; +} +/* * Following LRU functions are allowed to be used without PCG_LOCK. * Operations are called by routine of global LRU independently from memcg.
@@ -4697,7 +4719,6 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node) {struct mem_cgroup_per_node *pn;struct mem_cgroup_per_zone *mz; -enum lru_list lru;int zone, tmp = node;/* * This routine is called against possible nodes.
@@ -4715,8 +4736,8 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)for (zone =0 ; zone <MAX_NR_ZONES ; zone++) { mz = &pn->zoneinfo[zone]; -for_each_lru (lru) -INIT_LIST_HEAD (&mz->lruvec.pages_lru[lru]); +init_zone_lruvec (&NODE_DATA (node)->node_zones[zone], + &mz->lruvec); mz->usage_in_excess =0 ; mz->on_tree = false; mz->memcg = memcg;
mm/page_alloc.c +14 -3
@@ -4289,6 +4289,19 @@ static inline int pageblock_default_order(unsigned int order)#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ +void init_zone_lruvec (struct zone *zone,struct lruvec *lruvec) +{ +enum lru_list lru; + +memset (lruvec,0 ,sizeof (struct lruvec )); +for_each_lru (lru) +INIT_LIST_HEAD (&lruvec->pages_lru[lru]); +#ifdef CONFIG_CGROUP_MEM_RES_CTLR + lruvec->node = zone->zone_pgdat; + lruvec->zone = zone; +#endif +} +/* * Set up the zone data structures: * - mark all pages reserved
@@ -4312,7 +4325,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,for (j =0 ; j <MAX_NR_ZONES ; j++) {struct zone *zone = pgdat->node_zones + j;unsigned long size, realsize, memmap_pages; -enum lru_list lru; size =zone_spanned_pages_in_node (nid, j, zones_size); realsize = size - zone_absent_pages_in_node(nid, j,
@@ -4362,8 +4374,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat, zone->zone_pgdat = pgdat;zone_pcp_init (zone); -for_each_lru (lru) -INIT_LIST_HEAD (&zone->lruvec.pages_lru[lru]); +init_zone_lruvec (zone, &zone->lruvec); zone->reclaim_stat.recent_rotated[0 ] =0 ; zone->reclaim_stat.recent_rotated[1 ] =0 ; zone->reclaim_stat.recent_scanned[0 ] =0 ;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 06/21] mm: lruvec linking functions
Quoted 4 lines · On Thu, 23 Feb 2012 17:52:04 +0400 Konstantin Khlebnikov…
This patch adds links from page to its lruvec and from lruvec to its zone and node. If
CONFIG_CGROUP_MEM_RES_CTLR=nthey justpage_zone()andcontainer_of().Signed-off-by Konstantin Khlebnikov <[email protected]>
small comments in below.
Quoted 72 lines
--- include/linux/mm.h | 37 +++++++++++++++++++++++++++++++++++++ include/linux/mmzone.h | 12 ++++++++---- mm/internal.h | 1 + mm/memcontrol.c | 27 ++++++++++++++++++++++++--- mm/page_alloc.c | 17 ++++++++++++++--- 5 files changed, 84 insertions(+), 10 deletions(-)include/linux/mm.h +37
-0@@ -728,6 +728,43 @@ static inline void set_page_links(struct page *page, enum zone_type zone,#endif } +#ifdef CONFIG_CGROUP_MEM_RES_CTLR + +/* Multiple lruvecs in zone */ + +extern struct lruvec *page_lruvec (struct page *page); + +static inline struct zone *lruvec_zone (struct lruvec *lruvec) +{ +return lruvec->zone; +} + +static inline struct pglist_data *lruvec_node (struct lruvec *lruvec) +{ +return lruvec->node; +} + +#else /* CONFIG_CGROUP_MEM_RES_CTLR */ + +/* Single lruvec in zone */ + +static inline struct lruvec *page_lruvec (struct page *page) +{ +return &page_zone (page)->lruvec; +} + +static inline struct zone *lruvec_zone (struct lruvec *lruvec) +{ + return container_of(lruvec, struct zone, lruvec); +} + +static inline struct pglist_data *lruvec_node (struct lruvec *lruvec) +{ +return lruvec_zone (lruvec)->zone_pgdat; +} + +#endif /* CONFIG_CGROUP_MEM_RES_CTLR */ +/* * Some inline functions in vmstat.h depend on page_zone() */ include/linux/mmzone.h +0
-4@@ -159,10 +159,6 @@ static inline int is_unevictable_lru(enum lru_list lru)return (lru ==LRU_UNEVICTABLE ); } -struct lruvec { -struct list_head pages_lru[NR_LRU_LISTS ]; -}; -/* Mask used at gathering information at once (see memcontrol.c) */ #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
include/linux/mmzone.h struct zone_reclaim_stat { · line 304
Quoted 9 lines
include/linux/mmzone.h +6
-0@@ -300,6 +296,14 @@ struct zone_reclaim_stat {unsigned long recent_scanned[2 ]; }; +struct lruvec { +struct list_head pages_lru[NR_LRU_LISTS ]; +#ifdef CONFIG_CGROUP_MEM_RES_CTLR +struct zone *zone; +struct pglist_data *node; +#endif
I don't think this #ifdef is very good ....this adds other #ifdefs in other headers. How bad if we remove this #ifdef and use ->zone, ->pgdat in lruvec_zone, lruvec_page always ?
There may be concerns to fit lruvec at el into cache-line...but this set will add a (big) hash here later..
I'm sorry if you're asked to add this #ifdef in v1 or v2.
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 06/21] mm: lruvec linking functions
Quoted 85 lines · KAMEZAWA Hiroyuki
Quoted 4 lines · On Thu, 23 Feb 2012 17:52:04 +0400 Konstantin…
This patch adds links from page to its lruvec and from lruvec to its zone and node. If
CONFIG_CGROUP_MEM_RES_CTLR=nthey justpage_zone()andcontainer_of().Signed-off-by Konstantin Khlebnikov<[email protected]>small comments in below.
Quoted 75 lines
--- include/linux/mm.h | 37 +++++++++++++++++++++++++++++++++++++ include/linux/mmzone.h | 12 ++++++++---- mm/internal.h | 1 + mm/memcontrol.c | 27 ++++++++++++++++++++++++--- mm/page_alloc.c | 17 ++++++++++++++--- 5 files changed, 84 insertions(+), 10 deletions(-)include/linux/mm.h +37
-0@@ -728,6 +728,43 @@ static inline void set_page_links(struct page *page, enum zone_type zone,#endif } +#ifdef CONFIG_CGROUP_MEM_RES_CTLR + +/* Multiple lruvecs in zone */ + +extern struct lruvec *page_lruvec (struct page *page); + +static inline struct zone *lruvec_zone (struct lruvec *lruvec) +{ +return lruvec->zone; +} + +static inline struct pglist_data *lruvec_node (struct lruvec *lruvec) +{ +return lruvec->node; +} + +#else /* CONFIG_CGROUP_MEM_RES_CTLR */ + +/* Single lruvec in zone */ + +static inline struct lruvec *page_lruvec (struct page *page) +{ +return &page_zone (page)->lruvec; +} + +static inline struct zone *lruvec_zone (struct lruvec *lruvec) +{ + return container_of(lruvec, struct zone, lruvec); +} + +static inline struct pglist_data *lruvec_node (struct lruvec *lruvec) +{ +return lruvec_zone (lruvec)->zone_pgdat; +} + +#endif /* CONFIG_CGROUP_MEM_RES_CTLR */ +/* * Some inline functions in vmstat.h depend on page_zone() */ include/linux/mmzone.h +0
-4@@ -159,10 +159,6 @@ static inline int is_unevictable_lru(enum lru_list lru)return (lru ==LRU_UNEVICTABLE ); } -struct lruvec { -struct list_head pages_lru[NR_LRU_LISTS ]; -}; -/* Mask used at gathering information at once (see memcontrol.c) */ #define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))@@ -300,6 +296,14 @@ struct zone_reclaim_stat {unsigned long recent_scanned[2 ]; };
Quoted 10 lines
include/linux/mmzone.h struct zone_reclaim_stat { · line 304
Quoted 6 lines
include/linux/mmzone.h +6
-0+struct lruvec { +struct list_head pages_lru[NR_LRU_LISTS ]; +#ifdef CONFIG_CGROUP_MEM_RES_CTLR +struct zone *zone; +struct pglist_data *node; +#endif I don't think this #ifdef is very good ....this adds other #ifdefs in other headers. How bad if we remove this #ifdef and use ->zone, ->pgdat in
lruvec_zone,lruvec_pagealways ?
This adds one dereference in lruvec_zone() if memcg is disabled in config.
We can remove ifdef from declaration and initialization, but keep optimized variant of lruvec_zone()
Quoted 9 lines
There may be concerns to fit lruvec at el into cache-line...but this set will add a (big) hash here later..
I'm sorry if you're asked to add this #ifdef in v1 or v2.
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 7/21 mm: add lruvec->pages_count
In reply to Konstantin KhlebnikovMove lru pages counter from mem_cgroup_per_zone->count[] to lruvec->pages_count[]
Account pages in all lruvecs, incuding root,
this isn't a huge overhead, but it greatly simplifies all code.
Redundant page_lruvec() calls will be optimized in further patches.
--- include/linux/memcontrol.h | 29 -------------- include/linux/mm_inline.h | 15 +++++-- include/linux/mmzone.h | 1 mm/memcontrol.c | 93 +------------------------------------------- mm/swap.c | 7 +-- mm/vmscan.c | 25 +++++++++--- 6 files changed, 34 insertions(+), 136 deletions(-)
include/linux/memcontrol.h +0 -29
@@ -63,12 +63,6 @@ extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);struct lruvec *mem_cgroup_zone_lruvec (struct zone *,struct mem_cgroup *); -struct lruvec *mem_cgroup_lru_add_list (struct zone *,struct page *, -enum lru_list ); -void mem_cgroup_lru_del_list (struct page *,enum lru_list ); -void mem_cgroup_lru_del (struct page *); -struct lruvec *mem_cgroup_lru_move_lists (struct zone *,struct page *, -enum lru_list ,enum lru_list );/* For coalescing uncharge for reducing memcg' overhead*/ extern void mem_cgroup_uncharge_start (void );
@@ -212,29 +206,6 @@ static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,return &zone->lruvec; } -static inline struct lruvec *mem_cgroup_lru_add_list (struct zone *zone, -struct page *page, -enum lru_list lru) -{ -return &zone->lruvec; -} - -static inline void mem_cgroup_lru_del_list (struct page *page,enum lru_list lru) -{ -} - -static inline void mem_cgroup_lru_del (struct page *page) -{ -} - -static inline struct lruvec *mem_cgroup_lru_move_lists (struct zone *zone, -struct page *page, -enum lru_list from, -enum lru_list to) -{ -return &zone->lruvec; -} -static inline struct mem_cgroup *try_get_mem_cgroup_from_page (struct page *page) {return NULL ;
include/linux/mm_inline.h +10 -5
@@ -24,19 +24,24 @@ static inline int page_is_file_cache(struct page *page)static inline void add_page_to_lru_list (struct zone *zone,struct page *page,enum lru_list lru) { -struct lruvec *lruvec; +struct lruvec *lruvec =page_lruvec (page); +int numpages =hpage_nr_pages (page); - lruvec =mem_cgroup_lru_add_list (zone, page, lru);list_add (&page->lru, &lruvec->pages_lru[lru]); -__mod_zone_page_state (zone,NR_LRU_BASE + lru,hpage_nr_pages (page)); + lruvec->pages_count[lru] += numpages; +__mod_zone_page_state (zone,NR_LRU_BASE + lru, numpages); }static inline void del_page_from_lru_list (struct zone *zone,struct page *page,enum lru_list lru) { -mem_cgroup_lru_del_list (page, lru); +struct lruvec *lruvec =page_lruvec (page); +int numpages =hpage_nr_pages (page); +list_del (&page->lru); -__mod_zone_page_state (zone,NR_LRU_BASE + lru, -hpage_nr_pages (page)); + lruvec->pages_count[lru] -= numpages; +VM_BUG_ON ((long )lruvec->pages_count[lru] <0 ); +__mod_zone_page_state (zone,NR_LRU_BASE + lru, -numpages); }/**
include/linux/mmzone.h +1 -0
@@ -298,6 +298,7 @@ struct zone_reclaim_stat {struct lruvec {struct list_head pages_lru[NR_LRU_LISTS ]; +unsigned long pages_count[NR_LRU_LISTS ];#ifdef CONFIG_CGROUP_MEM_RES_CTLR struct zone *zone;struct pglist_data *node;
mm/memcontrol.c +2 -91
117 lines not on this page · raw message
mm/swap.c +2 -5
@@ -234,10 +234,8 @@ static void pagevec_move_tail_fn(struct page *page, void *arg)if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) {enum lru_list lru =page_lru_base_type (page); -struct lruvec *lruvec; +struct lruvec *lruvec =page_lruvec (page); - lruvec =mem_cgroup_lru_move_lists (page_zone (page), - page, lru, lru);list_move_tail (&page->lru, &lruvec->pages_lru[lru]); (*pgmoved)++; }
@@ -476,12 +474,11 @@ static void lru_deactivate_fn(struct page *page, void *arg)*/ SetPageReclaim (page); }else { -struct lruvec *lruvec; +struct lruvec *lruvec =page_lruvec (page);/* * The page's writeback ends up during pagevec * We moves tha page into tail of inactive. */ - lruvec =mem_cgroup_lru_move_lists (zone, page, lru, lru);list_move_tail (&page->lru, &lruvec->pages_lru[lru]);__count_vm_event (PGROTATED ); }
mm/vmscan.c +19 -6
@@ -1180,7 +1180,6 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,switch (__isolate_lru_page (page, mode, file)) {case 0 : -mem_cgroup_lru_del (page);list_move (&page->lru, dst); nr_taken +=hpage_nr_pages (page);break ;
@@ -1238,10 +1237,16 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,if (__isolate_lru_page (cursor_page, mode, file) ==0 ) {unsigned int isolated_pages; +struct lruvec *cursor_lruvec; +int cursor_lru =page_lru (cursor_page); -mem_cgroup_lru_del (cursor_page);list_move (&cursor_page->lru, dst); isolated_pages =hpage_nr_pages (cursor_page); + cursor_lruvec =page_lruvec (cursor_page); + cursor_lruvec->pages_count[cursor_lru] -= + isolated_pages; +VM_BUG_ON ((long )cursor_lruvec-> + pages_count[cursor_lru] <0 ); nr_taken += isolated_pages; nr_lumpy_taken += isolated_pages;if (PageDirty (cursor_page))
@@ -1273,6 +1278,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, nr_lumpy_failed++; } + lruvec->pages_count[lru] -= nr_taken - nr_lumpy_taken; +VM_BUG_ON ((long )lruvec->pages_count[lru] <0 ); + *nr_scanned = scan; trace_mm_vmscan_lru_isolate(sc->order,
@@ -1656,15 +1664,18 @@ static void move_active_pages_to_lru(struct zone *zone,while (!list_empty (list)) {struct lruvec *lruvec; +int numpages; page =lru_to_page (list);VM_BUG_ON (PageLRU (page));SetPageLRU (page); - lruvec =mem_cgroup_lru_add_list (zone, page, lru); + lruvec =page_lruvec (page);list_move (&page->lru, &lruvec->pages_lru[lru]); - pgmoved +=hpage_nr_pages (page); + numpages =hpage_nr_pages (page); + lruvec->pages_count[lru] += numpages; + pgmoved += numpages;if (put_page_testzero (page)) {__ClearPageLRU (page);
@@ -3590,8 +3601,10 @@ void check_move_unevictable_pages(struct page **pages, int nr_pages)VM_BUG_ON (PageActive (page));ClearPageUnevictable (page);__dec_zone_state (zone,NR_UNEVICTABLE ); - lruvec =mem_cgroup_lru_move_lists (zone, page, -LRU_UNEVICTABLE , lru); + lruvec =page_lruvec (page); + lruvec->pages_count[LRU_UNEVICTABLE ]--; +VM_BUG_ON ((long )lruvec->pages_count[LRU_UNEVICTABLE ] <0 ); + lruvec->pages_count[lru]++;list_move (&page->lru, &lruvec->pages_lru[lru]);__inc_zone_state (zone,NR_INACTIVE_ANON + lru); pgrescued++;
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 07/21] mm: add lruvec->pages_count
Quoted 8 lines · On Thu, 23 Feb 2012 17:52:08 +0400 Konstantin Khlebnikov…
Move lru pages counter from mem_cgroup_per_zone->count[] to lruvec->pages_count[]
Account pages in all lruvecs, incuding root,
this isn't a huge overhead, but it greatly simplifies all code.Redundant
page_lruvec()calls will be optimized in further patches.Signed-off-by Konstantin Khlebnikov <[email protected]>
Hmm, I like this but..a question below.
Quoted 95 lines
--- include/linux/memcontrol.h | 29 -------------- include/linux/mm_inline.h | 15 +++++-- include/linux/mmzone.h | 1 mm/memcontrol.c | 93 +------------------------------------------- mm/swap.c | 7 +-- mm/vmscan.c | 25 +++++++++--- 6 files changed, 34 insertions(+), 136 deletions(-)include/linux/memcontrol.h +0
-29@@ -63,12 +63,6 @@ extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);struct lruvec *mem_cgroup_zone_lruvec (struct zone *,struct mem_cgroup *); -struct lruvec *mem_cgroup_lru_add_list (struct zone *,struct page *, -enum lru_list ); -void mem_cgroup_lru_del_list (struct page *,enum lru_list ); -void mem_cgroup_lru_del (struct page *); -struct lruvec *mem_cgroup_lru_move_lists (struct zone *,struct page *, -enum lru_list ,enum lru_list );/* For coalescing uncharge for reducing memcg' overhead*/ extern void mem_cgroup_uncharge_start (void );@@ -212,29 +206,6 @@ static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,return &zone->lruvec; } -static inline struct lruvec *mem_cgroup_lru_add_list (struct zone *zone, -struct page *page, -enum lru_list lru) -{ -return &zone->lruvec; -} - -static inline void mem_cgroup_lru_del_list (struct page *page,enum lru_list lru) -{ -} - -static inline void mem_cgroup_lru_del (struct page *page) -{ -} - -static inline struct lruvec *mem_cgroup_lru_move_lists (struct zone *zone, -struct page *page, -enum lru_list from, -enum lru_list to) -{ -return &zone->lruvec; -} -static inline struct mem_cgroup *try_get_mem_cgroup_from_page (struct page *page) {return NULL ;include/linux/mm_inline.h +10
-5@@ -24,19 +24,24 @@ static inline int page_is_file_cache(struct page *page)static inline void add_page_to_lru_list (struct zone *zone,struct page *page,enum lru_list lru) { -struct lruvec *lruvec; +struct lruvec *lruvec =page_lruvec (page); +int numpages =hpage_nr_pages (page); - lruvec =mem_cgroup_lru_add_list (zone, page, lru);list_add (&page->lru, &lruvec->pages_lru[lru]); -__mod_zone_page_state (zone,NR_LRU_BASE + lru,hpage_nr_pages (page)); + lruvec->pages_count[lru] += numpages; +__mod_zone_page_state (zone,NR_LRU_BASE + lru, numpages); }static inline void del_page_from_lru_list (struct zone *zone,struct page *page,enum lru_list lru) { -mem_cgroup_lru_del_list (page, lru); +struct lruvec *lruvec =page_lruvec (page); +int numpages =hpage_nr_pages (page); +list_del (&page->lru); -__mod_zone_page_state (zone,NR_LRU_BASE + lru, -hpage_nr_pages (page)); + lruvec->pages_count[lru] -= numpages; +VM_BUG_ON ((long )lruvec->pages_count[lru] <0 ); +__mod_zone_page_state (zone,NR_LRU_BASE + lru, -numpages); }/** include/linux/mmzone.h +0
-0
include/linux/mmzone.h struct zone_reclaim_stat { · line 301
Quoted 4 lines
include/linux/mmzone.h +1
-0@@ -298,6 +298,7 @@ struct zone_reclaim_stat {struct lruvec {struct list_head pages_lru[NR_LRU_LISTS ]; +unsigned long pages_count[NR_LRU_LISTS ];
In this time, you don't put the objects under #ifdef...why ?
How do you handle duplication "the number of pages in LRU" of zone->vm_stat and this ?
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 07/21] mm: add lruvec->pages_count
Quoted 110 lines · KAMEZAWA Hiroyuki
Quoted 8 lines · On Thu, 23 Feb 2012 17:52:08 +0400 Konstantin…
Move lru pages counter from mem_cgroup_per_zone->count[] to lruvec->pages_count[]
Account pages in all lruvecs, incuding root,
this isn't a huge overhead, but it greatly simplifies all code.Redundant
page_lruvec()calls will be optimized in further patches.Signed-off-by Konstantin Khlebnikov<[email protected]>Hmm, I like this but..a question below.
Quoted 96 lines
--- include/linux/memcontrol.h | 29 -------------- include/linux/mm_inline.h | 15 +++++-- include/linux/mmzone.h | 1 mm/memcontrol.c | 93 +------------------------------------------- mm/swap.c | 7 +-- mm/vmscan.c | 25 +++++++++--- 6 files changed, 34 insertions(+), 136 deletions(-)include/linux/memcontrol.h +0
-29@@ -63,12 +63,6 @@ extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);struct lruvec *mem_cgroup_zone_lruvec (struct zone *,struct mem_cgroup *); -struct lruvec *mem_cgroup_lru_add_list (struct zone *,struct page *, -enum lru_list ); -void mem_cgroup_lru_del_list (struct page *,enum lru_list ); -void mem_cgroup_lru_del (struct page *); -struct lruvec *mem_cgroup_lru_move_lists (struct zone *,struct page *, -enum lru_list ,enum lru_list );/* For coalescing uncharge for reducing memcg' overhead*/ extern void mem_cgroup_uncharge_start (void );@@ -212,29 +206,6 @@ static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,return &zone->lruvec; } -static inline struct lruvec *mem_cgroup_lru_add_list (struct zone *zone, -struct page *page, -enum lru_list lru) -{ -return &zone->lruvec; -} - -static inline void mem_cgroup_lru_del_list (struct page *page,enum lru_list lru) -{ -} - -static inline void mem_cgroup_lru_del (struct page *page) -{ -} - -static inline struct lruvec *mem_cgroup_lru_move_lists (struct zone *zone, -struct page *page, -enum lru_list from, -enum lru_list to) -{ -return &zone->lruvec; -} -static inline struct mem_cgroup *try_get_mem_cgroup_from_page (struct page *page) {return NULL ;include/linux/mm_inline.h +10
-5@@ -24,19 +24,24 @@ static inline int page_is_file_cache(struct page *page)static inline void add_page_to_lru_list (struct zone *zone,struct page *page,enum lru_list lru) { -struct lruvec *lruvec; +struct lruvec *lruvec =page_lruvec (page); +int numpages =hpage_nr_pages (page); - lruvec =mem_cgroup_lru_add_list (zone, page, lru);list_add (&page->lru,&lruvec->pages_lru[lru]); -__mod_zone_page_state (zone,NR_LRU_BASE + lru,hpage_nr_pages (page)); + lruvec->pages_count[lru] += numpages; +__mod_zone_page_state (zone,NR_LRU_BASE + lru, numpages); }static inline void del_page_from_lru_list (struct zone *zone,struct page *page,enum lru_list lru) { -mem_cgroup_lru_del_list (page, lru); +struct lruvec *lruvec =page_lruvec (page); +int numpages =hpage_nr_pages (page); +list_del (&page->lru); -__mod_zone_page_state (zone,NR_LRU_BASE + lru, -hpage_nr_pages (page)); + lruvec->pages_count[lru] -= numpages; +VM_BUG_ON ((long )lruvec->pages_count[lru]<0 ); +__mod_zone_page_state (zone,NR_LRU_BASE + lru, -numpages); }/** include/linux/mmzone.h +0
-0@@ -298,6 +298,7 @@ struct zone_reclaim_stat {
Quoted 5 lines
include/linux/mmzone.h struct zone_reclaim_stat { · line 301
Quoted 3 lines
include/linux/mmzone.h +1
-0struct lruvec {struct list_head pages_lru[NR_LRU_LISTS ]; +unsigned long pages_count[NR_LRU_LISTS ];In this time, you don't put the objects under #ifdef...why ?
It will make the code much uglier and does not speed up it at all.
Quoted 2 lines
How do you handle duplication "the number of pages in LRU" of
zone->vm_statand this ?
I don't think this is totally bad, vmstat usually has per-cpu drift, these numbers will be exact.
Quoted 4 lines
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 8/21 mm: unify inactive_list_is_low()
In reply to Konstantin KhlebnikovUnify memcg and non-memcg logic, always use exact counters from struct lruvec.
--- mm/vmscan.c | 30 ++++++++---------------------- 1 files changed, 8 insertions(+), 22 deletions(-)
mm/vmscan.c +8 -22
@@ -1822,6 +1822,7 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz) {unsigned long active, inactive;unsigned int gb, ratio; +struct lruvec *lruvec;/* * If we don't have swap space, anonymous page deactivation
@@ -1830,17 +1831,9 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz)if (!total_swap_pages)return 0 ; -if (scanning_global_lru (mz)) { - active =zone_page_state (mz->zone,NR_ACTIVE_ANON ); - inactive =zone_page_state (mz->zone,NR_INACTIVE_ANON ); - }else { - active =mem_cgroup_zone_nr_lru_pages (mz->mem_cgroup, -zone_to_nid (mz->zone),zone_idx (mz->zone), -BIT (LRU_ACTIVE_ANON )); - inactive =mem_cgroup_zone_nr_lru_pages (mz->mem_cgroup, -zone_to_nid (mz->zone),zone_idx (mz->zone), -BIT (LRU_INACTIVE_ANON )); - } + lruvec =mem_cgroup_zone_lruvec (mz->zone, mz->mem_cgroup); + active = lruvec->pages_count[LRU_ACTIVE_ANON ]; + inactive = lruvec->pages_count[LRU_INACTIVE_ANON ];/* Total size in gigabytes */ gb = (active + inactive) >> (30 -PAGE_SHIFT );
@@ -1875,18 +1868,11 @@ static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)static int inactive_file_is_low (struct mem_cgroup_zone *mz) {unsigned long active, inactive; +struct lruvec *lruvec; -if (scanning_global_lru (mz)) { - active =zone_page_state (mz->zone,NR_ACTIVE_FILE ); - inactive =zone_page_state (mz->zone,NR_INACTIVE_FILE ); - }else { - active =mem_cgroup_zone_nr_lru_pages (mz->mem_cgroup, -zone_to_nid (mz->zone),zone_idx (mz->zone), -BIT (LRU_ACTIVE_FILE )); - inactive =mem_cgroup_zone_nr_lru_pages (mz->mem_cgroup, -zone_to_nid (mz->zone),zone_idx (mz->zone), -BIT (LRU_INACTIVE_FILE )); - } + lruvec =mem_cgroup_zone_lruvec (mz->zone, mz->mem_cgroup); + active = lruvec->pages_count[LRU_ACTIVE_FILE ]; + inactive = lruvec->pages_count[LRU_INACTIVE_FILE ];return inactive < active; }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 08/21] mm: unify inactive_list_is_low()
Quoted 3 lines · On Thu, 23 Feb 2012 17:52:19 +0400 Konstantin Khlebnikov…
Unify memcg and non-memcg logic, always use exact counters from struct lruvec.
Signed-off-by Konstantin Khlebnikov <[email protected]>
Nice.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 9/21 mm: add lruvec->reclaim_stat
In reply to Konstantin KhlebnikovMerge memcg and non-memcg reclaim stat. We need to update only one. Move zone->reclaimer_stat and mem_cgroup_per_zone->reclaimer_stat to struct lruvec.
struct lruvec will become operating unit for recalimer logic, thus this is perfect place for these counters.
--- include/linux/memcontrol.h | 17 -------------- include/linux/mmzone.h | 5 +++- mm/memcontrol.c | 52 +++++--------------------------------------- mm/page_alloc.c | 4 --- mm/swap.c | 12 ++-------- mm/vmscan.c | 5 +++- 6 files changed, 14 insertions(+), 81 deletions(-)
include/linux/memcontrol.h +0 -17
@@ -103,10 +103,6 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);int mem_cgroup_select_victim_node (struct mem_cgroup *memcg);unsigned long mem_cgroup_zone_nr_lru_pages (struct mem_cgroup *memcg,int nid,int zid,unsigned int lrumask); -struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat (struct mem_cgroup *memcg, -struct zone *zone); -struct zone_reclaim_stat * -mem_cgroup_get_reclaim_stat_from_page (struct page *page);extern void mem_cgroup_print_oom_info (struct mem_cgroup *memcg,struct task_struct *p); extern void mem_cgroup_replace_page_cache(struct page *oldpage,
@@ -286,19 +282,6 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,return 0 ; } - -static inline struct zone_reclaim_stat * -mem_cgroup_get_reclaim_stat (struct mem_cgroup *memcg,struct zone *zone) -{ -return NULL ; -} - -static inline struct zone_reclaim_stat * -mem_cgroup_get_reclaim_stat_from_page (struct page *page) -{ -return NULL ; -} -static inline void mem_cgroup_print_oom_info (struct mem_cgroup *memcg,struct task_struct *p) {
include/linux/mmzone.h +3 -2
@@ -299,6 +299,9 @@ struct zone_reclaim_stat {struct lruvec {struct list_head pages_lru[NR_LRU_LISTS ];unsigned long pages_count[NR_LRU_LISTS ]; + +struct zone_reclaim_stat reclaim_stat; +#ifdef CONFIG_CGROUP_MEM_RES_CTLR struct zone *zone;struct pglist_data *node;
@@ -379,8 +382,6 @@ struct zone {spinlock_t lru_lock;struct lruvec lruvec; -struct zone_reclaim_stat reclaim_stat; -unsigned long pages_scanned;/* since last reclaim */ unsigned long flags;/* zone flags, see below */
mm/memcontrol.c +6 -46
83 lines not on this page · raw message
mm/page_alloc.c +0 -4
@@ -4375,10 +4375,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,zone_pcp_init (zone);init_zone_lruvec (zone, &zone->lruvec); - zone->reclaim_stat.recent_rotated[0 ] =0 ; - zone->reclaim_stat.recent_rotated[1 ] =0 ; - zone->reclaim_stat.recent_scanned[0 ] =0 ; - zone->reclaim_stat.recent_scanned[1 ] =0 ;zap_zone_vm_stats (zone); zone->flags =0 ;if (!size)
mm/swap.c +2 -10
@@ -277,21 +277,13 @@ void rotate_reclaimable_page(struct page *page)static void update_page_reclaim_stat (struct zone *zone,struct page *page,int file,int rotated) { -struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat; -struct zone_reclaim_stat *memcg_reclaim_stat; +struct zone_reclaim_stat *reclaim_stat; - memcg_reclaim_stat =mem_cgroup_get_reclaim_stat_from_page (page); + reclaim_stat = &page_lruvec (page)->reclaim_stat; reclaim_stat->recent_scanned[file]++;if (rotated) reclaim_stat->recent_rotated[file]++; - -if (!memcg_reclaim_stat) -return ; - - memcg_reclaim_stat->recent_scanned[file]++; -if (rotated) - memcg_reclaim_stat->recent_rotated[file]++; }static void __activate_page (struct page *page,void *arg)
mm/vmscan.c +3 -2
@@ -184,9 +184,10 @@ static bool scanning_global_lru(struct mem_cgroup_zone *mz)static struct zone_reclaim_stat *get_reclaim_stat (struct mem_cgroup_zone *mz) {if (!scanning_global_lru (mz)) -return mem_cgroup_get_reclaim_stat (mz->mem_cgroup, mz->zone); +return &mem_cgroup_zone_lruvec (mz->zone, + mz->mem_cgroup)->reclaim_stat; -return &mz->zone->reclaim_stat; +return &mz->zone->lruvec.reclaim_stat; } static unsigned long zone_nr_lru_pages(struct mem_cgroup_zone *mz,
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 09/21] mm: add lruvec->reclaim_stat
Quoted 7 lines · On Thu, 23 Feb 2012 17:52:24 +0400 Konstantin Khlebnikov…
Merge memcg and non-memcg reclaim stat. We need to update only one. Move
zone->reclaimer_statandmem_cgroup_per_zone->reclaimer_statto struct lruvec.struct lruvec will become operating unit for recalimer logic, thus this is perfect place for these counters.
Signed-off-by Konstantin Khlebnikov <[email protected]>
I like this.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 10/21 mm: kill struct mem_cgroup_zone
In reply to Konstantin Khlebnikovstruct mem_cgroup_zone always points to one lruvec, either root zone->lruvec or to some from memcg. So this fancy pointer can be replaced with direct pointer to struct lruvec, because all required infromation already collected on lruvec.
--- mm/vmscan.c | 186 ++++++++++++++++++++++------------------------------------- 1 files changed, 70 insertions(+), 116 deletions(-)
mm/vmscan.c +70 -116
@@ -115,11 +115,6 @@ struct scan_control {nodemask_t *nodemask; }; -struct mem_cgroup_zone { -struct mem_cgroup *mem_cgroup; -struct zone *zone; -}; -#define lru_to_page (_head) (list_entry((_head)->prev, struct page, lru))#ifdef ARCH_HAS_PREFETCH
@@ -164,45 +159,13 @@ static bool global_reclaim(struct scan_control *sc) {return !sc->target_mem_cgroup; } - -static bool scanning_global_lru (struct mem_cgroup_zone *mz) -{ -return !mz->mem_cgroup; -}#else static bool global_reclaim (struct scan_control *sc) {return true; } - -static bool scanning_global_lru (struct mem_cgroup_zone *mz) -{ -return true; -}#endif -static struct zone_reclaim_stat *get_reclaim_stat (struct mem_cgroup_zone *mz) -{ -if (!scanning_global_lru (mz)) -return &mem_cgroup_zone_lruvec (mz->zone, - mz->mem_cgroup)->reclaim_stat; - -return &mz->zone->lruvec.reclaim_stat; -} - -static unsigned long zone_nr_lru_pages (struct mem_cgroup_zone *mz, -enum lru_list lru) -{ -if (!scanning_global_lru (mz)) -return mem_cgroup_zone_nr_lru_pages (mz->mem_cgroup, -zone_to_nid (mz->zone), -zone_idx (mz->zone), -BIT (lru)); - -return zone_page_state (mz->zone,NR_LRU_BASE + lru); -} - -/* * Add a shrinker callback to be called from the vm */
@@ -764,7 +727,7 @@ static enum page_references page_check_references(struct page *page,* shrink_page_list() returns the number of reclaimed pages */ static unsigned long shrink_page_list(struct list_head *page_list, - struct mem_cgroup_zone *mz, + struct lruvec *lruvec, struct scan_control *sc, int priority, unsigned long *ret_nr_dirty,
@@ -795,7 +758,7 @@ static unsigned long shrink_page_list(struct list_head *page_list, goto keep;VM_BUG_ON (PageActive (page)); -VM_BUG_ON (page_zone (page) != mz->zone); +VM_BUG_ON (page_zone (page) !=lruvec_zone (lruvec)); sc->nr_scanned++;
@@ -1021,7 +984,7 @@ keep_lumpy:* will encounter the same problem */ if (nr_dirty && nr_dirty == nr_congested &&global_reclaim (sc)) -zone_set_flag (mz->zone,ZONE_CONGESTED ); +zone_set_flag (lruvec_zone (lruvec),ZONE_CONGESTED );free_hot_cold_page_list (&free_pages,1 );
@@ -1136,7 +1099,7 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode, int file)* Appropriate locks must be held before calling this function. * * @nr_to_scan: The number of pages to look through on the list. -* @mz: The mem_cgroup_zone to pull pages from. +* @lruvec The struct lruvec to pull pages from. * @dst: The temp list to put pages on to. * @nr_scanned: The number of pages that were scanned. * @sc: The scan_control struct for this reclaim session
@@ -1147,11 +1110,10 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode, int file)* returns how many pages were moved onto *@dst. */ static unsigned long isolate_lru_pages (unsigned long nr_to_scan, -struct mem_cgroup_zone *mz,struct list_head *dst, +struct lruvec *lruvec,struct list_head *dst,unsigned long *nr_scanned,struct scan_control *sc,isolate_mode_t mode,int active,int file) { -struct lruvec *lruvec;struct list_head *src;unsigned long nr_taken =0 ;unsigned long nr_lumpy_taken =0 ;
@@ -1160,7 +1122,6 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,unsigned long scan;int lru =LRU_BASE ; - lruvec =mem_cgroup_zone_lruvec (mz->zone, mz->mem_cgroup);if (active) lru +=LRU_ACTIVE ;if (file)
@@ -1366,11 +1327,11 @@ static int too_many_isolated(struct zone *zone, int file, } static noinline_for_stack void -putback_inactive_pages (struct mem_cgroup_zone *mz, +putback_inactive_pages (struct lruvec *lruvec,struct list_head *page_list) { -struct zone_reclaim_stat *reclaim_stat =get_reclaim_stat (mz); -struct zone *zone = mz->zone; +struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; +struct zone *zone =lruvec_zone (lruvec);LIST_HEAD (pages_to_free);/*
@@ -1417,13 +1378,13 @@ putback_inactive_pages(struct mem_cgroup_zone *mz, } static noinline_for_stack void -update_isolated_counts (struct mem_cgroup_zone *mz, +update_isolated_counts (struct lruvec *lruvec,struct list_head *page_list,unsigned long *nr_anon,unsigned long *nr_file) { -struct zone_reclaim_stat *reclaim_stat =get_reclaim_stat (mz); -struct zone *zone = mz->zone; +struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; +struct zone *zone =lruvec_zone (lruvec);unsigned int count[NR_LRU_LISTS ] = {0 , };unsigned long nr_active =0 ;struct page *page;
@@ -1507,7 +1468,7 @@ static inline bool should_reclaim_stall(unsigned long nr_taken,* of reclaimed pages */ static noinline_for_stack unsigned long -shrink_inactive_list (unsigned long nr_to_scan,struct mem_cgroup_zone *mz, +shrink_inactive_list (unsigned long nr_to_scan,struct lruvec *lruvec,struct scan_control *sc,int priority,int file) {LIST_HEAD (page_list);
@@ -1519,7 +1480,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,unsigned long nr_dirty =0 ;unsigned long nr_writeback =0 ;isolate_mode_t isolate_mode =ISOLATE_INACTIVE ; -struct zone *zone = mz->zone; +struct zone *zone =lruvec_zone (lruvec);while (unlikely (too_many_isolated (zone, file, sc))) {congestion_wait (BLK_RW_ASYNC ,HZ /10 );
@@ -1542,8 +1503,9 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,spin_lock_irq (&zone->lru_lock); - nr_taken =isolate_lru_pages (nr_to_scan, mz, &page_list, &nr_scanned, - sc, isolate_mode,0 , file); + nr_taken =isolate_lru_pages (nr_to_scan, lruvec, &page_list, + &nr_scanned, sc, isolate_mode,0 , file); +if (global_reclaim (sc)) { zone->pages_scanned += nr_scanned;if (current_is_kswapd ())
@@ -1559,20 +1521,20 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,return 0 ; } -update_isolated_counts (mz, &page_list, &nr_anon, &nr_file); +update_isolated_counts (lruvec, &page_list, &nr_anon, &nr_file);__mod_zone_page_state (zone,NR_ISOLATED_ANON , nr_anon);__mod_zone_page_state (zone,NR_ISOLATED_FILE , nr_file);spin_unlock_irq (&zone->lru_lock); - nr_reclaimed =shrink_page_list (&page_list, mz, sc, priority, + nr_reclaimed =shrink_page_list (&page_list, lruvec, sc, priority, &nr_dirty, &nr_writeback);/* Check if we should syncronously wait for writeback */ if (should_reclaim_stall (nr_taken, nr_reclaimed, priority, sc)) {set_reclaim_mode (priority, sc, true); - nr_reclaimed +=shrink_page_list (&page_list, mz, sc, + nr_reclaimed +=shrink_page_list (&page_list, lruvec, sc, priority, &nr_dirty, &nr_writeback); }
@@ -1582,7 +1544,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct mem_cgroup_zone *mz,__count_vm_events (KSWAPD_STEAL , nr_reclaimed);__count_zone_vm_events (PGSTEAL , zone, nr_reclaimed); -putback_inactive_pages (mz, &page_list); +putback_inactive_pages (lruvec, &page_list);__mod_zone_page_state (zone,NR_ISOLATED_ANON , -nr_anon);__mod_zone_page_state (zone,NR_ISOLATED_FILE , -nr_file);
@@ -1697,7 +1659,7 @@ static void move_active_pages_to_lru(struct zone *zone, }static void shrink_active_list (unsigned long nr_to_scan, -struct mem_cgroup_zone *mz, +struct lruvec *lruvec,struct scan_control *sc,int priority,int file) {
@@ -1708,10 +1670,10 @@ static void shrink_active_list(unsigned long nr_to_scan,LIST_HEAD (l_active);LIST_HEAD (l_inactive);struct page *page; -struct zone_reclaim_stat *reclaim_stat =get_reclaim_stat (mz); +struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;unsigned long nr_rotated =0 ;isolate_mode_t isolate_mode =ISOLATE_ACTIVE ; -struct zone *zone = mz->zone; +struct zone *zone =lruvec_zone (lruvec);lru_add_drain ();
@@ -1722,8 +1684,9 @@ static void shrink_active_list(unsigned long nr_to_scan,spin_lock_irq (&zone->lru_lock); - nr_taken =isolate_lru_pages (nr_to_scan, mz, &l_hold, &nr_scanned, sc, - isolate_mode,1 , file); + nr_taken =isolate_lru_pages (nr_to_scan, lruvec, &l_hold, &nr_scanned, + sc, isolate_mode,1 , file); +if (global_reclaim (sc)) zone->pages_scanned += nr_scanned;
@@ -1819,11 +1782,10 @@ static void shrink_active_list(unsigned long nr_to_scan,* 1TB 101 10GB * 10TB 320 32GB */ -static int inactive_anon_is_low (struct mem_cgroup_zone *mz) +static int inactive_anon_is_low (struct lruvec *lruvec) {unsigned long active, inactive;unsigned int gb, ratio; -struct lruvec *lruvec;/* * If we don't have swap space, anonymous page deactivation
@@ -1832,7 +1794,6 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz)if (!total_swap_pages)return 0 ; - lruvec =mem_cgroup_zone_lruvec (mz->zone, mz->mem_cgroup); active = lruvec->pages_count[LRU_ACTIVE_ANON ]; inactive = lruvec->pages_count[LRU_INACTIVE_ANON ];
@@ -1846,7 +1807,7 @@ static int inactive_anon_is_low(struct mem_cgroup_zone *mz) return inactive * ratio < active; }#else -static inline int inactive_anon_is_low (struct mem_cgroup_zone *mz) +static inline int inactive_anon_is_low (struct lruvec *lruvec) {return 0 ; }
@@ -1866,39 +1827,38 @@ static inline int inactive_anon_is_low(struct mem_cgroup_zone *mz)* This uses a different ratio than the anonymous pages, because * the page cache uses a use-once replacement algorithm. */ -static int inactive_file_is_low (struct mem_cgroup_zone *mz) +static int inactive_file_is_low (struct lruvec *lruvec) {unsigned long active, inactive; -struct lruvec *lruvec; - lruvec =mem_cgroup_zone_lruvec (mz->zone, mz->mem_cgroup); active = lruvec->pages_count[LRU_ACTIVE_FILE ]; inactive = lruvec->pages_count[LRU_INACTIVE_FILE ];return inactive < active; } -static int inactive_list_is_low (struct mem_cgroup_zone *mz,int file) +static int inactive_list_is_low (struct lruvec *lruvec,int file) {if (file) -return inactive_file_is_low (mz); +return inactive_file_is_low (lruvec);else -return inactive_anon_is_low (mz); +return inactive_anon_is_low (lruvec); }static unsigned long shrink_list (enum lru_list lru,unsigned long nr_to_scan, -struct mem_cgroup_zone *mz, +struct lruvec *lruvec,struct scan_control *sc,int priority) {int file =is_file_lru (lru);if (is_active_lru (lru)) { -if (inactive_list_is_low (mz, file)) -shrink_active_list (nr_to_scan, mz, sc, priority, file); +if (inactive_list_is_low (lruvec, file)) +shrink_active_list (nr_to_scan, lruvec, + sc, priority, file);return 0 ; } -return shrink_inactive_list (nr_to_scan, mz, sc, priority, file); +return shrink_inactive_list (nr_to_scan, lruvec, sc, priority, file); }static int vmscan_swappiness (struct scan_control *sc)
@@ -1916,17 +1876,18 @@ static int vmscan_swappiness(struct scan_control *sc)* * nr[0] = anon pages to scan; nr[1] = file pages to scan */ -static void get_scan_count (struct mem_cgroup_zone *mz,struct scan_control *sc, +static void get_scan_count (struct lruvec *lruvec,struct scan_control *sc,unsigned long *nr,int priority) {unsigned long anon, file, free;unsigned long anon_prio, file_prio;unsigned long ap, fp; -struct zone_reclaim_stat *reclaim_stat =get_reclaim_stat (mz); +struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;u64 fraction[2 ], denominator;enum lru_list lru;int noswap =0 ;bool force_scan = false; +struct zone *zone =lruvec_zone (lruvec);/* * If the zone or memcg is small, nr[l] can be 0. This
@@ -1938,7 +1899,7 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,* latencies, so it's better to scan a minimum amount there as * well. */ -if (current_is_kswapd () && mz->zone->all_unreclaimable) +if (current_is_kswapd () && zone->all_unreclaimable) force_scan = true;if (!global_reclaim (sc)) force_scan = true;
@@ -1952,16 +1913,16 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc, goto out; } - anon =zone_nr_lru_pages (mz,LRU_ACTIVE_ANON ) + -zone_nr_lru_pages (mz,LRU_INACTIVE_ANON ); - file =zone_nr_lru_pages (mz,LRU_ACTIVE_FILE ) + -zone_nr_lru_pages (mz,LRU_INACTIVE_FILE ); + anon = lruvec->pages_count[LRU_ACTIVE_ANON ] + + lruvec->pages_count[LRU_INACTIVE_ANON ]; + file = lruvec->pages_count[LRU_ACTIVE_FILE ] + + lruvec->pages_count[LRU_INACTIVE_FILE ];if (global_reclaim (sc)) { - free =zone_page_state (mz->zone,NR_FREE_PAGES ); + free =zone_page_state (zone,NR_FREE_PAGES );/* If we have very few page cache pages, force-scan anon pages. */ -if (unlikely (file + free <=high_wmark_pages (mz->zone))) { +if (unlikely (file + free <=high_wmark_pages (zone))) { fraction[0 ] =1 ; fraction[1 ] =0 ; denominator =1 ;
@@ -1987,7 +1948,7 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc,* * anon in [0], file in [1] */ -spin_lock_irq (&mz->zone->lru_lock); +spin_lock_irq (&zone->lru_lock);if (unlikely (reclaim_stat->recent_scanned[0 ] > anon /4 )) { reclaim_stat->recent_scanned[0 ] /=2 ; reclaim_stat->recent_rotated[0 ] /=2 ;
@@ -2008,7 +1969,7 @@ static void get_scan_count(struct mem_cgroup_zone *mz, struct scan_control *sc, fp = (file_prio +1 ) * (reclaim_stat->recent_scanned[1 ] +1 ); fp /= reclaim_stat->recent_rotated[1 ] +1 ; -spin_unlock_irq (&mz->zone->lru_lock); +spin_unlock_irq (&zone->lru_lock); fraction[0 ] = ap; fraction[1 ] = fp;
@@ -2018,7 +1979,7 @@ out:int file =is_file_lru (lru);unsigned long scan; - scan =zone_nr_lru_pages (mz, lru); + scan = lruvec->pages_count[lru];if (priority || noswap) { scan >>= priority;if (!scan && force_scan)
@@ -2036,7 +1997,7 @@ out:* back to the allocator and call try_to_compact_zone(), we ensure that * there are enough free pages for it to be likely successful */ -static inline bool should_continue_reclaim (struct mem_cgroup_zone *mz, +static inline bool should_continue_reclaim (struct lruvec *lruvec,unsigned long nr_reclaimed,unsigned long nr_scanned,struct scan_control *sc)
@@ -2076,15 +2037,15 @@ static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,* inactive lists are large enough, continue reclaiming */ pages_for_compaction = (2UL << sc->order); - inactive_lru_pages =zone_nr_lru_pages (mz,LRU_INACTIVE_FILE ); + inactive_lru_pages = lruvec->pages_count[LRU_INACTIVE_FILE ];if (nr_swap_pages >0 ) - inactive_lru_pages +=zone_nr_lru_pages (mz,LRU_INACTIVE_ANON ); + inactive_lru_pages += lruvec->pages_count[LRU_INACTIVE_ANON ];if (sc->nr_reclaimed < pages_for_compaction && inactive_lru_pages > pages_for_compaction)return true;/* If compaction would go ahead or the allocation would succeed, stop */ -switch (compaction_suitable (mz->zone, sc->order)) { +switch (compaction_suitable (lruvec_zone (lruvec), sc->order)) {case COMPACT_PARTIAL :case COMPACT_CONTINUE :return false;
@@ -2096,8 +2057,8 @@ static inline bool should_continue_reclaim(struct mem_cgroup_zone *mz,/* * This is a basic per-zone page freer. Used by both kswapd and direct reclaim. */ -static void shrink_mem_cgroup_zone (int priority,struct mem_cgroup_zone *mz, -struct scan_control *sc) +static void shrink_lruvec (int priority,struct lruvec *lruvec, +struct scan_control *sc) {unsigned long nr[NR_LRU_LISTS ];unsigned long nr_to_scan;
@@ -2109,7 +2070,7 @@ static void shrink_mem_cgroup_zone(int priority, struct mem_cgroup_zone *mz, restart: nr_reclaimed =0 ; nr_scanned = sc->nr_scanned; -get_scan_count (mz, sc, nr, priority); +get_scan_count (lruvec, sc, nr, priority);blk_start_plug (&plug); while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
@@ -2121,7 +2082,7 @@ restart: nr[lru] -= nr_to_scan; nr_reclaimed +=shrink_list (lru, nr_to_scan, - mz, sc, priority); + lruvec, sc, priority); } }/*
@@ -2147,11 +2108,11 @@ restart:* Even if we did not try to evict anon pages at all, we want to * rebalance the anon lru active/inactive ratio. */ -if (inactive_anon_is_low (mz)) -shrink_active_list (SWAP_CLUSTER_MAX , mz, sc, priority,0 ); +if (inactive_anon_is_low (lruvec)) +shrink_active_list (SWAP_CLUSTER_MAX , lruvec, sc, priority,0 );/* reclaim/compaction might need reclaim to continue */ -if (should_continue_reclaim (mz, nr_reclaimed, +if (should_continue_reclaim (lruvec, nr_reclaimed, sc->nr_scanned - nr_scanned, sc)) goto restart;
@@ -2167,15 +2128,14 @@ static void shrink_zone(int priority, struct zone *zone, .priority = priority, };struct mem_cgroup *memcg; +struct lruvec *lruvec; memcg =mem_cgroup_iter (root,NULL , &reclaim); do { - struct mem_cgroup_zone mz = { - .mem_cgroup = memcg, - .zone = zone, - }; + lruvec = mem_cgroup_zone_lruvec(zone, memcg); + + shrink_lruvec(priority, lruvec, sc); - shrink_mem_cgroup_zone(priority, &mz, sc); /* * Limit reclaim has historically picked one memcg and * scanned it with decreasing priority levels until
@@ -2495,10 +2455,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg, .order = 0, .target_mem_cgroup = memcg, }; -struct mem_cgroup_zone mz = { - .mem_cgroup = memcg, - .zone = zone, - }; +struct lruvec *lruvec =mem_cgroup_zone_lruvec (zone, memcg); sc.gfp_mask = (gfp_mask &GFP_RECLAIM_MASK ) | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK );
@@ -2514,7 +2471,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,* will pick up pages from other mem cgroup's as well. We hack * the priority and make it zero. */ -shrink_mem_cgroup_zone (0 , &mz, &sc); +shrink_lruvec (0 , lruvec, &sc);trace_mm_vmscan_memcg_softlimit_reclaim_end (sc.nr_reclaimed);
@@ -2575,13 +2532,10 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc, memcg =mem_cgroup_iter (NULL ,NULL ,NULL ); do { - struct mem_cgroup_zone mz = { - .mem_cgroup = memcg, - .zone = zone, - }; + struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg); - if (inactive_anon_is_low(&mz)) - shrink_active_list(SWAP_CLUSTER_MAX, &mz, + if (inactive_anon_is_low(lruvec)) + shrink_active_list(SWAP_CLUSTER_MAX, lruvec, sc, priority, 0); memcg = mem_cgroup_iter(NULL, memcg, NULL);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 10/21] mm: kill struct mem_cgroup_zone
Quoted 5 lines · On Thu, 23 Feb 2012 17:52:29 +0400 Konstantin Khlebnikov…
struct
mem_cgroup_zonealways points to one lruvec, either rootzone->lruvecor to some from memcg. So this fancy pointer can be replaced with direct pointer to struct lruvec, because all required infromation already collected on lruvec.Signed-off-by Konstantin Khlebnikov <[email protected]>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 11/21 mm: move page-to-lruvec translation upper
In reply to Konstantin Khlebnikovmove page_lruvec() out of add_page_to_lru_list() and del_page_from_lru_list() switch its first argument from zone to lruvec.
--- include/linux/mm_inline.h | 10 ++++------ mm/compaction.c | 4 +++- mm/memcontrol.c | 7 +++++-- mm/swap.c | 33 +++++++++++++++++++++------------ mm/vmscan.c | 14 +++++++++----- 5 files changed, 42 insertions(+), 26 deletions(-)
include/linux/mm_inline.h +4 -6
@@ -22,26 +22,24 @@ static inline int page_is_file_cache(struct page *page) }static inline void -add_page_to_lru_list (struct zone *zone,struct page *page,enum lru_list lru) +add_page_to_lru_list (struct lruvec *lruvec,struct page *page,enum lru_list lru) { -struct lruvec *lruvec =page_lruvec (page);int numpages =hpage_nr_pages (page);list_add (&page->lru, &lruvec->pages_lru[lru]); lruvec->pages_count[lru] += numpages; -__mod_zone_page_state (zone,NR_LRU_BASE + lru, numpages); +__mod_zone_page_state (lruvec_zone (lruvec),NR_LRU_BASE + lru, numpages); }static inline void -del_page_from_lru_list (struct zone *zone,struct page *page,enum lru_list lru) +del_page_from_lru_list (struct lruvec *lruvec,struct page *page,enum lru_list lru) { -struct lruvec *lruvec =page_lruvec (page);int numpages =hpage_nr_pages (page);list_del (&page->lru); lruvec->pages_count[lru] -= numpages;VM_BUG_ON ((long )lruvec->pages_count[lru] <0 ); -__mod_zone_page_state (zone,NR_LRU_BASE + lru, -numpages); +__mod_zone_page_state (lruvec_zone (lruvec),NR_LRU_BASE + lru, -numpages); }/**
mm/compaction.c +3 -1
@@ -262,6 +262,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,unsigned long nr_scanned =0 , nr_isolated =0 ;struct list_head *migratelist = &cc->migratepages;isolate_mode_t mode =ISOLATE_ACTIVE |ISOLATE_INACTIVE ; +struct lruvec *lruvec;/* Do not scan outside zone boundaries */ low_pfn =max (cc->migrate_pfn, zone->zone_start_pfn);
@@ -381,7 +382,8 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,VM_BUG_ON (PageTransCompound (page));/* Successfully isolated */ -del_page_from_lru_list (zone, page,page_lru (page)); + lruvec =page_lruvec (page); +del_page_from_lru_list (lruvec, page,page_lru (page));list_add (&page->lru, migratelist); cc->nr_migratepages++; nr_isolated++;
mm/memcontrol.c +5 -2
@@ -2543,6 +2543,7 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg, {struct page_cgroup *pc =lookup_page_cgroup (page);struct zone *zone =page_zone (page); +struct lruvec *lruvec;unsigned long flags;bool removed = false;
@@ -2553,13 +2554,15 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,*/ spin_lock_irqsave (&zone->lru_lock, flags);if (PageLRU (page)) { -del_page_from_lru_list (zone, page,page_lru (page)); + lruvec =page_lruvec (page); +del_page_from_lru_list (lruvec, page,page_lru (page));ClearPageLRU (page); removed = true; }__mem_cgroup_commit_charge (memcg, page,1 , pc, ctype);if (removed) { -add_page_to_lru_list (zone, page,page_lru (page)); + lruvec =page_lruvec (page); +add_page_to_lru_list (lruvec, page,page_lru (page));SetPageLRU (page); }spin_unlock_irqrestore (&zone->lru_lock, flags);
mm/swap.c +21 -12
@@ -49,11 +49,13 @@ static void __page_cache_release(struct page *page)if (PageLRU (page)) {unsigned long flags;struct zone *zone =page_zone (page); +struct lruvec *lruvec;spin_lock_irqsave (&zone->lru_lock, flags);VM_BUG_ON (!PageLRU (page));__ClearPageLRU (page); -del_page_from_lru_list (zone, page,page_off_lru (page)); + lruvec =page_lruvec (page); +del_page_from_lru_list (lruvec, page,page_off_lru (page));spin_unlock_irqrestore (&zone->lru_lock, flags); } }
@@ -293,11 +295,13 @@ static void __activate_page(struct page *page, void *arg)if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) {int file =page_is_file_cache (page);int lru =page_lru_base_type (page); -del_page_from_lru_list (zone, page, lru); +struct lruvec *lruvec =page_lruvec (page); + +del_page_from_lru_list (lruvec, page, lru);SetPageActive (page); lru +=LRU_ACTIVE ; -add_page_to_lru_list (zone, page, lru); +add_page_to_lru_list (lruvec, page, lru);__count_vm_event (PGACTIVATE );update_page_reclaim_stat (zone, page, file,1 );
@@ -404,11 +408,13 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)void add_page_to_unevictable_list (struct page *page) {struct zone *zone =page_zone (page); +struct lruvec *lruvec;spin_lock_irq (&zone->lru_lock);SetPageUnevictable (page);SetPageLRU (page); -add_page_to_lru_list (zone, page,LRU_UNEVICTABLE ); + lruvec =page_lruvec (page); +add_page_to_lru_list (lruvec, page,LRU_UNEVICTABLE );spin_unlock_irq (&zone->lru_lock); }
@@ -438,6 +444,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)int lru, file;bool active;struct zone *zone =page_zone (page); +struct lruvec *lruvec;if (!PageLRU (page))return ;
@@ -453,10 +460,11 @@ static void lru_deactivate_fn(struct page *page, void *arg) file =page_is_file_cache (page); lru =page_lru_base_type (page); -del_page_from_lru_list (zone, page, lru + active); + lruvec =page_lruvec (page); +del_page_from_lru_list (lruvec, page, lru + active);ClearPageActive (page);ClearPageReferenced (page); -add_page_to_lru_list (zone, page, lru); +add_page_to_lru_list (lruvec, page, lru);if (PageWriteback (page) ||PageDirty (page)) {/*
@@ -466,7 +474,6 @@ static void lru_deactivate_fn(struct page *page, void *arg)*/ SetPageReclaim (page); }else { -struct lruvec *lruvec =page_lruvec (page);/* * The page's writeback ends up during pagevec * We moves tha page into tail of inactive.
@@ -596,6 +603,7 @@ void release_pages(struct page **pages, int nr, int cold)if (PageLRU (page)) {struct zone *pagezone =page_zone (page); +struct lruvec *lruvec =page_lruvec (page);if (pagezone != zone) {if (zone)
@@ -606,7 +614,7 @@ void release_pages(struct page **pages, int nr, int cold) }VM_BUG_ON (!PageLRU (page));__ClearPageLRU (page); -del_page_from_lru_list (zone, page,page_off_lru (page)); +del_page_from_lru_list (lruvec, page,page_off_lru (page)); }list_add (&page->lru, &pages_to_free);
@@ -644,6 +652,7 @@ void lru_add_page_tail(struct zone* zone,int active;enum lru_list lru;const int file =0 ; +struct lruvec *lruvec =page_lruvec (page);VM_BUG_ON (!PageHead (page));VM_BUG_ON (PageCompound (page_tail));
@@ -678,7 +687,7 @@ void lru_add_page_tail(struct zone* zone,* Use the standard add function to put page_tail on the list, * but then correct its position so they all end up in order. */ -add_page_to_lru_list (zone, page_tail, lru); +add_page_to_lru_list (lruvec, page_tail, lru); list_head = page_tail->lru.prev;list_move_tail (&page_tail->lru, list_head); }
@@ -688,7 +697,7 @@ void lru_add_page_tail(struct zone* zone,static void __pagevec_lru_add_fn (struct page *page,void *arg) {enum lru_list lru = (enum lru_list )arg; -struct zone *zone =page_zone (page); +struct lruvec *lruvec =page_lruvec (page);int file =is_file_lru (lru);int active =is_active_lru (lru);
@@ -699,8 +708,8 @@ static void __pagevec_lru_add_fn(struct page *page, void *arg)SetPageLRU (page);if (active)SetPageActive (page); -update_page_reclaim_stat (zone, page, file, active); -add_page_to_lru_list (zone, page, lru); +update_page_reclaim_stat (lruvec_zone (lruvec), page, file, active); +add_page_to_lru_list (lruvec, page, lru); }/*
mm/vmscan.c +9 -5
@@ -1286,6 +1286,7 @@ int isolate_lru_page(struct page *page)if (PageLRU (page)) {struct zone *zone =page_zone (page); +struct lruvec *lruvec;spin_lock_irq (&zone->lru_lock);if (PageLRU (page)) {
@@ -1293,8 +1294,8 @@ int isolate_lru_page(struct page *page) ret =0 ;get_page (page);ClearPageLRU (page); - -del_page_from_lru_list (zone, page, lru); + lruvec =page_lruvec (page); +del_page_from_lru_list (lruvec, page, lru); }spin_unlock_irq (&zone->lru_lock); }
@@ -1339,6 +1340,7 @@ putback_inactive_pages(struct lruvec *lruvec,*/ while (!list_empty (page_list)) {struct page *page =lru_to_page (page_list); +struct lruvec *lruvec;int lru;VM_BUG_ON (PageLRU (page));
@@ -1351,7 +1353,9 @@ putback_inactive_pages(struct lruvec *lruvec, }SetPageLRU (page); lru =page_lru (page); -add_page_to_lru_list (zone, page, lru); + + lruvec =page_lruvec (page); +add_page_to_lru_list (lruvec, page, lru);if (is_active_lru (lru)) {int file =is_file_lru (lru);int numpages =hpage_nr_pages (page);
@@ -1360,7 +1364,7 @@ putback_inactive_pages(struct lruvec *lruvec,if (put_page_testzero (page)) {__ClearPageLRU (page);__ClearPageActive (page); -del_page_from_lru_list (zone, page, lru); +del_page_from_lru_list (lruvec, page, lru);if (unlikely (PageCompound (page))) {spin_unlock_irq (&zone->lru_lock);
@@ -1643,7 +1647,7 @@ static void move_active_pages_to_lru(struct zone *zone,if (put_page_testzero (page)) {__ClearPageLRU (page);__ClearPageActive (page); -del_page_from_lru_list (zone, page, lru); +del_page_from_lru_list (lruvec, page, lru);if (unlikely (PageCompound (page))) {spin_unlock_irq (&zone->lru_lock);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 11/21] mm: move page-to-lruvec translation upper
Quoted 4 lines · On Thu, 23 Feb 2012 17:52:33 +0400 Konstantin Khlebnikov…
move
page_lruvec()out ofadd_page_to_lru_list()anddel_page_from_lru_list()switch its first argument from zone to lruvec.Signed-off-by Konstantin Khlebnikov <[email protected]>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 12/21 mm: push lruvec into update_page_reclaim_stat()
In reply to Konstantin KhlebnikovPush lruvec pointer into update_page_reclaim_stat()
* drop page argument
* drop active and file arguments, use lru instead
--- mm/swap.c | 30 +++++++++--------------------- 1 files changed, 9 insertions(+), 21 deletions(-)
mm/swap.c +9 -21
@@ -276,24 +276,19 @@ void rotate_reclaimable_page(struct page *page) } } -static void update_page_reclaim_stat (struct zone *zone,struct page *page, -int file,int rotated) +static void update_page_reclaim_stat (struct lruvec *lruvec,enum lru_list lru) { -struct zone_reclaim_stat *reclaim_stat; - - reclaim_stat = &page_lruvec (page)->reclaim_stat; +struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; +int file =is_file_lru (lru); reclaim_stat->recent_scanned[file]++; -if (rotated) +if (is_active_lru (lru)) reclaim_stat->recent_rotated[file]++; }static void __activate_page (struct page *page,void *arg) { -struct zone *zone =page_zone (page); -if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) { -int file =page_is_file_cache (page);int lru =page_lru_base_type (page);struct lruvec *lruvec =page_lruvec (page);
@@ -304,7 +299,7 @@ static void __activate_page(struct page *page, void *arg)add_page_to_lru_list (lruvec, page, lru);__count_vm_event (PGACTIVATE ); -update_page_reclaim_stat (zone, page, file,1 ); +update_page_reclaim_stat (lruvec, lru); } }
@@ -443,7 +438,6 @@ static void lru_deactivate_fn(struct page *page, void *arg) {int lru, file;bool active; -struct zone *zone =page_zone (page);struct lruvec *lruvec;if (!PageLRU (page))
@@ -484,7 +478,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)if (active)__count_vm_event (PGDEACTIVATE ); -update_page_reclaim_stat (zone, page, file,0 ); +update_page_reclaim_stat (lruvec, lru); }/*
@@ -649,9 +643,7 @@ EXPORT_SYMBOL(__pagevec_release);void lru_add_page_tail (struct zone * zone,struct page *page,struct page *page_tail) { -int active;enum lru_list lru; -const int file =0 ;struct lruvec *lruvec =page_lruvec (page);VM_BUG_ON (!PageHead (page));
@@ -664,13 +656,11 @@ void lru_add_page_tail(struct zone* zone,if (page_evictable (page_tail,NULL )) {if (PageActive (page)) {SetPageActive (page_tail); - active =1 ; lru =LRU_ACTIVE_ANON ; }else { - active =0 ; lru =LRU_INACTIVE_ANON ; } -update_page_reclaim_stat (zone, page_tail, file, active); +update_page_reclaim_stat (lruvec, lru); }else {SetPageUnevictable (page_tail); lru =LRU_UNEVICTABLE ;
@@ -698,17 +688,15 @@ static void __pagevec_lru_add_fn(struct page *page, void *arg) {enum lru_list lru = (enum lru_list )arg;struct lruvec *lruvec =page_lruvec (page); -int file =is_file_lru (lru); -int active =is_active_lru (lru);VM_BUG_ON (PageActive (page));VM_BUG_ON (PageUnevictable (page));VM_BUG_ON (PageLRU (page));SetPageLRU (page); -if (active) +if (is_active_lru (lru))SetPageActive (page); -update_page_reclaim_stat (lruvec_zone (lruvec), page, file, active); +update_page_reclaim_stat (lruvec, lru);add_page_to_lru_list (lruvec, page, lru); }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 12/21] mm: push lruvec into update_page_reclaim_stat()
Quoted 5 lines · On Thu, 23 Feb 2012 17:52:38 +0400 Konstantin Khlebnikov…
Push lruvec pointer into
update_page_reclaim_stat()
* drop page argument
* drop active and file arguments, use lru insteadSigned-off-by Konstantin Khlebnikov <[email protected]>
Quoted 110 lines
--- mm/swap.c | 30 +++++++++--------------------- 1 files changed, 9 insertions(+), 21 deletions(-)mm/swap.c +9
-22@@ -276,24 +276,19 @@ void rotate_reclaimable_page(struct page *page) } } -static void update_page_reclaim_stat (struct zone *zone,struct page *page, -int file,int rotated) +static void update_page_reclaim_stat (struct lruvec *lruvec,enum lru_list lru) { -struct zone_reclaim_stat *reclaim_stat; - - reclaim_stat = &page_lruvec (page)->reclaim_stat; +struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat; +int file =is_file_lru (lru); reclaim_stat->recent_scanned[file]++; -if (rotated) +if (is_active_lru (lru)) reclaim_stat->recent_rotated[file]++; }static void __activate_page (struct page *page,void *arg) { -struct zone *zone =page_zone (page); -if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) { -int file =page_is_file_cache (page);int lru =page_lru_base_type (page);struct lruvec *lruvec =page_lruvec (page);@@ -304,7 +299,7 @@ static void __activate_page(struct page *page, void *arg)add_page_to_lru_list (lruvec, page, lru);__count_vm_event (PGACTIVATE ); -update_page_reclaim_stat (zone, page, file,1 ); +update_page_reclaim_stat (lruvec, lru); } }@@ -443,7 +438,6 @@ static void lru_deactivate_fn(struct page *page, void *arg) {int lru, file;bool active; -struct zone *zone =page_zone (page);struct lruvec *lruvec;if (!PageLRU (page))@@ -484,7 +478,7 @@ static void lru_deactivate_fn(struct page *page, void *arg)if (active)__count_vm_event (PGDEACTIVATE ); -update_page_reclaim_stat (zone, page, file,0 ); +update_page_reclaim_stat (lruvec, lru); }/* @@ -649,9 +643,7 @@ EXPORT_SYMBOL(__pagevec_release);void lru_add_page_tail (struct zone * zone,struct page *page,struct page *page_tail) { -int active;enum lru_list lru; -const int file =0 ;struct lruvec *lruvec =page_lruvec (page);VM_BUG_ON (!PageHead (page));@@ -664,13 +656,11 @@ void lru_add_page_tail(struct zone* zone,if (page_evictable (page_tail,NULL )) {if (PageActive (page)) {SetPageActive (page_tail); - active =1 ; lru =LRU_ACTIVE_ANON ; }else { - active =0 ; lru =LRU_INACTIVE_ANON ; } -update_page_reclaim_stat (zone, page_tail, file, active); +update_page_reclaim_stat (lruvec, lru); }else {SetPageUnevictable (page_tail); lru =LRU_UNEVICTABLE ;@@ -698,17 +688,15 @@ static void __pagevec_lru_add_fn(struct page *page, void *arg) {enum lru_list lru = (enum lru_list )arg;struct lruvec *lruvec =page_lruvec (page); -int file =is_file_lru (lru); -int active =is_active_lru (lru);VM_BUG_ON (PageActive (page));VM_BUG_ON (PageUnevictable (page));VM_BUG_ON (PageLRU (page));SetPageLRU (page); -if (active) +if (is_active_lru (lru))SetPageActive (page); -update_page_reclaim_stat (lruvec_zone (lruvec), page, file, active); +update_page_reclaim_stat (lruvec, lru);add_page_to_lru_list (lruvec, page, lru); } --
Quoted 5 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 13/21 mm: push lruvecs from pagevec_lru_move_fn() to iterator
In reply to Konstantin KhlebnikovPush lruvec pointer from pagevec_lru_move_fn() to iterator function. Push lruvec pointer into lru_add_page_tail()
--- include/linux/swap.h | 2 +- mm/huge_memory.c | 4 +++- mm/swap.c | 27 +++++++++++++-------------- 3 files changed, 17 insertions(+), 16 deletions(-)
include/linux/swap.h +1 -1
@@ -224,7 +224,7 @@ extern unsigned int nr_free_pagecache_pages(void);/* linux/mm/swap.c */ extern void __lru_cache_add (struct page *,enum lru_list lru);extern void lru_cache_add_lru (struct page *,enum lru_list lru); -extern void lru_add_page_tail (struct zone * zone, +extern void lru_add_page_tail (struct lruvec *lruvec,struct page *page,struct page *page_tail);extern void activate_page (struct page *);extern void mark_page_accessed (struct page *);
mm/huge_memory.c +3 -1
@@ -1229,10 +1229,12 @@ static void __split_huge_page_refcount(struct page *page) {int i;struct zone *zone =page_zone (page); +struct lruvec *lruvec;int tail_count =0 ;/* prevent PageLRU to go away from under us, and freeze lru stats */ spin_lock_irq (&zone->lru_lock); + lruvec =page_lruvec (page);compound_lock (page);/* complete memcg works before add pages to LRU */ mem_cgroup_split_huge_fixup (page);
@@ -1308,7 +1310,7 @@ static void __split_huge_page_refcount(struct page *page)BUG_ON (!PageSwapBacked (page_tail)); -lru_add_page_tail (zone, page, page_tail); +lru_add_page_tail (lruvec, page, page_tail); }atomic_sub (tail_count, &page->_count);BUG_ON (atomic_read (&page->_count) <=0 );
mm/swap.c +13 -14
@@ -204,7 +204,8 @@ void put_pages_list(struct list_head *pages)EXPORT_SYMBOL (put_pages_list);static void pagevec_lru_move_fn (struct pagevec *pvec, -void (*move_fn)(struct page *page,void *arg), +void (*move_fn)(struct lruvec *lruvec, +struct page *page,void *arg),void *arg) {int i;
@@ -214,6 +215,7 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,for (i =0 ; i <pagevec_count (pvec); i++) {struct page *page = pvec->pages[i];struct zone *pagezone =page_zone (page); +struct lruvec *lruvec;if (pagezone != zone) {if (zone)
@@ -222,7 +224,8 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,spin_lock_irqsave (&zone->lru_lock, flags); } - (*move_fn)(page, arg); + lruvec =page_lruvec (page); + (*move_fn)(lruvec, page, arg); }if (zone)spin_unlock_irqrestore (&zone->lru_lock, flags);
@@ -230,13 +233,13 @@ static void pagevec_lru_move_fn(struct pagevec *pvec,pagevec_reinit (pvec); } -static void pagevec_move_tail_fn (struct page *page,void *arg) +static void pagevec_move_tail_fn (struct lruvec *lruvec, +struct page *page,void *arg) {int *pgmoved = arg;if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) {enum lru_list lru =page_lru_base_type (page); -struct lruvec *lruvec =page_lruvec (page);list_move_tail (&page->lru, &lruvec->pages_lru[lru]); (*pgmoved)++;
@@ -286,11 +289,10 @@ static void update_page_reclaim_stat(struct lruvec *lruvec, enum lru_list lru) reclaim_stat->recent_rotated[file]++; } -static void __activate_page (struct page *page,void *arg) +static void __activate_page (struct lruvec *lruvec,struct page *page,void *arg) {if (PageLRU (page) && !PageActive (page) && !PageUnevictable (page)) {int lru =page_lru_base_type (page); -struct lruvec *lruvec =page_lruvec (page);del_page_from_lru_list (lruvec, page, lru);
@@ -434,11 +436,10 @@ void add_page_to_unevictable_list(struct page *page)* be write it out by flusher threads as this is much more effective * than the single-page writeout from reclaim. */ -static void lru_deactivate_fn (struct page *page,void *arg) +static void lru_deactivate_fn (struct lruvec *lruvec,struct page *page,void *arg) {int lru, file;bool active; -struct lruvec *lruvec;if (!PageLRU (page))return ;
@@ -454,7 +455,6 @@ static void lru_deactivate_fn(struct page *page, void *arg) file =page_is_file_cache (page); lru =page_lru_base_type (page); - lruvec =page_lruvec (page);del_page_from_lru_list (lruvec, page, lru + active);ClearPageActive (page);ClearPageReferenced (page);
@@ -640,16 +640,15 @@ EXPORT_SYMBOL(__pagevec_release);#ifdef CONFIG_TRANSPARENT_HUGEPAGE /* used by __split_huge_page_refcount() */ -void lru_add_page_tail (struct zone * zone, +void lru_add_page_tail (struct lruvec *lruvec,struct page *page,struct page *page_tail) {enum lru_list lru; -struct lruvec *lruvec =page_lruvec (page);VM_BUG_ON (!PageHead (page));VM_BUG_ON (PageCompound (page_tail));VM_BUG_ON (PageLRU (page_tail)); -VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&zone->lru_lock)); +VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec_zone (lruvec)->lru_lock));SetPageLRU (page_tail);
@@ -684,10 +683,10 @@ void lru_add_page_tail(struct zone* zone, }#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static void __pagevec_lru_add_fn (struct page *page,void *arg) +static void __pagevec_lru_add_fn (struct lruvec *lruvec, +struct page *page,void *arg) {enum lru_list lru = (enum lru_list )arg; -struct lruvec *lruvec =page_lruvec (page);VM_BUG_ON (PageActive (page));VM_BUG_ON (PageUnevictable (page));
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 13/21] mm: push lruvecs from pagevec_lru_move_fn() to iterator
Quoted 4 lines · On Thu, 23 Feb 2012 17:52:42 +0400 Konstantin Khlebnikov…
Push lruvec pointer from
pagevec_lru_move_fn()to iterator function. Push lruvec pointer intolru_add_page_tail()Signed-off-by Konstantin Khlebnikov <[email protected]>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 14/21 mm: introduce lruvec locking primitives
In reply to Konstantin KhlebnikovThis is initial preparation for lru_lock splitting.
This locking primites designed to hide splitted nature of lru_lock and to avoid overhead for non-splitted lru_lock in non-memcg case.
* Lock via lruvec reference
lock_lruvec(lruvec, flags)lock_lruvec_irq(lruvec)
* Lock via page reference
lock_page_lruvec(page, flags)lock_page_lruvec_irq(page)relock_page_lruvec(lruvec, page, flags) relock_page_lruvec_irq(lruvec, page) __relock_page_lruvec(lruvec, page) ( lruvec != NULL, page in same zone )
They always returns pointer to some locked lruvec, page anyway can be not in lru, PageLRU() sign is stable while we hold returned lruvec lock. Caller must guarantee page to lruvec reference validity.
* Lock via page, without stable page reference
__lock_page_lruvec_irq(&lruvec, page)
It returns true of lruvec succesfully locked and PageLRU is set. Initial lruvec can be NULL. Consequent calls must be in the same zone.
* Unlock
unlock_lruvec(lruvec, flags)unlock_lruvec_irq(lruvec)
* Wait
wait_lruvec_unlock(lruvec)
Wait for lruvec unlock, caller must have stable reference to lruvec.
__wait_lruvec_unlock(lruvec)
Wait for lruvec unlock before locking other lrulock for same page, nothing if there only one possible lruvec per page.
Used at page-to-lruvec reference switching to stabilize PageLRU sign.
--- mm/huge_memory.c | 8 +- mm/internal.h | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/memcontrol.c | 14 ++-- mm/swap.c | 58 ++++++------------ mm/vmscan.c | 77 ++++++++++-------------- 5 files changed, 237 insertions(+), 96 deletions(-)
mm/huge_memory.c +3 -5
@@ -1228,13 +1228,11 @@ static int __split_huge_page_splitting(struct page *page,static void __split_huge_page_refcount (struct page *page) {int i; -struct zone *zone =page_zone (page);struct lruvec *lruvec;int tail_count =0 ;/* prevent PageLRU to go away from under us, and freeze lru stats */ -spin_lock_irq (&zone->lru_lock); - lruvec =page_lruvec (page); + lruvec =lock_page_lruvec_irq (page);compound_lock (page);/* complete memcg works before add pages to LRU */ mem_cgroup_split_huge_fixup (page);
@@ -1316,11 +1314,11 @@ static void __split_huge_page_refcount(struct page *page)BUG_ON (atomic_read (&page->_count) <=0 );__dec_zone_page_state (page,NR_ANON_TRANSPARENT_HUGEPAGES ); -__mod_zone_page_state (zone,NR_ANON_PAGES ,HPAGE_PMD_NR ); +__mod_zone_page_state (lruvec_zone (lruvec),NR_ANON_PAGES ,HPAGE_PMD_NR );ClearPageCompound (page);compound_unlock (page); -spin_unlock_irq (&zone->lru_lock); +unlock_lruvec_irq (lruvec);for (i =1 ; i <HPAGE_PMD_NR ; i++) {struct page *page_tail = page + i;
mm/internal.h +176 -0
182 lines not on this page · raw message
mm/memcontrol.c +8 -6
@@ -3487,12 +3487,14 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,struct list_head *list;struct page *busy;struct zone *zone; +struct lruvec *lruvec;int ret =0 ; zone = &NODE_DATA (node)->node_zones[zid]; mz =mem_cgroup_zoneinfo (memcg, node, zid); - list = &mz->lruvec.pages_lru[lru]; - loop = mz->lruvec.pages_count[lru]; + lruvec = &mz->lruvec; + list = &lruvec->pages_lru[lru]; + loop = lruvec->pages_count[lru];/* give some margin against EBUSY etc...*/ loop +=256 ; busy =NULL ;
@@ -3501,19 +3503,19 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,struct page *page; ret =0 ; -spin_lock_irqsave (&zone->lru_lock, flags); +lock_lruvec (lruvec, &flags);if (list_empty (list)) { -spin_unlock_irqrestore (&zone->lru_lock, flags); +unlock_lruvec (lruvec, &flags);break ; } page = list_entry(list->prev, struct page, lru);if (busy == page) {list_move (&page->lru, list); busy =NULL ; -spin_unlock_irqrestore (&zone->lru_lock, flags); +unlock_lruvec (lruvec, &flags);continue ; } -spin_unlock_irqrestore (&zone->lru_lock, flags); +unlock_lruvec (lruvec, &flags); pc =lookup_page_cgroup (page);
mm/swap.c +18 -40
125 lines not on this page · raw message
mm/vmscan.c +32 -45
248 lines not on this page · raw message
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 14/21] mm: introduce lruvec locking primitives
Quoted 39 lines · On Thu, 23 Feb 2012 17:52:47 +0400 Konstantin Khlebnikov…
This is initial preparation for
lru_locksplitting.This locking primites designed to hide splitted nature of
lru_lockand to avoid overhead for non-splittedlru_lockin non-memcg case.* Lock via lruvec reference
lock_lruvec(lruvec, flags)lock_lruvec_irq(lruvec)* Lock via page reference
lock_page_lruvec(page, flags)lock_page_lruvec_irq(page)relock_page_lruvec(lruvec, page, flags)relock_page_lruvec_irq(lruvec, page)__relock_page_lruvec(lruvec, page)( lruvec != NULL, page in same zone )They always returns pointer to some locked lruvec, page anyway can be not in lru,
PageLRU()sign is stable while we hold returned lruvec lock. Caller must guarantee page to lruvec reference validity.* Lock via page, without stable page reference
__lock_page_lruvec_irq(&lruvec, page)It returns true of lruvec succesfully locked and PageLRU is set. Initial lruvec can be NULL. Consequent calls must be in the same zone.
* Unlock
unlock_lruvec(lruvec, flags)unlock_lruvec_irq(lruvec)* Wait
wait_lruvec_unlock(lruvec)
Wait for lruvec unlock, caller must have stable reference to lruvec.
Quoted 6 lines
__wait_lruvec_unlock(lruvec)
Wait for lruvec unlock before locking other lrulock for same page, nothing if there only one possible lruvec per page.
Used at page-to-lruvec reference switching to stabilize PageLRU sign.Signed-off-by Konstantin Khlebnikov <[email protected]>
O.K. I like this.
Hmm....Could you add a comment in memcg part ? (see below)
Quoted 89 lines
--- mm/huge_memory.c | 8 +- mm/internal.h | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/memcontrol.c | 14 ++-- mm/swap.c | 58 ++++++------------ mm/vmscan.c | 77 ++++++++++-------------- 5 files changed, 237 insertions(+), 96 deletions(-)mm/huge_memory.c +3
-5@@ -1228,13 +1228,11 @@ static int __split_huge_page_splitting(struct page *page,static void __split_huge_page_refcount (struct page *page) {int i; -struct zone *zone =page_zone (page);struct lruvec *lruvec;int tail_count =0 ;/* prevent PageLRU to go away from under us, and freeze lru stats */ -spin_lock_irq (&zone->lru_lock); - lruvec =page_lruvec (page); + lruvec =lock_page_lruvec_irq (page);compound_lock (page);/* complete memcg works before add pages to LRU */ mem_cgroup_split_huge_fixup (page);@@ -1316,11 +1314,11 @@ static void __split_huge_page_refcount(struct page *page)BUG_ON (atomic_read (&page->_count) <=0 );__dec_zone_page_state (page,NR_ANON_TRANSPARENT_HUGEPAGES ); -__mod_zone_page_state (zone,NR_ANON_PAGES ,HPAGE_PMD_NR ); +__mod_zone_page_state (lruvec_zone (lruvec),NR_ANON_PAGES ,HPAGE_PMD_NR );ClearPageCompound (page);compound_unlock (page); -spin_unlock_irq (&zone->lru_lock); +unlock_lruvec_irq (lruvec);for (i =1 ; i <HPAGE_PMD_NR ; i++) {struct page *page_tail = page + i;mm/internal.h +40
-0@@ -13,6 +13,182 @@#include <linux/mm.h> +static inline void lock_lruvec (struct lruvec *lruvec,unsigned long *flags) +{ +spin_lock_irqsave (&lruvec_zone (lruvec)->lru_lock, *flags); +} + +static inline void lock_lruvec_irq (struct lruvec *lruvec) +{ +spin_lock_irq (&lruvec_zone (lruvec)->lru_lock); +} + +static inline void unlock_lruvec (struct lruvec *lruvec,unsigned long *flags) +{ +spin_unlock_irqrestore (&lruvec_zone (lruvec)->lru_lock, *flags); +} + +static inline void unlock_lruvec_irq (struct lruvec *lruvec) +{ +spin_unlock_irq (&lruvec_zone (lruvec)->lru_lock); +} + +static inline void wait_lruvec_unlock (struct lruvec *lruvec) +{ +spin_unlock_wait (&lruvec_zone (lruvec)->lru_lock); +} + +#ifdef CONFIG_CGROUP_MEM_RES_CTLR + +/* Dynamic page to lruvec mapping */ + +/* Lock other lruvec for other page in the same zone */ +static inline struct lruvec *__relock_page_lruvec (struct lruvec *locked_lruvec, +struct page *page) +{ +/* Currenyly only one lru_lock per-zone */ +return page_lruvec (page); +} + +static inline struct lruvec *relock_page_lruvec_irq (struct lruvec *lruvec, +struct page *page) +{
mm/internal.h line 65
Quoted 10 lines
mm/internal.h +10
-0+struct zone *zone =page_zone (page); + +if (!lruvec) { +spin_lock_irq (&zone->lru_lock); + }else if (zone !=lruvec_zone (lruvec)) { +unlock_lruvec_irq (lruvec); +spin_lock_irq (&zone->lru_lock); + } +return page_lruvec (page); +}
Could you add comments/caution to the caller
- !PageLRU(page) case ?
- Can the caller assume page_lruvec(page) == lruvec ? If no, which lru_vec is locked ?
etc...
Quoted 5 lines
mm/internal.h +5
-0+ +static inline struct lruvec *relock_page_lruvec (struct lruvec *lruvec, +struct page *page, +unsigned long *flags) +{
mm/internal.h line 80
Quoted 10 lines
mm/internal.h +10
-0+struct zone *zone =page_zone (page); + +if (!lruvec) { +spin_lock_irqsave (&zone->lru_lock, *flags); + }else if (zone !=lruvec_zone (lruvec)) { +unlock_lruvec (lruvec, flags); +spin_lock_irqsave (&zone->lru_lock, *flags); + } +return page_lruvec (page); +}
Same here.
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 14/21] mm: introduce lruvec locking primitives
Quoted 159 lines · KAMEZAWA Hiroyuki
Quoted 39 lines · On Thu, 23 Feb 2012 17:52:47 +0400 Konstantin…
This is initial preparation for
lru_locksplitting.This locking primites designed to hide splitted nature of
lru_lockand to avoid overhead for non-splittedlru_lockin non-memcg case.* Lock via lruvec reference
lock_lruvec(lruvec, flags)lock_lruvec_irq(lruvec)* Lock via page reference
lock_page_lruvec(page, flags)lock_page_lruvec_irq(page)relock_page_lruvec(lruvec, page, flags)relock_page_lruvec_irq(lruvec, page)__relock_page_lruvec(lruvec, page)( lruvec != NULL, page in same zone )They always returns pointer to some locked lruvec, page anyway can be not in lru,
PageLRU()sign is stable while we hold returned lruvec lock. Caller must guarantee page to lruvec reference validity.* Lock via page, without stable page reference
__lock_page_lruvec_irq(&lruvec, page)It returns true of lruvec succesfully locked and PageLRU is set. Initial lruvec can be NULL. Consequent calls must be in the same zone.
* Unlock
unlock_lruvec(lruvec, flags)unlock_lruvec_irq(lruvec)* Wait
wait_lruvec_unlock(lruvec)
Wait for lruvec unlock, caller must have stable reference to lruvec.Quoted 6 lines
__wait_lruvec_unlock(lruvec)
Wait for lruvec unlock before locking other lrulock for same page, nothing if there only one possible lruvec per page.
Used at page-to-lruvec reference switching to stabilize PageLRU sign.Signed-off-by Konstantin Khlebnikov<[email protected]>O.K. I like this.
Acked-by KAMEZAWA Hiroyuki<[email protected]>Hmm....Could you add a comment in memcg part ? (see below)
Quoted 89 lines
--- mm/huge_memory.c | 8 +- mm/internal.h | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ mm/memcontrol.c | 14 ++-- mm/swap.c | 58 ++++++------------ mm/vmscan.c | 77 ++++++++++-------------- 5 files changed, 237 insertions(+), 96 deletions(-)mm/huge_memory.c +3
-5@@ -1228,13 +1228,11 @@ static int __split_huge_page_splitting(struct page *page,static void __split_huge_page_refcount (struct page *page) {int i; -struct zone *zone =page_zone (page);struct lruvec *lruvec;int tail_count =0 ;/* prevent PageLRU to go away from under us, and freeze lru stats */ -spin_lock_irq (&zone->lru_lock); - lruvec =page_lruvec (page); + lruvec =lock_page_lruvec_irq (page);compound_lock (page);/* complete memcg works before add pages to LRU */ mem_cgroup_split_huge_fixup (page);@@ -1316,11 +1314,11 @@ static void __split_huge_page_refcount(struct page *page)BUG_ON (atomic_read (&page->_count)<=0 );__dec_zone_page_state (page,NR_ANON_TRANSPARENT_HUGEPAGES ); -__mod_zone_page_state (zone,NR_ANON_PAGES ,HPAGE_PMD_NR ); +__mod_zone_page_state (lruvec_zone (lruvec),NR_ANON_PAGES ,HPAGE_PMD_NR );ClearPageCompound (page);compound_unlock (page); -spin_unlock_irq (&zone->lru_lock); +unlock_lruvec_irq (lruvec);for (i =1 ; i<HPAGE_PMD_NR ; i++) {struct page *page_tail = page + i;mm/internal.h +40
-0@@ -13,6 +13,182 @@#include <linux/mm.h> +static inline void lock_lruvec (struct lruvec *lruvec,unsigned long *flags) +{ +spin_lock_irqsave (&lruvec_zone (lruvec)->lru_lock, *flags); +} + +static inline void lock_lruvec_irq (struct lruvec *lruvec) +{ +spin_lock_irq (&lruvec_zone (lruvec)->lru_lock); +} + +static inline void unlock_lruvec (struct lruvec *lruvec,unsigned long *flags) +{ +spin_unlock_irqrestore (&lruvec_zone (lruvec)->lru_lock, *flags); +} + +static inline void unlock_lruvec_irq (struct lruvec *lruvec) +{ +spin_unlock_irq (&lruvec_zone (lruvec)->lru_lock); +} + +static inline void wait_lruvec_unlock (struct lruvec *lruvec) +{ +spin_unlock_wait (&lruvec_zone (lruvec)->lru_lock); +} + +#ifdef CONFIG_CGROUP_MEM_RES_CTLR + +/* Dynamic page to lruvec mapping */ + +/* Lock other lruvec for other page in the same zone */ +static inline struct lruvec *__relock_page_lruvec (struct lruvec *locked_lruvec, +struct page *page) +{ +/* Currenyly only one lru_lock per-zone */ +return page_lruvec (page); +} + +static inline struct lruvec *relock_page_lruvec_irq (struct lruvec *lruvec, +struct page *page) +{mm/internal.h line 65
Quoted 10 lines
mm/internal.h +10
-0+struct zone *zone =page_zone (page); + +if (!lruvec) { +spin_lock_irq (&zone->lru_lock); + }else if (zone !=lruvec_zone (lruvec)) { +unlock_lruvec_irq (lruvec); +spin_lock_irq (&zone->lru_lock); + } +return page_lruvec (page); +}Could you add comments/caution to the caller
Quoted 2 lines
- !PageLRU(page) case ?
- Can the caller assumepage_lruvec(page)== lruvec ? If no, whichlru_vecis locked ?
Yes, caller can assume page_lruvec(page) == lruvec. And PageLRU() is stable, it means it stays true or false while this lruvec is locked.
Quoted 23 lines
etc...
Quoted 5 lines
mm/internal.h +5
-0+ +static inline struct lruvec *relock_page_lruvec (struct lruvec *lruvec, +struct page *page, +unsigned long *flags) +{mm/internal.h line 80
Quoted 10 lines
mm/internal.h +10
-0+struct zone *zone =page_zone (page); + +if (!lruvec) { +spin_lock_irqsave (&zone->lru_lock, *flags); + }else if (zone !=lruvec_zone (lruvec)) { +unlock_lruvec (lruvec, flags); +spin_lock_irqsave (&zone->lru_lock, *flags); + } +return page_lruvec (page); +}Same here.
Quoted 3 lines
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 15/21 mm: handle lruvec relocks on lumpy reclaim
In reply to Konstantin KhlebnikovPrepare for lock splitting in lumly reclaim logic. Now move_active_pages_to_lru() and putback_inactive_pages() can put pages into different lruvecs.
* relock book before SetPageLRU()
* update reclaim_stat pointer after relocks
* return currently locked lruvec
--- mm/vmscan.c | 45 +++++++++++++++++++++++++++++++++------------ 1 files changed, 33 insertions(+), 12 deletions(-)
mm/vmscan.c +33 -12
@@ -1114,6 +1114,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, unsigned long *nr_scanned, struct scan_control *sc, isolate_mode_t mode, int active, int file) { + struct lruvec *cursor_lruvec = lruvec;struct list_head *src;unsigned long nr_taken =0 ;unsigned long nr_lumpy_taken =0 ;
@@ -1197,14 +1198,17 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, !PageSwapCache(cursor_page))break ; +/* Switch cursor_lruvec lock for lumpy isolate */ +if (!__lock_page_lruvec_irq (&cursor_lruvec, + cursor_page)) +continue ; + if (__isolate_lru_page(cursor_page, mode, file) == 0) { unsigned int isolated_pages; - struct lruvec *cursor_lruvec; int cursor_lru = page_lru(cursor_page); list_move(&cursor_page->lru, dst); isolated_pages = hpage_nr_pages(cursor_page); - cursor_lruvec = page_lruvec(cursor_page); cursor_lruvec->pages_count[cursor_lru] -= isolated_pages; VM_BUG_ON((long)cursor_lruvec->
@@ -1235,6 +1239,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, } } +/* Restore original lruvec lock */ + cursor_lruvec =__relock_page_lruvec (cursor_lruvec, page); +/* If we break out of the loop above, lumpy reclaim failed */ if (pfn < end_pfn) nr_lumpy_failed++;
@@ -1325,7 +1332,10 @@ static int too_many_isolated(struct zone *zone, int file,return isolated > inactive; } -static noinline_for_stack void +/* +* Returns currently locked lruvec +*/ +static noinline_for_stack struct lruvec *putback_inactive_pages (struct lruvec *lruvec,struct list_head *page_list) {
@@ -1347,10 +1357,13 @@ putback_inactive_pages(struct lruvec *lruvec,lock_lruvec_irq (lruvec);continue ; } + + lruvec =__relock_page_lruvec (lruvec, page); + reclaim_stat = &lruvec->reclaim_stat; +SetPageLRU (page); lru =page_lru (page); - lruvec =page_lruvec (page);add_page_to_lru_list (lruvec, page, lru);if (is_active_lru (lru)) {int file =is_file_lru (lru);
@@ -1375,6 +1388,8 @@ putback_inactive_pages(struct lruvec *lruvec,* To save our caller's stack, now use input list for pages to free. */ list_splice (&pages_to_free, page_list); + +return lruvec; }static noinline_for_stack void
@@ -1544,7 +1559,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,__count_vm_events (KSWAPD_STEAL , nr_reclaimed);__count_zone_vm_events (PGSTEAL , zone, nr_reclaimed); -putback_inactive_pages (lruvec, &page_list); + lruvec =putback_inactive_pages (lruvec, &page_list);__mod_zone_page_state (zone,NR_ISOLATED_ANON , -nr_anon);__mod_zone_page_state (zone,NR_ISOLATED_FILE , -nr_file);
@@ -1603,12 +1618,15 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,* * The downside is that we have to touch page->_count against each page. * But we had to alter page->flags anyway. +* +* Returns currently locked lruvec */ -static void move_active_pages_to_lru (struct lruvec *lruvec, -struct list_head *list, -struct list_head *pages_to_free, -enum lru_list lru) +static struct lruvec * +move_active_pages_to_lru (struct lruvec *lruvec, +struct list_head *list, +struct list_head *pages_to_free, +enum lru_list lru) {unsigned long pgmoved =0 ;struct page *page;
@@ -1630,10 +1648,11 @@ static void move_active_pages_to_lru(struct lruvec *lruvec, page =lru_to_page (list); + lruvec =__relock_page_lruvec (lruvec, page); +VM_BUG_ON (PageLRU (page));SetPageLRU (page); - lruvec =page_lruvec (page);list_move (&page->lru, &lruvec->pages_lru[lru]); numpages =hpage_nr_pages (page); lruvec->pages_count[lru] += numpages;
@@ -1655,6 +1674,8 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,__mod_zone_page_state (lruvec_zone (lruvec),NR_LRU_BASE + lru, pgmoved);if (!is_active_lru (lru))__count_vm_events (PGDEACTIVATE , pgmoved); + +return lruvec; } static void shrink_active_list(unsigned long nr_to_scan,
@@ -1744,9 +1765,9 @@ static void shrink_active_list(unsigned long nr_to_scan,*/ reclaim_stat->recent_rotated[file] += nr_rotated; -move_active_pages_to_lru (lruvec, &l_active, &l_hold, + lruvec =move_active_pages_to_lru (lruvec, &l_active, &l_hold,LRU_ACTIVE + file *LRU_FILE ); -move_active_pages_to_lru (lruvec, &l_inactive, &l_hold, + lruvec =move_active_pages_to_lru (lruvec, &l_inactive, &l_hold,LRU_BASE + file *LRU_FILE );__mod_zone_page_state (zone,NR_ISOLATED_ANON + file, -nr_taken);unlock_lruvec_irq (lruvec);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 15/21] mm: handle lruvec relocks on lumpy reclaim
Quoted 5 lines · On Thu, 23 Feb 2012 17:52:52 +0400 Konstantin Khlebnikov…
Prepare for lock splitting in lumly reclaim logic. Now
move_active_pages_to_lru()andputback_inactive_pages()can put pages into different lruvecs.* relock book before
SetPageLRU()
lruvec ?
Quoted 140 lines
* update
reclaim_statpointer after relocks
* return currently locked lruvecSigned-off-by Konstantin Khlebnikov <[email protected]>--- mm/vmscan.c | 45 +++++++++++++++++++++++++++++++++------------ 1 files changed, 33 insertions(+), 12 deletions(-)mm/vmscan.c +31
-10@@ -1114,6 +1114,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, unsigned long *nr_scanned, struct scan_control *sc, isolate_mode_t mode, int active, int file) { + struct lruvec *cursor_lruvec = lruvec;struct list_head *src;unsigned long nr_taken =0 ;unsigned long nr_lumpy_taken =0 ;@@ -1197,14 +1198,17 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, !PageSwapCache(cursor_page))break ; +/* Switch cursor_lruvec lock for lumpy isolate */ +if (!__lock_page_lruvec_irq (&cursor_lruvec, + cursor_page)) +continue ; + if (__isolate_lru_page(cursor_page, mode, file) == 0) { unsigned int isolated_pages; - struct lruvec *cursor_lruvec; int cursor_lru = page_lru(cursor_page); list_move(&cursor_page->lru, dst); isolated_pages = hpage_nr_pages(cursor_page); - cursor_lruvec = page_lruvec(cursor_page); cursor_lruvec->pages_count[cursor_lru] -= isolated_pages; VM_BUG_ON((long)cursor_lruvec->@@ -1235,6 +1239,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, } } +/* Restore original lruvec lock */ + cursor_lruvec =__relock_page_lruvec (cursor_lruvec, page); +/* If we break out of the loop above, lumpy reclaim failed */ if (pfn < end_pfn) nr_lumpy_failed++;@@ -1325,7 +1332,10 @@ static int too_many_isolated(struct zone *zone, int file,return isolated > inactive; } -static noinline_for_stack void +/* +* Returns currently locked lruvec +*/ +static noinline_for_stack struct lruvec *putback_inactive_pages (struct lruvec *lruvec,struct list_head *page_list) {@@ -1347,10 +1357,13 @@ putback_inactive_pages(struct lruvec *lruvec,lock_lruvec_irq (lruvec);continue ; } + + lruvec =__relock_page_lruvec (lruvec, page); + reclaim_stat = &lruvec->reclaim_stat; +SetPageLRU (page); lru =page_lru (page); - lruvec =page_lruvec (page);add_page_to_lru_list (lruvec, page, lru);if (is_active_lru (lru)) {int file =is_file_lru (lru);@@ -1375,6 +1388,8 @@ putback_inactive_pages(struct lruvec *lruvec,* To save our caller's stack, now use input list for pages to free. */ list_splice (&pages_to_free, page_list); + +return lruvec; }static noinline_for_stack void@@ -1544,7 +1559,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,__count_vm_events (KSWAPD_STEAL , nr_reclaimed);__count_zone_vm_events (PGSTEAL , zone, nr_reclaimed); -putback_inactive_pages (lruvec, &page_list); + lruvec =putback_inactive_pages (lruvec, &page_list);__mod_zone_page_state (zone,NR_ISOLATED_ANON , -nr_anon);__mod_zone_page_state (zone,NR_ISOLATED_FILE , -nr_file);@@ -1603,12 +1618,15 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,* * The downside is that we have to touch page->_count against each page. * But we had to alter page->flags anyway. +* +* Returns currently locked lruvec */ -static void move_active_pages_to_lru (struct lruvec *lruvec, -struct list_head *list, -struct list_head *pages_to_free, -enum lru_list lru) +static struct lruvec * +move_active_pages_to_lru (struct lruvec *lruvec, +struct list_head *list, +struct list_head *pages_to_free, +enum lru_list lru) {unsigned long pgmoved =0 ;struct page *page;@@ -1630,10 +1648,11 @@ static void move_active_pages_to_lru(struct lruvec *lruvec, page =lru_to_page (list); + lruvec =__relock_page_lruvec (lruvec, page); +VM_BUG_ON (PageLRU (page));SetPageLRU (page); - lruvec =page_lruvec (page);list_move (&page->lru, &lruvec->pages_lru[lru]); numpages =hpage_nr_pages (page); lruvec->pages_count[lru] += numpages;@@ -1655,6 +1674,8 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,__mod_zone_page_state (lruvec_zone (lruvec),NR_LRU_BASE + lru, pgmoved);if (!is_active_lru (lru))__count_vm_events (PGDEACTIVATE , pgmoved); + +return lruvec; } static void shrink_active_list(unsigned long nr_to_scan,@@ -1744,9 +1765,9 @@ static void shrink_active_list(unsigned long nr_to_scan,*/
mm/vmscan.c static void shrink_active_list(unsigned long nr_to_scan, · line 1773
Quoted 10 lines
mm/vmscan.c +2
-2reclaim_stat->recent_rotated[file] += nr_rotated; -move_active_pages_to_lru (lruvec, &l_active, &l_hold, + lruvec =move_active_pages_to_lru (lruvec, &l_active, &l_hold,LRU_ACTIVE + file *LRU_FILE ); -move_active_pages_to_lru (lruvec, &l_inactive, &l_hold, + lruvec =move_active_pages_to_lru (lruvec, &l_inactive, &l_hold,LRU_BASE + file *LRU_FILE );__mod_zone_page_state (zone,NR_ISOLATED_ANON + file, -nr_taken);unlock_lruvec_irq (lruvec);
Hmm...could you add comments to each function as
"The caller should _lock_ lruvec before calling this functions.
This function returns a lruvec with _locked_. It may be different from passed one. And The callser should unlock lruvec"
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 15/21] mm: handle lruvec relocks on lumpy reclaim
Quoted 10 lines · KAMEZAWA Hiroyuki
Quoted 5 lines · On Thu, 23 Feb 2012 17:52:52 +0400 Konstantin…
Prepare for lock splitting in lumly reclaim logic. Now
move_active_pages_to_lru()andputback_inactive_pages()can put pages into different lruvecs.* relock book before
SetPageLRU()lruvec ?
yeah, this came from v1
Quoted 142 lines
Quoted 142 lines
* update
reclaim_statpointer after relocks
* return currently locked lruvecSigned-off-by Konstantin Khlebnikov<[email protected]>--- mm/vmscan.c | 45 +++++++++++++++++++++++++++++++++------------ 1 files changed, 33 insertions(+), 12 deletions(-)mm/vmscan.c +31
-10@@ -1114,6 +1114,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, unsigned long *nr_scanned, struct scan_control *sc, isolate_mode_t mode, int active, int file) { + struct lruvec *cursor_lruvec = lruvec;struct list_head *src;unsigned long nr_taken =0 ;unsigned long nr_lumpy_taken =0 ;@@ -1197,14 +1198,17 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, !PageSwapCache(cursor_page))break ; +/* Switch cursor_lruvec lock for lumpy isolate */ +if (!__lock_page_lruvec_irq (&cursor_lruvec, + cursor_page)) +continue ; + if (__isolate_lru_page(cursor_page, mode, file) == 0) { unsigned int isolated_pages; - struct lruvec *cursor_lruvec; int cursor_lru = page_lru(cursor_page); list_move(&cursor_page->lru, dst); isolated_pages = hpage_nr_pages(cursor_page); - cursor_lruvec = page_lruvec(cursor_page); cursor_lruvec->pages_count[cursor_lru] -= isolated_pages; VM_BUG_ON((long)cursor_lruvec->@@ -1235,6 +1239,9 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, } } +/* Restore original lruvec lock */ + cursor_lruvec =__relock_page_lruvec (cursor_lruvec, page); +/* If we break out of the loop above, lumpy reclaim failed */ if (pfn< end_pfn) nr_lumpy_failed++;@@ -1325,7 +1332,10 @@ static int too_many_isolated(struct zone *zone, int file,return isolated> inactive; } -static noinline_for_stack void +/* +* Returns currently locked lruvec +*/ +static noinline_for_stack struct lruvec *putback_inactive_pages (struct lruvec *lruvec,struct list_head *page_list) {@@ -1347,10 +1357,13 @@ putback_inactive_pages(struct lruvec *lruvec,lock_lruvec_irq (lruvec);continue ; } + + lruvec =__relock_page_lruvec (lruvec, page); + reclaim_stat =&lruvec->reclaim_stat; +SetPageLRU (page); lru =page_lru (page); - lruvec =page_lruvec (page);add_page_to_lru_list (lruvec, page, lru);if (is_active_lru (lru)) {int file =is_file_lru (lru);@@ -1375,6 +1388,8 @@ putback_inactive_pages(struct lruvec *lruvec,* To save our caller's stack, now use input list for pages to free. */ list_splice (&pages_to_free, page_list); + +return lruvec; }static noinline_for_stack void@@ -1544,7 +1559,7 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,__count_vm_events (KSWAPD_STEAL , nr_reclaimed);__count_zone_vm_events (PGSTEAL , zone, nr_reclaimed); -putback_inactive_pages (lruvec,&page_list); + lruvec =putback_inactive_pages (lruvec,&page_list);__mod_zone_page_state (zone,NR_ISOLATED_ANON , -nr_anon);__mod_zone_page_state (zone,NR_ISOLATED_FILE , -nr_file);@@ -1603,12 +1618,15 @@ shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,* * The downside is that we have to touch page->_count against each page. * But we had to alter page->flags anyway. +* +* Returns currently locked lruvec */ -static void move_active_pages_to_lru (struct lruvec *lruvec, -struct list_head *list, -struct list_head *pages_to_free, -enum lru_list lru) +static struct lruvec * +move_active_pages_to_lru (struct lruvec *lruvec, +struct list_head *list, +struct list_head *pages_to_free, +enum lru_list lru) {unsigned long pgmoved =0 ;struct page *page;@@ -1630,10 +1648,11 @@ static void move_active_pages_to_lru(struct lruvec *lruvec, page =lru_to_page (list); + lruvec =__relock_page_lruvec (lruvec, page); +VM_BUG_ON (PageLRU (page));SetPageLRU (page); - lruvec =page_lruvec (page);list_move (&page->lru,&lruvec->pages_lru[lru]); numpages =hpage_nr_pages (page); lruvec->pages_count[lru] += numpages;@@ -1655,6 +1674,8 @@ static void move_active_pages_to_lru(struct lruvec *lruvec,__mod_zone_page_state (lruvec_zone (lruvec),NR_LRU_BASE + lru, pgmoved);if (!is_active_lru (lru))__count_vm_events (PGDEACTIVATE , pgmoved); + +return lruvec; } static void shrink_active_list(unsigned long nr_to_scan,@@ -1744,9 +1765,9 @@ static void shrink_active_list(unsigned long nr_to_scan,*/ reclaim_stat->recent_rotated[file] += nr_rotated;
Quoted 14 lines
mm/vmscan.c static void shrink_active_list(unsigned long nr_to_scan, · line 1773
Quoted 9 lines
mm/vmscan.c +2
-2-move_active_pages_to_lru (lruvec,&l_active,&l_hold, + lruvec =move_active_pages_to_lru (lruvec,&l_active,&l_hold,LRU_ACTIVE + file *LRU_FILE ); -move_active_pages_to_lru (lruvec,&l_inactive,&l_hold, + lruvec =move_active_pages_to_lru (lruvec,&l_inactive,&l_hold,LRU_BASE + file *LRU_FILE );__mod_zone_page_state (zone,NR_ISOLATED_ANON + file, -nr_taken);unlock_lruvec_irq (lruvec);Hmm...could you add comments to each function as
"The caller should _lock_ lruvec before calling this functions.
This function returns a lruvec with _locked_. It may be different from passed one. And The callser should unlock lruvec"
Ok. Documenting is not my best side =)
Quoted 8 lines
Thanks,
-Kame
Quoted 6 lines
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email:<a href=mailto:"[email protected]"> [email protected]</a>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 16/21 mm: handle lruvec relocks in compaction
In reply to Konstantin KhlebnikovPrepare for lru_lock splitting in memory compaction code.
* disable irqs in acct_isolated() for __mod_zone_page_state(), lru_lock isn't required there.
--- mm/compaction.c | 30 ++++++++++++++++-------------- 1 files changed, 16 insertions(+), 14 deletions(-)
mm/compaction.c +16 -14
@@ -224,8 +224,10 @@ static void acct_isolated(struct zone *zone, struct compact_control *cc)list_for_each_entry (page, &cc->migratepages, lru) count[!!page_is_file_cache (page)]++; +local_irq_disable ();__mod_zone_page_state (zone,NR_ISOLATED_ANON , count[0 ]);__mod_zone_page_state (zone,NR_ISOLATED_FILE , count[1 ]); +local_irq_enable (); }/* Similar to reclaim, but different enough that they don't share logic */
@@ -262,7 +264,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,unsigned long nr_scanned =0 , nr_isolated =0 ;struct list_head *migratelist = &cc->migratepages;isolate_mode_t mode =ISOLATE_ACTIVE |ISOLATE_INACTIVE ; -struct lruvec *lruvec; +struct lruvec *lruvec =NULL ;/* Do not scan outside zone boundaries */ low_pfn =max (cc->migrate_pfn, zone->zone_start_pfn);
@@ -294,25 +296,24 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,/* Time to isolate some pages for migration */ cond_resched (); -spin_lock_irq (&zone->lru_lock);for (; low_pfn < end_pfn; low_pfn++) {struct page *page; -bool locked = true;/* give a chance to irqs before checking need_resched() */ if (!((low_pfn+1 ) %SWAP_CLUSTER_MAX )) { -spin_unlock_irq (&zone->lru_lock); - locked = false; +if (lruvec) +unlock_lruvec_irq (lruvec); + lruvec =NULL ; } -if (need_resched () ||spin_is_contended (&zone->lru_lock)) { -if (locked) -spin_unlock_irq (&zone->lru_lock); +if (need_resched () || + (lruvec &&spin_is_contended (&zone->lru_lock))) { +if (lruvec) +unlock_lruvec_irq (lruvec); + lruvec =NULL ;cond_resched (); -spin_lock_irq (&zone->lru_lock);if (fatal_signal_pending (current))break ; - }else if (!locked) -spin_lock_irq (&zone->lru_lock); + }/* * migrate_pfn does not necessarily start aligned to a
@@ -359,7 +360,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,continue ; } -if (!PageLRU (page)) +if (!__lock_page_lruvec_irq (&lruvec, page))continue ;/*
@@ -382,7 +383,6 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,VM_BUG_ON (PageTransCompound (page));/* Successfully isolated */ - lruvec =page_lruvec (page);del_page_from_lru_list (lruvec, page,page_lru (page));list_add (&page->lru, migratelist); cc->nr_migratepages++;
@@ -395,9 +395,11 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, } } +if (lruvec) +unlock_lruvec_irq (lruvec); +acct_isolated (zone, cc); -spin_unlock_irq (&zone->lru_lock); cc->migrate_pfn = low_pfn;trace_mm_compaction_isolate_migratepages (nr_scanned, nr_isolated);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 16/21] mm: handle lruvec relocks in compaction
Quoted 15 lines · On Thu, 23 Feb 2012 17:52:56 +0400 Konstantin Khlebnikov…
Prepare for
lru_locksplitting in memory compaction code.* disable irqs in
acct_isolated()for__mod_zone_page_state(),lru_lockisn't required there.Signed-off-by Konstantin Khlebnikov <[email protected]>--- mm/compaction.c | 30 ++++++++++++++++-------------- 1 files changed, 16 insertions(+), 14 deletions(-)mm/compaction.c +0
-0
mm/compaction.c static void acct_isolated(struct zone *zone, struct compact_control *cc) · line 230
Quoted 7 lines
mm/compaction.c +2
-0@@ -224,8 +224,10 @@ static void acct_isolated(struct zone *zone, struct compact_control *cc)list_for_each_entry (page, &cc->migratepages, lru) count[!!page_is_file_cache (page)]++; +local_irq_disable ();__mod_zone_page_state (zone,NR_ISOLATED_ANON , count[0 ]);__mod_zone_page_state (zone,NR_ISOLATED_FILE , count[1 ]); +local_irq_enable ();
Why we need to disable Irq here ??
Quoted 48 lines
mm/compaction.c +10
-11}/* Similar to reclaim, but different enough that they don't share logic */ @@ -262,7 +264,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,unsigned long nr_scanned =0 , nr_isolated =0 ;struct list_head *migratelist = &cc->migratepages;isolate_mode_t mode =ISOLATE_ACTIVE |ISOLATE_INACTIVE ; -struct lruvec *lruvec; +struct lruvec *lruvec =NULL ;/* Do not scan outside zone boundaries */ low_pfn =max (cc->migrate_pfn, zone->zone_start_pfn);@@ -294,25 +296,24 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,/* Time to isolate some pages for migration */ cond_resched (); -spin_lock_irq (&zone->lru_lock);for (; low_pfn < end_pfn; low_pfn++) {struct page *page; -bool locked = true;/* give a chance to irqs before checking need_resched() */ if (!((low_pfn+1 ) %SWAP_CLUSTER_MAX )) { -spin_unlock_irq (&zone->lru_lock); - locked = false; +if (lruvec) +unlock_lruvec_irq (lruvec); + lruvec =NULL ; } -if (need_resched () ||spin_is_contended (&zone->lru_lock)) { -if (locked) -spin_unlock_irq (&zone->lru_lock); +if (need_resched () || + (lruvec &&spin_is_contended (&zone->lru_lock))) { +if (lruvec) +unlock_lruvec_irq (lruvec); + lruvec =NULL ;cond_resched (); -spin_lock_irq (&zone->lru_lock);if (fatal_signal_pending (current))break ; - }else if (!locked) -spin_lock_irq (&zone->lru_lock); + }/* * migrate_pfn does not necessarily start aligned to a
mm/compaction.c static isolate_migrate_t isolate_migratepages(struct zone *zone, · line 364
Quoted 6 lines
mm/compaction.c +1
-1@@ -359,7 +360,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,continue ; } -if (!PageLRU (page)) +if (!__lock_page_lruvec_irq (&lruvec, page))continue ;
Could you add more comments onto __lock_page_lruvec_irq() ?
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 16/21] mm: handle lruvec relocks in compaction
Quoted 21 lines · KAMEZAWA Hiroyuki
Quoted 18 lines · On Thu, 23 Feb 2012 17:52:56 +0400 Konstantin…
Prepare for
lru_locksplitting in memory compaction code.* disable irqs in
acct_isolated()for__mod_zone_page_state(),lru_lockisn't required there.Signed-off-by Konstantin Khlebnikov<[email protected]>--- mm/compaction.c | 30 ++++++++++++++++-------------- 1 files changed, 16 insertions(+), 14 deletions(-)mm/compaction.c +0
-0@@ -224,8 +224,10 @@ static void acct_isolated(struct zone *zone, struct compact_control *cc)list_for_each_entry (page,&cc->migratepages, lru) count[!!page_is_file_cache (page)]++;
Quoted 6 lines
mm/compaction.c static void acct_isolated(struct zone *zone, struct compact_control *cc) · line 230
Quoted 4 lines
mm/compaction.c +2
-0+local_irq_disable ();__mod_zone_page_state (zone,NR_ISOLATED_ANON , count[0 ]);__mod_zone_page_state (zone,NR_ISOLATED_FILE , count[1 ]); +local_irq_enable ();Why we need to disable Irq here ??
__mod_zone_page_state() want this to protect per-cpu counters, maybe preempt_disable() is enough.
Quoted 54 lines
Quoted 54 lines
mm/compaction.c +10
-11}/* Similar to reclaim, but different enough that they don't share logic */ @@ -262,7 +264,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,unsigned long nr_scanned =0 , nr_isolated =0 ;struct list_head *migratelist =&cc->migratepages;isolate_mode_t mode =ISOLATE_ACTIVE |ISOLATE_INACTIVE ; -struct lruvec *lruvec; +struct lruvec *lruvec =NULL ;/* Do not scan outside zone boundaries */ low_pfn =max (cc->migrate_pfn, zone->zone_start_pfn);@@ -294,25 +296,24 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,/* Time to isolate some pages for migration */ cond_resched (); -spin_lock_irq (&zone->lru_lock);for (; low_pfn< end_pfn; low_pfn++) {struct page *page; -bool locked = true;/* give a chance to irqs before checking need_resched() */ if (!((low_pfn+1 ) %SWAP_CLUSTER_MAX )) { -spin_unlock_irq (&zone->lru_lock); - locked = false; +if (lruvec) +unlock_lruvec_irq (lruvec); + lruvec =NULL ; } -if (need_resched () ||spin_is_contended (&zone->lru_lock)) { -if (locked) -spin_unlock_irq (&zone->lru_lock); +if (need_resched () || + (lruvec&&spin_is_contended (&zone->lru_lock))) { +if (lruvec) +unlock_lruvec_irq (lruvec); + lruvec =NULL ;cond_resched (); -spin_lock_irq (&zone->lru_lock);if (fatal_signal_pending (current))break ; - }else if (!locked) -spin_lock_irq (&zone->lru_lock); + }/* * migrate_pfn does not necessarily start aligned to a @@ -359,7 +360,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,continue ; }
Quoted 5 lines
mm/compaction.c static isolate_migrate_t isolate_migratepages(struct zone *zone, · line 364
Quoted 3 lines
mm/compaction.c +1
-1-if (!PageLRU (page)) +if (!__lock_page_lruvec_irq (&lruvec, page))continue ;Could you add more comments onto
__lock_page_lruvec_irq()?
Actually there is a very unlikely race with page free-realloc,
(which is fixed in Hugh's patchset, and surprisingly fixed in my old memory controller) thus this part will be redesigned.
Quoted 4 lines
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 17/21 mm: handle lruvec relock in memory controller
In reply to Konstantin KhlebnikovCarefully relock lruvec lru lock at page memory cgroup change.
* In free_pn_rcu() wait for lruvec lock release.
Locking primitives keep lruvec pointer after successful lock held.
--- mm/memcontrol.c | 19 +++++++++++++------ 1 files changed, 13 insertions(+), 6 deletions(-)
mm/memcontrol.c +13 -6
@@ -2542,7 +2542,6 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg, enum charge_type ctype) {struct page_cgroup *pc =lookup_page_cgroup (page); -struct zone *zone =page_zone (page);struct lruvec *lruvec;unsigned long flags;bool removed = false;
@@ -2552,20 +2551,19 @@ __mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,* is already on LRU. It means the page may on some other page_cgroup's * LRU. Take care of it. */ -spin_lock_irqsave (&zone->lru_lock, flags); + lruvec =lock_page_lruvec (page, &flags);if (PageLRU (page)) { - lruvec =page_lruvec (page);del_page_from_lru_list (lruvec, page,page_lru (page));ClearPageLRU (page); removed = true; }__mem_cgroup_commit_charge (memcg, page,1 , pc, ctype);if (removed) { - lruvec =page_lruvec (page); + lruvec =__relock_page_lruvec (lruvec, page);add_page_to_lru_list (lruvec, page,page_lru (page));SetPageLRU (page); } -spin_unlock_irqrestore (&zone->lru_lock, flags); +unlock_lruvec (lruvec, &flags); } int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
@@ -4624,7 +4622,16 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)static void free_mem_cgroup_per_zone_info (struct mem_cgroup *memcg,int node) { -kfree (memcg->info.nodeinfo[node]); +struct mem_cgroup_per_node *pn = memcg->info.nodeinfo[node]; +int zone; + +if (!pn) +return ; + +for (zone =0 ; zone <MAX_NR_ZONES ; zone++) +wait_lruvec_unlock (&pn->zoneinfo[zone].lruvec); + +kfree (pn); }static struct mem_cgroup *mem_cgroup_alloc (void )
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 17/21] mm: handle lruvec relock in memory controller
Quoted 6 lines · On Thu, 23 Feb 2012 17:53:10 +0400 Konstantin Khlebnikov…
Carefully relock lruvec lru lock at page memory cgroup change.
* In
free_pn_rcu()wait for lruvec lock release.
Locking primitives keep lruvec pointer after successful lock held.Signed-off-by Konstantin Khlebnikov <[email protected]>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 18/21 mm: add to lruvec isolated pages counters
In reply to Konstantin KhlebnikovThis patch adds into struct lruvec counter of isolated pages.
It is required for keeping lruvec alive till the isolated page putback. We cannot rely on resource counter in memory controller, because it does not account uncharged memory. And it much better to have common engine for dynamical lruvec management, than to tie all logic with memory cgroup magic. Plus this is useful information for memory reclaimer balancing.
There also appears per-cpu page-vectors for putting isolated pages back, and function add_page_to_evictable_list(). It is similar to lru_cache_add_lru() but it reuse page reference from caller and can adjust isolated pages counters. There also new function free_isolated_page_list() which is used at the end of shrink_page_list() for freeing pages and adjusting counters of isolated pages.
Memory cgroups code can shuffle pages between lruvecs without isolation if page is already isolated with someone else. Thus page lruvec reference is unstable even if page is isolated. It is stable only under lru_lock or if page reference count is zero. That's why we must always recheck page_lruvec() even on non-lumpy 0-order reclaim, where all pages are isolated from one lruvec.
Memory controller at moving pege between cgroups now adjust isolated pages counter for old lruvec before inserting page to new lruvec. Locking lruvec->lru_lock in mem_cgroup_adjust_isolated() also effectively stabilizes PageLRU() sign, so nobody will see PageLRU() under old lru_lock while page is already moved into other lruvec.
[ BTW, all lru-id arithmetic can be simplified if we devide unevictable list
into file and anon parts. After that we can swap bits in page->flags and calculate lru-id with single instruction ]
--- include/linux/mmzone.h | 11 ++++- include/linux/swap.h | 4 +- mm/compaction.c | 1 mm/huge_memory.c | 4 ++ mm/internal.h | 6 ++ mm/ksm.c | 2 - mm/memcontrol.c | 39 ++++++++++++++-- mm/migrate.c | 2 - mm/rmap.c | 2 - mm/swap.c | 76 +++++++++++++++++++++++++++++++ mm/vmscan.c | 116 +++++++++++++++++++++++++++++++++++------------- 11 files changed, 218 insertions(+), 45 deletions(-)
include/linux/mmzone.h +9 -2
@@ -137,13 +137,20 @@ enum lru_list {LRU_INACTIVE_FILE =LRU_BASE +LRU_FILE ,LRU_ACTIVE_FILE =LRU_BASE +LRU_FILE +LRU_ACTIVE ,LRU_UNEVICTABLE , - NR_LRU_LISTS +NR_EVICTABLE_LRU_LISTS =LRU_UNEVICTABLE , +NR_LRU_LISTS , +LRU_ISOLATED =NR_LRU_LISTS , +LRU_ISOLATED_ANON =LRU_ISOLATED , +LRU_ISOLATED_FILE , + NR_LRU_COUNTERS, };#define for_each_lru (lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)#define for_each_evictable_lru (lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++) +#define for_each_lru_counter (cnt) for (cnt = 0; cnt < NR_LRU_COUNTERS; cnt++) +static inline int is_file_lru (enum lru_list lru) {return (lru ==LRU_INACTIVE_FILE || lru ==LRU_ACTIVE_FILE );
@@ -298,7 +305,7 @@ struct zone_reclaim_stat {struct lruvec {struct list_head pages_lru[NR_LRU_LISTS ]; -unsigned long pages_count[NR_LRU_LISTS ]; +unsigned long pages_count[NR_LRU_COUNTERS ];struct zone_reclaim_stat reclaim_stat;
include/linux/swap.h +3 -1
@@ -234,7 +234,9 @@ extern void rotate_reclaimable_page(struct page *page);extern void deactivate_page (struct page *page);extern void swap_setup (void ); -extern void add_page_to_unevictable_list (struct page *page); +extern void add_page_to_unevictable_list (struct page *page,bool isolated); +extern void add_page_to_evictable_list (struct page *page, +enum lru_list lru,bool isolated);/** * lru_cache_add: add a page to the page lists
mm/compaction.c +1 -0
@@ -384,6 +384,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,/* Successfully isolated */ del_page_from_lru_list (lruvec, page,page_lru (page)); + lruvec->pages_count[LRU_ISOLATED +page_is_file_cache (page)]++;list_add (&page->lru, migratelist); cc->nr_migratepages++; nr_isolated++;
mm/huge_memory.c +4 -0
@@ -1316,6 +1316,10 @@ static void __split_huge_page_refcount(struct page *page)__dec_zone_page_state (page,NR_ANON_TRANSPARENT_HUGEPAGES );__mod_zone_page_state (lruvec_zone (lruvec),NR_ANON_PAGES ,HPAGE_PMD_NR ); +/* Fixup isolated pages counter if head page currently isolated */ +if (!PageLRU (page)) + lruvec->pages_count[LRU_ISOLATED_ANON ] -=HPAGE_PMD_NR -1 ; +ClearPageCompound (page);compound_unlock (page);unlock_lruvec_irq (lruvec);
mm/internal.h +5 -1
@@ -265,7 +265,11 @@ extern unsigned long highest_memmap_pfn;* in mm/vmscan.c: */ extern int isolate_lru_page (struct page *page); -extern void putback_lru_page (struct page *page); +extern void __putback_lru_page (struct page *page,bool isolated); +static inline void putback_lru_page (struct page *page) +{ +__putback_lru_page (page, true); +}/* * in mm/page_alloc.c
mm/ksm.c +1 -1
@@ -1592,7 +1592,7 @@ struct page *ksm_does_need_to_copy(struct page *page,if (page_evictable (new_page, vma))lru_cache_add_lru (new_page,LRU_ACTIVE_ANON );else -add_page_to_unevictable_list (new_page); +add_page_to_unevictable_list (new_page, false); }return new_page;
mm/memcontrol.c +34 -5
@@ -697,7 +697,7 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid, mz =mem_cgroup_zoneinfo (memcg, nid, zid); -for_each_lru (lru) { +for_each_lru_counter (lru) {if (BIT (lru) & lru_mask) ret += mz->lruvec.pages_count[lru]; }
@@ -2354,6 +2354,17 @@ void mem_cgroup_split_huge_fixup(struct page *head) }#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +static void mem_cgroup_adjust_isolated (struct lruvec *lruvec, +struct page *page,int delta) +{ +int file =page_is_file_cache (page); +unsigned long flags; + +lock_lruvec (lruvec, &flags); + lruvec->pages_count[LRU_ISOLATED + file] += delta; +unlock_lruvec (lruvec, &flags); +} +/** * mem_cgroup_move_account - move account of the page * @page: the page
@@ -2452,6 +2463,7 @@ static int mem_cgroup_move_parent(struct page *page,struct mem_cgroup *parent;unsigned int nr_pages;unsigned long uninitialized_var (flags ); +struct lruvec *lruvec;int ret;/* Is ROOT ? */
@@ -2471,6 +2483,8 @@ static int mem_cgroup_move_parent(struct page *page,if (ret) goto put_back; + lruvec =page_lruvec (page); +if (nr_pages >1 ) flags =compound_lock_irqsave (page);
@@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page,if (nr_pages >1 )compound_unlock_irqrestore (page, flags); +if (!ret) +/* This also stabilize PageLRU() sign for lruvec lock holder. */ +mem_cgroup_adjust_isolated (lruvec, page, -nr_pages); put_back: -putback_lru_page (page); +__putback_lru_page (page, !ret); put:put_page (page); out:
@@ -3879,6 +3896,8 @@ enum {MCS_INACTIVE_FILE ,MCS_ACTIVE_FILE ,MCS_UNEVICTABLE , +MCS_ISOLATED_ANON , +MCS_ISOLATED_FILE , NR_MCS_STAT, };
@@ -3902,7 +3921,9 @@ struct { {"active_anon", "total_active_anon"}, {"inactive_file", "total_inactive_file"}, {"active_file", "total_active_file"}, - {"unevictable", "total_unevictable"} + {"unevictable", "total_unevictable"}, + {"isolated_anon", "total_isolated_anon"}, + {"isolated_file", "total_isolated_file"}, };
@@ -3942,6 +3963,10 @@ mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s) s->stat[MCS_ACTIVE_FILE ] += val *PAGE_SIZE ; val =mem_cgroup_nr_lru_pages (memcg,BIT (LRU_UNEVICTABLE )); s->stat[MCS_UNEVICTABLE ] += val *PAGE_SIZE ; + val =mem_cgroup_nr_lru_pages (memcg,BIT (LRU_ISOLATED_ANON )); + s->stat[MCS_ISOLATED_ANON ] += val *PAGE_SIZE ; + val =mem_cgroup_nr_lru_pages (memcg,BIT (LRU_ISOLATED_FILE )); + s->stat[MCS_ISOLATED_FILE ] += val *PAGE_SIZE ; } static void
@@ -5243,6 +5268,7 @@ retry:struct page *page;struct page_cgroup *pc;swp_entry_t ent; +struct lruvec *lruvec;if (!mc.precharge)break ;
@@ -5253,14 +5279,17 @@ retry: page = target.page;if (isolate_lru_page (page)) goto put; + lruvec =page_lruvec (page); pc =lookup_page_cgroup (page);if (!mem_cgroup_move_account (page,1 , pc, mc.from, mc.to, false)) { mc.precharge--;/* we uncharge from mc.from later. */ mc.moved_charge++; - } -putback_lru_page (page); +mem_cgroup_adjust_isolated (lruvec, page,-1 ); +__putback_lru_page (page, false); + }else +__putback_lru_page (page, true); put:/* is_target_pte_for_mc() gets the page */ put_page (page);break ;
mm/migrate.c +1 -1
@@ -868,7 +868,7 @@ out:* Move the new page to the LRU. If migration was not successful * then this will free the page. */ -putback_lru_page (newpage); +__putback_lru_page (newpage, false);if (result) {if (rc) *result = rc;
mm/rmap.c +1 -1
@@ -1139,7 +1139,7 @@ void page_add_new_anon_rmap(struct page *page,if (page_evictable (page, vma))lru_cache_add_lru (page,LRU_ACTIVE_ANON );else -add_page_to_unevictable_list (page); +add_page_to_unevictable_list (page, false); }/**
mm/swap.c +75 -1
106 lines not on this page · raw message
mm/vmscan.c +84 -32
216 lines not on this page · raw message
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 18/21] mm: add to lruvec isolated pages counters
Review static int mem_cgroup_move_parent(struct page *page, · line 2504
Quoted 13 lines · Konstantin Khlebnikov
(quoted diff) +4
-1@@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page, if (nr_pages> 1) compound_unlock_irqrestore(page, flags); + if (!ret) + /* This also stabilize PageLRU() sign for lruvec lock holder. */ + mem_cgroup_adjust_isolated(lruvec, page, -nr_pages); put_back: - putback_lru_page(page); + __putback_lru_page(page, !ret); put: put_page(page); out:
Oh, no. There must be !!ret
mm/memcontrol.c +1 -1
@@ -2482,7 +2482,7 @@ static int mem_cgroup_move_parent(struct page *page,/* This also stabilize PageLRU() sign for lruvec lock holder. */ mem_cgroup_adjust_isolated (lruvec, page, -nr_pages); put_back: -__putback_lru_page (page, !ret); +__putback_lru_page (page, !!ret); put:put_page (page); out:
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 18/21] mm: add to lruvec isolated pages counters
In reply to Konstantin KhlebnikovQuoted 25 lines · On Thu, 23 Feb 2012 17:53:14 +0400 Konstantin Khlebnikov…
This patch adds into struct lruvec counter of isolated pages.
It is required for keeping lruvec alive till the isolated page putback. We cannot rely on resource counter in memory controller, because it does not account uncharged memory. And it much better to have common engine for dynamical lruvec management, than to tie all logic with memory cgroup magic. Plus this is useful information for memory reclaimer balancing.There also appears per-cpu page-vectors for putting isolated pages back, and function
add_page_to_evictable_list(). It is similar tolru_cache_add_lru()but it reuse page reference from caller and can adjust isolated pages counters. There also new functionfree_isolated_page_list()which is used at the end ofshrink_page_list()for freeing pages and adjusting counters of isolated pages.Memory cgroups code can shuffle pages between lruvecs without isolation if page is already isolated with someone else. Thus page lruvec reference is unstable even if page is isolated. It is stable only under
lru_lockor if page reference count is zero. That's why we must always recheckpage_lruvec()even on non-lumpy 0-order reclaim, where all pages are isolated from one lruvec.Memory controller at moving pege between cgroups now adjust isolated pages counter for old lruvec before inserting page to new lruvec. Locking
lruvec->lru_lockinmem_cgroup_adjust_isolated()also effectively stabilizesPageLRU()sign, so nobody will seePageLRU()under oldlru_lockwhile page is already moved into other lruvec.
Quoted 5 lines
[ BTW, all lru-id arithmetic can be simplified if we devide unevictable list
into file and anon parts. After that we can swap bits inpage->flagsand calculate lru-id with single instruction ]Signed-off-by Konstantin Khlebnikov <[email protected]>
Could you split the part showing statistics of isolated pages into another patch ?
Quoted 502 lines
--- include/linux/mmzone.h | 11 ++++- include/linux/swap.h | 4 +- mm/compaction.c | 1 mm/huge_memory.c | 4 ++ mm/internal.h | 6 ++ mm/ksm.c | 2 - mm/memcontrol.c | 39 ++++++++++++++-- mm/migrate.c | 2 - mm/rmap.c | 2 - mm/swap.c | 76 +++++++++++++++++++++++++++++++ mm/vmscan.c | 116 +++++++++++++++++++++++++++++++++++------------- 11 files changed, 218 insertions(+), 45 deletions(-)include/linux/mmzone.h +9
-2@@ -137,13 +137,20 @@ enum lru_list {LRU_INACTIVE_FILE =LRU_BASE +LRU_FILE ,LRU_ACTIVE_FILE =LRU_BASE +LRU_FILE +LRU_ACTIVE ,LRU_UNEVICTABLE , - NR_LRU_LISTS +NR_EVICTABLE_LRU_LISTS =LRU_UNEVICTABLE , +NR_LRU_LISTS , +LRU_ISOLATED =NR_LRU_LISTS , +LRU_ISOLATED_ANON =LRU_ISOLATED , +LRU_ISOLATED_FILE , + NR_LRU_COUNTERS, };#define for_each_lru (lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)#define for_each_evictable_lru (lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++) +#define for_each_lru_counter (cnt) for (cnt = 0; cnt < NR_LRU_COUNTERS; cnt++) +static inline int is_file_lru (enum lru_list lru) {return (lru ==LRU_INACTIVE_FILE || lru ==LRU_ACTIVE_FILE );@@ -298,7 +305,7 @@ struct zone_reclaim_stat {struct lruvec {struct list_head pages_lru[NR_LRU_LISTS ]; -unsigned long pages_count[NR_LRU_LISTS ]; +unsigned long pages_count[NR_LRU_COUNTERS ];struct zone_reclaim_stat reclaim_stat;include/linux/swap.h +3
-1@@ -234,7 +234,9 @@ extern void rotate_reclaimable_page(struct page *page);extern void deactivate_page (struct page *page);extern void swap_setup (void ); -extern void add_page_to_unevictable_list (struct page *page); +extern void add_page_to_unevictable_list (struct page *page,bool isolated); +extern void add_page_to_evictable_list (struct page *page, +enum lru_list lru,bool isolated);/** * lru_cache_add: add a page to the page lists mm/compaction.c +1
-0@@ -384,6 +384,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,/* Successfully isolated */ del_page_from_lru_list (lruvec, page,page_lru (page)); + lruvec->pages_count[LRU_ISOLATED +page_is_file_cache (page)]++;list_add (&page->lru, migratelist); cc->nr_migratepages++; nr_isolated++;mm/huge_memory.c +4
-0@@ -1316,6 +1316,10 @@ static void __split_huge_page_refcount(struct page *page)__dec_zone_page_state (page,NR_ANON_TRANSPARENT_HUGEPAGES );__mod_zone_page_state (lruvec_zone (lruvec),NR_ANON_PAGES ,HPAGE_PMD_NR ); +/* Fixup isolated pages counter if head page currently isolated */ +if (!PageLRU (page)) + lruvec->pages_count[LRU_ISOLATED_ANON ] -=HPAGE_PMD_NR -1 ; +ClearPageCompound (page);compound_unlock (page);unlock_lruvec_irq (lruvec);mm/internal.h +5
-1@@ -265,7 +265,11 @@ extern unsigned long highest_memmap_pfn;* in mm/vmscan.c: */ extern int isolate_lru_page (struct page *page); -extern void putback_lru_page (struct page *page); +extern void __putback_lru_page (struct page *page,bool isolated); +static inline void putback_lru_page (struct page *page) +{ +__putback_lru_page (page, true); +}/* * in mm/page_alloc.c mm/ksm.c +1
-1@@ -1592,7 +1592,7 @@ struct page *ksm_does_need_to_copy(struct page *page,if (page_evictable (new_page, vma))lru_cache_add_lru (new_page,LRU_ACTIVE_ANON );else -add_page_to_unevictable_list (new_page); +add_page_to_unevictable_list (new_page, false); }return new_page;mm/memcontrol.c +34
-5@@ -697,7 +697,7 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid, mz =mem_cgroup_zoneinfo (memcg, nid, zid); -for_each_lru (lru) { +for_each_lru_counter (lru) {if (BIT (lru) & lru_mask) ret += mz->lruvec.pages_count[lru]; }@@ -2354,6 +2354,17 @@ void mem_cgroup_split_huge_fixup(struct page *head) }#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +static void mem_cgroup_adjust_isolated (struct lruvec *lruvec, +struct page *page,int delta) +{ +int file =page_is_file_cache (page); +unsigned long flags; + +lock_lruvec (lruvec, &flags); + lruvec->pages_count[LRU_ISOLATED + file] += delta; +unlock_lruvec (lruvec, &flags); +} +/** * mem_cgroup_move_account - move account of the page * @page: the page @@ -2452,6 +2463,7 @@ static int mem_cgroup_move_parent(struct page *page,struct mem_cgroup *parent;unsigned int nr_pages;unsigned long uninitialized_var (flags ); +struct lruvec *lruvec;int ret;/* Is ROOT ? */ @@ -2471,6 +2483,8 @@ static int mem_cgroup_move_parent(struct page *page,if (ret) goto put_back; + lruvec =page_lruvec (page); +if (nr_pages >1 ) flags =compound_lock_irqsave (page);@@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page,if (nr_pages >1 )compound_unlock_irqrestore (page, flags); +if (!ret) +/* This also stabilize PageLRU() sign for lruvec lock holder. */ +mem_cgroup_adjust_isolated (lruvec, page, -nr_pages); put_back: -putback_lru_page (page); +__putback_lru_page (page, !ret); put:put_page (page); out:@@ -3879,6 +3896,8 @@ enum {MCS_INACTIVE_FILE ,MCS_ACTIVE_FILE ,MCS_UNEVICTABLE , +MCS_ISOLATED_ANON , +MCS_ISOLATED_FILE , NR_MCS_STAT, };@@ -3902,7 +3921,9 @@ struct { {"active_anon", "total_active_anon"}, {"inactive_file", "total_inactive_file"}, {"active_file", "total_active_file"}, - {"unevictable", "total_unevictable"} + {"unevictable", "total_unevictable"}, + {"isolated_anon", "total_isolated_anon"}, + {"isolated_file", "total_isolated_file"}, };@@ -3942,6 +3963,10 @@ mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s) s->stat[MCS_ACTIVE_FILE ] += val *PAGE_SIZE ; val =mem_cgroup_nr_lru_pages (memcg,BIT (LRU_UNEVICTABLE )); s->stat[MCS_UNEVICTABLE ] += val *PAGE_SIZE ; + val =mem_cgroup_nr_lru_pages (memcg,BIT (LRU_ISOLATED_ANON )); + s->stat[MCS_ISOLATED_ANON ] += val *PAGE_SIZE ; + val =mem_cgroup_nr_lru_pages (memcg,BIT (LRU_ISOLATED_FILE )); + s->stat[MCS_ISOLATED_FILE ] += val *PAGE_SIZE ; } static void@@ -5243,6 +5268,7 @@ retry:struct page *page;struct page_cgroup *pc;swp_entry_t ent; +struct lruvec *lruvec;if (!mc.precharge)break ;@@ -5253,14 +5279,17 @@ retry: page = target.page;if (isolate_lru_page (page)) goto put; + lruvec =page_lruvec (page); pc =lookup_page_cgroup (page);if (!mem_cgroup_move_account (page,1 , pc, mc.from, mc.to, false)) { mc.precharge--;/* we uncharge from mc.from later. */ mc.moved_charge++; - } -putback_lru_page (page); +mem_cgroup_adjust_isolated (lruvec, page,-1 ); +__putback_lru_page (page, false); + }else +__putback_lru_page (page, true); put:/* is_target_pte_for_mc() gets the page */ put_page (page);break ;mm/migrate.c +1
-1@@ -868,7 +868,7 @@ out:* Move the new page to the LRU. If migration was not successful * then this will free the page. */ -putback_lru_page (newpage); +__putback_lru_page (newpage, false);if (result) {if (rc) *result = rc;mm/rmap.c +1
-1@@ -1139,7 +1139,7 @@ void page_add_new_anon_rmap(struct page *page,if (page_evictable (page, vma))lru_cache_add_lru (page,LRU_ACTIVE_ANON );else -add_page_to_unevictable_list (page); +add_page_to_unevictable_list (page, false); }/** mm/swap.c +75
-1@@ -37,6 +37,8 @@int page_cluster; static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs); +static DEFINE_PER_CPU(struct pagevec[NR_EVICTABLE_LRU_LISTS], + lru_add_isolated_pvecs); static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);@@ -381,6 +383,67 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)__lru_cache_add (page, lru); } +static void __lru_add_isolated_fn (struct lruvec *lruvec, +struct page *page,void *arg) +{ +enum lru_list lru = (enum lru_list )arg; + +VM_BUG_ON (PageActive (page)); +VM_BUG_ON (PageUnevictable (page)); +VM_BUG_ON (PageLRU (page)); + +SetPageLRU (page); +if (is_active_lru (lru)) +SetPageActive (page); +update_page_reclaim_stat (lruvec, lru); +add_page_to_lru_list (lruvec, page, lru); + lruvec->pages_count[LRU_ISOLATED +is_file_lru (lru)] -= +hpage_nr_pages (page); +} + +static void __lru_add_isolated (struct pagevec *pvec,enum lru_list lru) +{ +VM_BUG_ON (is_unevictable_lru (lru)); +pagevec_lru_move_fn (pvec, __lru_add_isolated_fn, (void *)lru); +} + +/** +* add_page_to_evictable_list - add page to lru list +* @page the page to be added into the lru list +* @lru lru list id +* @isolated need to adjust isolated pages counter +* +* Like lru_cache_add_lru() but reuses caller's reference to page and +* taking care about isolated pages counter on lruvec if isolated = true. +*/ +void add_page_to_evictable_list (struct page *page, +enum lru_list lru,bool isolated) +{ +struct pagevec *pvec; + +if (PageActive (page)) { +VM_BUG_ON (PageUnevictable (page)); +ClearPageActive (page); + }else if (PageUnevictable (page)) { +VM_BUG_ON (PageActive (page)); +ClearPageUnevictable (page); + } + +VM_BUG_ON (PageLRU (page) ||PageActive (page) ||PageUnevictable (page)); + +preempt_disable (); +if (isolated) { + pvec =__this_cpu_ptr (lru_add_isolated_pvecs + lru); +if (!pagevec_add (pvec, page)) +__lru_add_isolated (pvec, lru); + }else { + pvec =__this_cpu_ptr (lru_add_pvecs + lru); +if (!pagevec_add (pvec, page)) +__pagevec_lru_add (pvec, lru); + } +preempt_enable (); +} +/** * add_page_to_unevictable_list - add a page to the unevictable list * @page: the page to be added to the unevictable list @@ -391,7 +454,7 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)* while it's locked or otherwise "invisible" to other tasks. This is * difficult to do when using the pagevec cache, so bypass that. */ -void add_page_to_unevictable_list (struct page *page) +void add_page_to_unevictable_list (struct page *page,bool isolated) {struct lruvec *lruvec;@@ -399,6 +462,10 @@ void add_page_to_unevictable_list(struct page *page)SetPageUnevictable (page);SetPageLRU (page);add_page_to_lru_list (lruvec, page,LRU_UNEVICTABLE ); +if (isolated) { +int type =LRU_ISOLATED +page_is_file_cache (page); + lruvec->pages_count[type] -=hpage_nr_pages (page); + }unlock_lruvec_irq (lruvec); }@@ -485,6 +552,13 @@ static void drain_cpu_pagevecs(int cpu)__pagevec_lru_add (pvec, lru); } + pvecs =per_cpu (lru_add_isolated_pvecs, cpu); +for_each_evictable_lru (lru) { + pvec = &pvecs[lru]; +if (pagevec_count (pvec)) +__lru_add_isolated (pvec, lru); + } + pvec = &per_cpu (lru_rotate_pvecs, cpu);if (pagevec_count (pvec)) {unsigned long flags;mm/vmscan.c +43
-26@@ -42,6 +42,7 @@#include <linux/sysctl.h> #include <linux/oom.h> #include <linux/prefetch.h> +#include <trace/events/kmem.h> #include <asm/tlbflush.h> #include <asm/div64.h> @@ -585,15 +586,16 @@ int remove_mapping(struct address_space *mapping, struct page *page) }/** -* putback_lru_page - put previously isolated page onto appropriate LRU list +* __putback_lru_page - put previously isolated page onto appropriate LRU list * @page: page to be put back to appropriate lru list +* @isolated: isolated pages counter update required * * Add previously isolated @page to appropriate LRU list. * Page may still be unevictable for other reasons. * * lru_lock must not be held, interrupts must be enabled. */ -void putback_lru_page (struct page *page) +void __putback_lru_page (struct page *page,bool isolated) {int lru;int active = !!TestClearPageActive (page);@@ -612,14 +614,16 @@ redo:* We know how to handle that. */ lru = active +page_lru_base_type (page); -lru_cache_add_lru (page, lru); +add_page_to_evictable_list (page, lru, isolated); +if (was_unevictable) +count_vm_event (UNEVICTABLE_PGRESCUED ); }else {/* * Put unevictable pages directly on zone's unevictable * list. */ lru =LRU_UNEVICTABLE ; -add_page_to_unevictable_list (page); +add_page_to_unevictable_list (page, isolated);/* * When racing with an mlock or AS_UNEVICTABLE clearing * (page is unlocked) make sure that if the other thread @@ -631,30 +635,26 @@ redo:* The other side is TestClearPageMlocked() or shmem_lock(). */ smp_mb (); - } - -/* -* page's status can change while we move it among lru. If an evictable -* page is on unevictable list, it never be freed. To avoid that, -* check after we added it to the list, again. -*/ -if (lru ==LRU_UNEVICTABLE &&page_evictable (page,NULL )) { -if (!isolate_lru_page (page)) { -put_page (page); - goto redo; - } -/* This means someone else dropped this page from LRU -* So, it will be freed or putback to LRU again. There is -* nothing to do here. +/* +* page's status can change while we move it among lru. +* If an evictable page is on unevictable list, it never be freed. +* To avoid that, check after we added it to the list, again. */ +if (page_evictable (page,NULL )) { +if (!isolate_lru_page (page)) { + isolated = true; +put_page (page); + goto redo; + } +/* This means someone else dropped this page from LRU +* So, it will be freed or putback to LRU again. There is +* nothing to do here. +*/ + } +put_page (page);/* drop ref from isolate */ +if (!was_unevictable) +count_vm_event (UNEVICTABLE_PGCULLED ); } - -if (was_unevictable && lru !=LRU_UNEVICTABLE ) -count_vm_event (UNEVICTABLE_PGRESCUED ); -else if (!was_unevictable && lru ==LRU_UNEVICTABLE ) -count_vm_event (UNEVICTABLE_PGCULLED ); - -put_page (page);/* drop ref from isolate */ }enum page_references {@@ -724,6 +724,48 @@ static enum page_references page_check_references(struct page *page, }/* +* Free a list of isolated 0-order pages +*/ +static void free_isolated_page_list (struct lruvec *lruvec, +struct list_head *list,int cold) +{ +struct page *page, *next; +unsigned long nr_pages[2 ]; +struct list_head queue; + +again: +INIT_LIST_HEAD (&queue); + nr_pages[0 ] = nr_pages[1 ] =0 ; + +list_for_each_entry_safe (page, next, list, lru) { +if (unlikely (lruvec !=page_lruvec (page))) { +list_add_tail (&page->lru, &queue); +continue ;
mm/vmscan.c static enum page_references page_check_references(struct page *page, · line 753
Quoted 10 lines
mm/vmscan.c +10
-0+ } + nr_pages[page_is_file_cache (page)]++; +trace_mm_page_free_batched (page, cold); +free_hot_cold_page (page, cold); + } + +lock_lruvec_irq (lruvec); + lruvec->pages_count[LRU_ISOLATED_ANON ] -= nr_pages[0 ]; + lruvec->pages_count[LRU_ISOLATED_FILE ] -= nr_pages[1 ]; +unlock_lruvec_irq (lruvec);
Is it guaranteed that all pages are from the same lruvec ?
Thanks,
-Kame
Quoted 104 lines
mm/vmscan.c +31
-7+ +/* +* Usually there will be only one iteration, because +* at 0-order reclaim all pages are from one lruvec +* if we didn't raced with memory cgroup shuffling. +*/ +if (unlikely (!list_empty (&queue))) { +list_replace (&queue, list); + lruvec =page_lruvec (list_first_entry (list, + struct page, lru)); + goto again; + } +} + +/* * shrink_page_list() returns the number of reclaimed pages */ static unsigned long shrink_page_list(struct list_head *page_list,@@ -986,7 +1028,7 @@ keep_lumpy:if (nr_dirty && nr_dirty == nr_congested &&global_reclaim (sc))zone_set_flag (lruvec_zone (lruvec),ZONE_CONGESTED ); -free_hot_cold_page_list (&free_pages,1 ); +free_isolated_page_list (lruvec, &free_pages,1 );list_splice (&ret_pages, page_list);count_vm_events (PGACTIVATE , pgactivate);@@ -1206,11 +1248,14 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,if (__isolate_lru_page (cursor_page, mode, file) ==0 ) {unsigned int isolated_pages;int cursor_lru =page_lru (cursor_page); +int cur_file =page_is_file_cache (cursor_page);list_move (&cursor_page->lru, dst); isolated_pages =hpage_nr_pages (cursor_page); cursor_lruvec->pages_count[cursor_lru] -= isolated_pages; + cursor_lruvec->pages_count[LRU_ISOLATED + + cur_file] += isolated_pages;VM_BUG_ON ((long )cursor_lruvec-> pages_count[cursor_lru] <0 ); nr_taken += isolated_pages;@@ -1248,6 +1293,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan, } lruvec->pages_count[lru] -= nr_taken - nr_lumpy_taken; + lruvec->pages_count[LRU_ISOLATED + file] += nr_taken - nr_lumpy_taken;VM_BUG_ON ((long )lruvec->pages_count[lru] <0 ); *nr_scanned = scan;@@ -1296,11 +1342,14 @@ int isolate_lru_page(struct page *page) lruvec =lock_page_lruvec_irq (page);if (PageLRU (page)) { +int file =page_is_file_cache (page);int lru =page_lru (page); ret =0 ;get_page (page);ClearPageLRU (page);del_page_from_lru_list (lruvec, page, lru); + lruvec->pages_count[LRU_ISOLATED + file] += +hpage_nr_pages (page); }unlock_lruvec_irq (lruvec); }@@ -1347,7 +1396,7 @@ putback_inactive_pages(struct lruvec *lruvec,*/ while (!list_empty (page_list)) {struct page *page =lru_to_page (page_list); -int lru; +int numpages, lru, file;VM_BUG_ON (PageLRU (page));list_del (&page->lru);@@ -1363,13 +1412,13 @@ putback_inactive_pages(struct lruvec *lruvec,SetPageLRU (page); lru =page_lru (page); + file =is_file_lru (lru); + numpages =hpage_nr_pages (page);add_page_to_lru_list (lruvec, page, lru); -if (is_active_lru (lru)) { -int file =is_file_lru (lru); -int numpages =hpage_nr_pages (page); + lruvec->pages_count[LRU_ISOLATED + file] -= numpages; +if (is_active_lru (lru)) reclaim_stat->recent_rotated[file] += numpages; - }if (put_page_testzero (page)) {__ClearPageLRU (page);__ClearPageActive (page);@@ -1656,6 +1705,9 @@ move_active_pages_to_lru(struct lruvec *lruvec,list_move (&page->lru, &lruvec->pages_lru[lru]); numpages =hpage_nr_pages (page); lruvec->pages_count[lru] += numpages; +/* There should be no mess between file and anon pages */ + lruvec->pages_count[LRU_ISOLATED + +is_file_lru (lru)] -= numpages; pgmoved += numpages;if (put_page_testzero (page)) { --
Quoted 5 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 19/21 memcg: check lru vectors emptiness in pre-destroy
In reply to Konstantin KhlebnikovWe must abort cgroup destroying if it still not empty, resource counter cannot catch isolated uncharged pages.
--- mm/memcontrol.c | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-)
mm/memcontrol.c +9 -1
@@ -4859,8 +4859,16 @@ free_out:static int mem_cgroup_pre_destroy (struct cgroup *cont) {struct mem_cgroup *memcg =mem_cgroup_from_cont (cont); +int ret; + + ret =mem_cgroup_force_empty (memcg, false); +if (ret) +return ret; -return mem_cgroup_force_empty (memcg, false); +if (mem_cgroup_nr_lru_pages (memcg,-1 )) +return -EBUSY ; + +return 0 ; }static void mem_cgroup_destroy (struct cgroup *cont)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 19/21] memcg: check lru vectors emptiness in pre-destroy
Quoted 4 lines · On Thu, 23 Feb 2012 17:53:19 +0400 Konstantin Khlebnikov…
We must abort cgroup destroying if it still not empty, resource counter cannot catch isolated uncharged pages.
Signed-off-by Konstantin Khlebnikov <[email protected]>
I like this.
Quoted 28 lines
--- mm/memcontrol.c | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-)mm/memcontrol.c +9
-2@@ -4859,8 +4859,16 @@ free_out:static int mem_cgroup_pre_destroy (struct cgroup *cont) {struct mem_cgroup *memcg =mem_cgroup_from_cont (cont); +int ret; + + ret =mem_cgroup_force_empty (memcg, false); +if (ret) +return ret; -return mem_cgroup_force_empty (memcg, false); +if (mem_cgroup_nr_lru_pages (memcg,-1 )) +return -EBUSY ; + +return 0 ; }static void mem_cgroup_destroy (struct cgroup *cont) --
Quoted 5 lines
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 20/21 mm: split zone->lru_lock
In reply to Konstantin KhlebnikovLooks like all ready for splitting zone->lru_lock into per-lruvec pieces.
lruvec locking loop protected with rcu, actually there is irq-disabling instead of rcu_read_lock(). Memory controller already releases its lru-vectors after syncronize_rcu() in cgroup_diput(). Probably it should be replaced with synchronize_sched()
--- include/linux/mmzone.h | 3 +- mm/compaction.c | 2 + mm/internal.h | 66 +++++++++++++++++++++++++----------------------- mm/page_alloc.c | 2 + mm/swap.c | 2 + 5 files changed, 40 insertions(+), 35 deletions(-)
include/linux/mmzone.h +2 -1
@@ -304,6 +304,8 @@ struct zone_reclaim_stat { };struct lruvec { +spinlock_t lru_lock; +struct list_head pages_lru[NR_LRU_LISTS ];unsigned long pages_count[NR_LRU_COUNTERS ];
@@ -386,7 +388,6 @@ struct zone {ZONE_PADDING (_pad1_)/* Fields commonly accessed by the page reclaim scanner */ - spinlock_t lru_lock; struct lruvec lruvec;unsigned long pages_scanned;/* since last reclaim */
mm/compaction.c +1 -1
@@ -306,7 +306,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, lruvec =NULL ; }if (need_resched () || - (lruvec &&spin_is_contended (&zone->lru_lock))) { + (lruvec &&spin_is_contended (&lruvec->lru_lock))) {if (lruvec)unlock_lruvec_irq (lruvec); lruvec =NULL ;
mm/internal.h +35 -31
128 lines not on this page · raw message
mm/page_alloc.c +1 -1
@@ -4294,6 +4294,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)enum lru_list lru;memset (lruvec,0 ,sizeof (struct lruvec )); +spin_lock_init (&lruvec->lru_lock);for_each_lru (lru)INIT_LIST_HEAD (&lruvec->pages_lru[lru]);#ifdef CONFIG_CGROUP_MEM_RES_CTLR
@@ -4369,7 +4370,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,#endif zone->name = zone_names[j];spin_lock_init (&zone->lock); -spin_lock_init (&zone->lru_lock);zone_seqlock_init (zone); zone->zone_pgdat = pgdat;
mm/swap.c +1 -1
@@ -700,7 +700,7 @@ void lru_add_page_tail(struct lruvec *lruvec,VM_BUG_ON (!PageHead (page));VM_BUG_ON (PageCompound (page_tail));VM_BUG_ON (PageLRU (page_tail)); -VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec_zone (lruvec)->lru_lock)); +VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec->lru_lock));SetPageLRU (page_tail);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 20/21] mm: split zone->lru_lock
Quoted 7 lines · On Thu, 23 Feb 2012 17:53:23 +0400 Konstantin Khlebnikov…
Looks like all ready for splitting
zone->lru_lockinto per-lruvec pieces.lruvec locking loop protected with rcu, actually there is irq-disabling instead of
rcu_read_lock(). Memory controller already releases its lru-vectors aftersyncronize_rcu()incgroup_diput(). Probably it should be replaced withsynchronize_sched()Signed-off-by Konstantin Khlebnikov <[email protected]>
Do we need rcu_read_lock() even if we check isolated pages at pre_destroy() ? If pre_destroy() ends, pages under a memcg being destroyed were moved to other cgroup while it's usolated.
So,
- PageLRU(page) guarantees lruvec is valid.
- if !PageLRU(page), the caller of lru_lock should know what it does. Once isolated, pre_destroy() never ends and page_lruvec(page) is always stable.
Thanks,
-Kame
Quoted 205 lines
--- include/linux/mmzone.h | 3 +- mm/compaction.c | 2 + mm/internal.h | 66 +++++++++++++++++++++++++----------------------- mm/page_alloc.c | 2 + mm/swap.c | 2 + 5 files changed, 40 insertions(+), 35 deletions(-)include/linux/mmzone.h +2
-1@@ -304,6 +304,8 @@ struct zone_reclaim_stat { };struct lruvec { +spinlock_t lru_lock; +struct list_head pages_lru[NR_LRU_LISTS ];unsigned long pages_count[NR_LRU_COUNTERS ];@@ -386,7 +388,6 @@ struct zone {ZONE_PADDING (_pad1_)/* Fields commonly accessed by the page reclaim scanner */ - spinlock_t lru_lock; struct lruvec lruvec;unsigned long pages_scanned;/* since last reclaim */ mm/compaction.c +1
-1@@ -306,7 +306,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, lruvec =NULL ; }if (need_resched () || - (lruvec &&spin_is_contended (&zone->lru_lock))) { + (lruvec &&spin_is_contended (&lruvec->lru_lock))) {if (lruvec)unlock_lruvec_irq (lruvec); lruvec =NULL ;mm/internal.h +35
-31@@ -15,27 +15,27 @@static inline void lock_lruvec (struct lruvec *lruvec,unsigned long *flags) { -spin_lock_irqsave (&lruvec_zone (lruvec)->lru_lock, *flags); +spin_lock_irqsave (&lruvec->lru_lock, *flags); }static inline void lock_lruvec_irq (struct lruvec *lruvec) { -spin_lock_irq (&lruvec_zone (lruvec)->lru_lock); +spin_lock_irq (&lruvec->lru_lock); }static inline void unlock_lruvec (struct lruvec *lruvec,unsigned long *flags) { -spin_unlock_irqrestore (&lruvec_zone (lruvec)->lru_lock, *flags); +spin_unlock_irqrestore (&lruvec->lru_lock, *flags); }static inline void unlock_lruvec_irq (struct lruvec *lruvec) { -spin_unlock_irq (&lruvec_zone (lruvec)->lru_lock); +spin_unlock_irq (&lruvec->lru_lock); }static inline void wait_lruvec_unlock (struct lruvec *lruvec) { -spin_unlock_wait (&lruvec_zone (lruvec)->lru_lock); +spin_unlock_wait (&lruvec->lru_lock); }#ifdef CONFIG_CGROUP_MEM_RES_CTLR @@ -46,37 +46,39 @@ static inline void wait_lruvec_unlock(struct lruvec *lruvec)static inline struct lruvec *__relock_page_lruvec (struct lruvec *locked_lruvec,struct page *page) { -/* Currenyly only one lru_lock per-zone */ -return page_lruvec (page); +struct lruvec *lruvec; + +do { + lruvec =page_lruvec (page); +if (likely (lruvec == locked_lruvec)) +return lruvec; +spin_unlock (&locked_lruvec->lru_lock); +spin_lock (&lruvec->lru_lock); + locked_lruvec = lruvec; + }while (1 ); }static inline struct lruvec *relock_page_lruvec_irq (struct lruvec *lruvec,struct page *page) { -struct zone *zone =page_zone (page); -if (!lruvec) { -spin_lock_irq (&zone->lru_lock); - }else if (zone !=lruvec_zone (lruvec)) { -unlock_lruvec_irq (lruvec); -spin_lock_irq (&zone->lru_lock); +local_irq_disable (); + lruvec =page_lruvec (page); +spin_lock (&lruvec->lru_lock); } -return page_lruvec (page); +return __relock_page_lruvec (lruvec, page); }static inline struct lruvec *relock_page_lruvec (struct lruvec *lruvec,struct page *page,unsigned long *flags) { -struct zone *zone =page_zone (page); -if (!lruvec) { -spin_lock_irqsave (&zone->lru_lock, *flags); - }else if (zone !=lruvec_zone (lruvec)) { -unlock_lruvec (lruvec, flags); -spin_lock_irqsave (&zone->lru_lock, *flags); +local_irq_save (*flags); + lruvec =page_lruvec (page); +spin_lock (&lruvec->lru_lock); } -return page_lruvec (page); +return __relock_page_lruvec (lruvec, page); }/* @@ -87,22 +89,24 @@ static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,static inline bool __lock_page_lruvec_irq (struct lruvec **lruvec,struct page *page) { -struct zone *zone;bool ret = false; +rcu_read_lock (); +/* +* If we see there PageLRU(), it means page has valid lruvec link. +* We need protect whole operation with single rcu-interval, otherwise +* lruvec which hold this LRU sign can run out before we secure it. +*/ if (PageLRU (page)) {if (!*lruvec) { - zone =page_zone (page); -spin_lock_irq (&zone->lru_lock); - }else - zone =lruvec_zone (*lruvec); - -if (PageLRU (page)) { *lruvec =page_lruvec (page); +lock_lruvec_irq (*lruvec); + } + *lruvec =__relock_page_lruvec (*lruvec, page); +if (PageLRU (page)) ret = true; - }else - *lruvec = &zone->lruvec; } +rcu_read_unlock ();return ret; }@@ -110,7 +114,7 @@ static inline bool __lock_page_lruvec_irq(struct lruvec **lruvec,/* Wait for lruvec unlock before locking other lruvec for the same page */ static inline void __wait_lruvec_unlock (struct lruvec *lruvec) { -/* Currently only one lru_lock per-zone */ +wait_lruvec_unlock (lruvec); }#else /* CONFIG_CGROUP_MEM_RES_CTLR */ mm/page_alloc.c +1
-1@@ -4294,6 +4294,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)enum lru_list lru;memset (lruvec,0 ,sizeof (struct lruvec )); +spin_lock_init (&lruvec->lru_lock);for_each_lru (lru)INIT_LIST_HEAD (&lruvec->pages_lru[lru]);#ifdef CONFIG_CGROUP_MEM_RES_CTLR @@ -4369,7 +4370,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,#endif zone->name = zone_names[j];spin_lock_init (&zone->lock); -spin_lock_init (&zone->lru_lock);zone_seqlock_init (zone); zone->zone_pgdat = pgdat;mm/swap.c +0
-0
mm/swap.c void lru_add_page_tail(struct lruvec *lruvec, · line 706
Quoted 8 lines
mm/swap.c +1
-1@@ -700,7 +700,7 @@ void lru_add_page_tail(struct lruvec *lruvec,VM_BUG_ON (!PageHead (page));VM_BUG_ON (PageCompound (page_tail));VM_BUG_ON (PageLRU (page_tail)); -VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec_zone (lruvec)->lru_lock)); +VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec->lru_lock));SetPageLRU (page_tail);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 20/21] mm: split zone->lru_lock
Quoted 11 lines · KAMEZAWA Hiroyuki
Quoted 7 lines · On Thu, 23 Feb 2012 17:53:23 +0400 Konstantin…
Looks like all ready for splitting
zone->lru_lockinto per-lruvec pieces.lruvec locking loop protected with rcu, actually there is irq-disabling instead of
rcu_read_lock(). Memory controller already releases its lru-vectors aftersyncronize_rcu()incgroup_diput(). Probably it should be replaced withsynchronize_sched()Signed-off-by Konstantin Khlebnikov<[email protected]>
Quoted 8 lines
Do we need
rcu_read_lock()even if we check isolated pages atpre_destroy()? Ifpre_destroy()ends, pages under a memcg being destroyed were moved to other cgroup while it's usolated.So,
- PageLRU(page) guarantees lruvec is valid.
- if !PageLRU(page), the caller oflru_lockshould know what it does. Once isolated,pre_destroy()never ends andpage_lruvec(page)is always stable.
This will be racy, because lruvec can be changed and released between checking PageLRU() and page_lruvec() or between page_lruvec() and spin_lock(), so we need to protect whole operation with single locking interval.
Quoted 213 lines
Thanks,
-KameQuoted 209 lines
--- include/linux/mmzone.h | 3 +- mm/compaction.c | 2 + mm/internal.h | 66 +++++++++++++++++++++++++----------------------- mm/page_alloc.c | 2 + mm/swap.c | 2 + 5 files changed, 40 insertions(+), 35 deletions(-)include/linux/mmzone.h +2
-1@@ -304,6 +304,8 @@ struct zone_reclaim_stat { };struct lruvec { +spinlock_t lru_lock; +struct list_head pages_lru[NR_LRU_LISTS ];unsigned long pages_count[NR_LRU_COUNTERS ];@@ -386,7 +388,6 @@ struct zone {ZONE_PADDING (_pad1_)/* Fields commonly accessed by the page reclaim scanner */ - spinlock_t lru_lock; struct lruvec lruvec;unsigned long pages_scanned;/* since last reclaim */ mm/compaction.c +1
-1@@ -306,7 +306,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone, lruvec =NULL ; }if (need_resched () || - (lruvec&&spin_is_contended (&zone->lru_lock))) { + (lruvec&&spin_is_contended (&lruvec->lru_lock))) {if (lruvec)unlock_lruvec_irq (lruvec); lruvec =NULL ;mm/internal.h +35
-31@@ -15,27 +15,27 @@static inline void lock_lruvec (struct lruvec *lruvec,unsigned long *flags) { -spin_lock_irqsave (&lruvec_zone (lruvec)->lru_lock, *flags); +spin_lock_irqsave (&lruvec->lru_lock, *flags); }static inline void lock_lruvec_irq (struct lruvec *lruvec) { -spin_lock_irq (&lruvec_zone (lruvec)->lru_lock); +spin_lock_irq (&lruvec->lru_lock); }static inline void unlock_lruvec (struct lruvec *lruvec,unsigned long *flags) { -spin_unlock_irqrestore (&lruvec_zone (lruvec)->lru_lock, *flags); +spin_unlock_irqrestore (&lruvec->lru_lock, *flags); }static inline void unlock_lruvec_irq (struct lruvec *lruvec) { -spin_unlock_irq (&lruvec_zone (lruvec)->lru_lock); +spin_unlock_irq (&lruvec->lru_lock); }static inline void wait_lruvec_unlock (struct lruvec *lruvec) { -spin_unlock_wait (&lruvec_zone (lruvec)->lru_lock); +spin_unlock_wait (&lruvec->lru_lock); }#ifdef CONFIG_CGROUP_MEM_RES_CTLR @@ -46,37 +46,39 @@ static inline void wait_lruvec_unlock(struct lruvec *lruvec)static inline struct lruvec *__relock_page_lruvec (struct lruvec *locked_lruvec,struct page *page) { -/* Currenyly only one lru_lock per-zone */ -return page_lruvec (page); +struct lruvec *lruvec; + +do { + lruvec =page_lruvec (page); +if (likely (lruvec == locked_lruvec)) +return lruvec; +spin_unlock (&locked_lruvec->lru_lock); +spin_lock (&lruvec->lru_lock); + locked_lruvec = lruvec; + }while (1 ); }static inline struct lruvec *relock_page_lruvec_irq (struct lruvec *lruvec,struct page *page) { -struct zone *zone =page_zone (page); -if (!lruvec) { -spin_lock_irq (&zone->lru_lock); - }else if (zone !=lruvec_zone (lruvec)) { -unlock_lruvec_irq (lruvec); -spin_lock_irq (&zone->lru_lock); +local_irq_disable (); + lruvec =page_lruvec (page); +spin_lock (&lruvec->lru_lock); } -return page_lruvec (page); +return __relock_page_lruvec (lruvec, page); }static inline struct lruvec *relock_page_lruvec (struct lruvec *lruvec,struct page *page,unsigned long *flags) { -struct zone *zone =page_zone (page); -if (!lruvec) { -spin_lock_irqsave (&zone->lru_lock, *flags); - }else if (zone !=lruvec_zone (lruvec)) { -unlock_lruvec (lruvec, flags); -spin_lock_irqsave (&zone->lru_lock, *flags); +local_irq_save (*flags); + lruvec =page_lruvec (page); +spin_lock (&lruvec->lru_lock); } -return page_lruvec (page); +return __relock_page_lruvec (lruvec, page); }/* @@ -87,22 +89,24 @@ static inline struct lruvec *relock_page_lruvec(struct lruvec *lruvec,static inline bool __lock_page_lruvec_irq (struct lruvec **lruvec,struct page *page) { -struct zone *zone;bool ret = false; +rcu_read_lock (); +/* +* If we see there PageLRU(), it means page has valid lruvec link. +* We need protect whole operation with single rcu-interval, otherwise +* lruvec which hold this LRU sign can run out before we secure it. +*/ if (PageLRU (page)) {if (!*lruvec) { - zone =page_zone (page); -spin_lock_irq (&zone->lru_lock); - }else - zone =lruvec_zone (*lruvec); - -if (PageLRU (page)) { *lruvec =page_lruvec (page); +lock_lruvec_irq (*lruvec); + } + *lruvec =__relock_page_lruvec (*lruvec, page); +if (PageLRU (page)) ret = true; - }else - *lruvec =&zone->lruvec; } +rcu_read_unlock ();return ret; }@@ -110,7 +114,7 @@ static inline bool __lock_page_lruvec_irq(struct lruvec **lruvec,/* Wait for lruvec unlock before locking other lruvec for the same page */ static inline void __wait_lruvec_unlock (struct lruvec *lruvec) { -/* Currently only one lru_lock per-zone */ +wait_lruvec_unlock (lruvec); }#else /* CONFIG_CGROUP_MEM_RES_CTLR */ mm/page_alloc.c +1
-1@@ -4294,6 +4294,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)enum lru_list lru;memset (lruvec,0 ,sizeof (struct lruvec )); +spin_lock_init (&lruvec->lru_lock);for_each_lru (lru)INIT_LIST_HEAD (&lruvec->pages_lru[lru]);#ifdef CONFIG_CGROUP_MEM_RES_CTLR @@ -4369,7 +4370,6 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,#endif zone->name = zone_names[j];spin_lock_init (&zone->lock); -spin_lock_init (&zone->lru_lock);zone_seqlock_init (zone); zone->zone_pgdat = pgdat;mm/swap.c +1
-1@@ -700,7 +700,7 @@ void lru_add_page_tail(struct lruvec *lruvec,VM_BUG_ON (!PageHead (page));VM_BUG_ON (PageCompound (page_tail));VM_BUG_ON (PageLRU (page_tail)); -VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec_zone (lruvec)->lru_lock)); +VM_BUG_ON (NR_CPUS !=1 && !spin_is_locked (&lruvec->lru_lock));
Quoted 5 lines
mm/swap.c void lru_add_page_tail(struct lruvec *lruvec, · line 706
Quoted 4 lines
mm/swap.c +0
-0SetPageLRU (page_tail);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH v3 21/21 mm: zone lru vectors interleaving
In reply to Konstantin KhlebnikovSplit zones into several lru vectors with pfn-based interleaving. Thus we can redeuce lru_lock contention without using cgroups.
By default there 4 lru with 16Mb interleaving.
--- include/linux/huge_mm.h | 3 ++ include/linux/memcontrol.h | 2 +- include/linux/mm.h | 45 +++++++++++++++++++++++++++++------ include/linux/mmzone.h | 4 ++- mm/Kconfig | 16 +++++++++++++ mm/internal.h | 19 ++++++++++++++- mm/memcontrol.c | 56 ++++++++++++++++++++++++-------------------- mm/page_alloc.c | 7 +++--- mm/vmscan.c | 18 ++++++++++---- 9 files changed, 124 insertions(+), 46 deletions(-)
include/linux/huge_mm.h +3 -0
@@ -107,6 +107,9 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);#if HPAGE_PMD_ORDER >MAX_ORDER #error "hugepages can't be allocated by the buddy allocator"#endif +#if HPAGE_PMD_ORDER >CONFIG_PAGE_LRU_INTERLEAVING +#error "zone lru interleaving order lower than huge page order" +#endif extern int hugepage_madvise (struct vm_area_struct *vma,unsigned long *vm_flags,int advice); extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
include/linux/memcontrol.h +1 -1
@@ -199,7 +199,7 @@ static inline void mem_cgroup_uncharge_cache_page(struct page *page)static inline struct lruvec *mem_cgroup_zone_lruvec (struct zone *zone,struct mem_cgroup *memcg) { -return &zone->lruvec; +return zone->lruvec; }static inline struct mem_cgroup *try_get_mem_cgroup_from_page (struct page *page)
include/linux/mm.h +37 -8
71 lines not on this page · raw message
include/linux/mmzone.h +2 -2
@@ -311,7 +311,7 @@ struct lruvec {struct zone_reclaim_stat reclaim_stat; -#ifdef CONFIG_CGROUP_MEM_RES_CTLR +#if defined(CONFIG_CGROUP_MEM_RES_CTLR ) || (CONFIG_PAGE_LRU_SPLIT !=1 )struct zone *zone;struct pglist_data *node;#endif
@@ -388,7 +388,7 @@ struct zone {ZONE_PADDING (_pad1_)/* Fields commonly accessed by the page reclaim scanner */ - struct lruvec lruvec; + struct lruvec lruvec[CONFIG_PAGE_LRU_SPLIT];unsigned long pages_scanned;/* since last reclaim */ unsigned long flags;/* zone flags, see below */
mm/Kconfig +16 -0
@@ -183,6 +183,22 @@ config SPLIT_PTLOCK_CPUS default "999999" if DEBUG_SPINLOCK || DEBUG_LOCK_ALLOC default "4" +config PAGE_LRU_SPLIT + int "Memory lru lists per zone" + default 4 if EXPERIMENTAL && SPARSEMEM_VMEMMAP + default 1 + help + The number of lru lists in each memory zone for interleaving. + Allows to redeuce lru_lock contention, but adds some overhead. + Without SPARSEMEM_VMEMMAP might be costly. "1" means no split. + +config PAGE_LRU_INTERLEAVING + int "Memory lru lists interleaving page-order" + default 12 + help + Page order for lru lists interleaving. By default 12 (16Mb). + Must be greater than huge-page order. + With CONFIG_PAGE_LRU_SPLIT=1 has no effect. # # support for memory compaction config COMPACTION
mm/internal.h +18 -1
@@ -13,6 +13,15 @@#include <linux/mm.h> +#define for_each_zone_id (zone_id) \ + for ( zone_id = 0 ; zone_id < MAX_NR_ZONES ; zone_id++ ) + +#define for_each_lruvec_id (lruvec_id) \ + for ( lruvec_id = 0 ; lruvec_id < CONFIG_PAGE_LRU_SPLIT ; lruvec_id++ ) + +#define for_each_zone_and_lruvec_id (zone_id, lruvec_id) \ + for_each_zone_id(zone_id) for_each_lruvec_id(lruvec_id) +static inline void lock_lruvec (struct lruvec *lruvec,unsigned long *flags) {spin_lock_irqsave (&lruvec->lru_lock, *flags);
@@ -125,7 +134,15 @@ static inline void __wait_lruvec_unlock(struct lruvec *lruvec)static inline struct lruvec *__relock_page_lruvec (struct lruvec *locked_lruvec,struct page *page) { -/* Currently ony one lruvec per-zone */ +#if CONFIG_PAGE_LRU_SPLIT !=1 +struct lruvec *lruvec =page_lruvec (page); + +if (unlikely (lruvec != locked_lruvec)) { +spin_unlock (&locked_lruvec->lru_lock); +spin_lock (&lruvec->lru_lock); + locked_lruvec = lruvec; + } +#endif return locked_lruvec; }
mm/memcontrol.c +30 -26
163 lines not on this page · raw message
mm/page_alloc.c +4 -3
@@ -4297,7 +4297,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)spin_lock_init (&lruvec->lru_lock);for_each_lru (lru)INIT_LIST_HEAD (&lruvec->pages_lru[lru]); -#ifdef CONFIG_CGROUP_MEM_RES_CTLR +#if defined(CONFIG_CGROUP_MEM_RES_CTLR ) || (CONFIG_PAGE_LRU_SPLIT !=1 ) lruvec->node = zone->zone_pgdat; lruvec->zone = zone;#endif
@@ -4312,7 +4312,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec) static void __paginginit free_area_init_core(struct pglist_data *pgdat,unsigned long *zones_size,unsigned long *zholes_size) { -enum zone_type j; +enum zone_type j, lruvec_id;int nid = pgdat->node_id;unsigned long zone_start_pfn = pgdat->node_start_pfn;int ret;
@@ -4374,7 +4374,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat, zone->zone_pgdat = pgdat;zone_pcp_init (zone); -init_zone_lruvec (zone, &zone->lruvec); +for_each_lruvec_id (lruvec_id) +init_zone_lruvec (zone, &zone->lruvec[lruvec_id]);zap_zone_vm_stats (zone); zone->flags =0 ;if (!size)
mm/vmscan.c +13 -5
@@ -2202,12 +2202,14 @@ static void shrink_zone(int priority, struct zone *zone, };struct mem_cgroup *memcg;struct lruvec *lruvec; +int lruvec_id; memcg =mem_cgroup_iter (root,NULL , &reclaim); do { lruvec = mem_cgroup_zone_lruvec(zone, memcg); - shrink_lruvec(priority, lruvec, sc); + for_each_lruvec_id(lruvec_id) + shrink_lruvec(priority, lruvec + lruvec_id, sc); /* * Limit reclaim has historically picked one memcg and
@@ -2529,6 +2531,7 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg, .target_mem_cgroup = memcg, };struct lruvec *lruvec =mem_cgroup_zone_lruvec (zone, memcg); +int lruvec_id; sc.gfp_mask = (gfp_mask &GFP_RECLAIM_MASK ) | (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK );
@@ -2544,7 +2547,8 @@ unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,* will pick up pages from other mem cgroup's as well. We hack * the priority and make it zero. */ -shrink_lruvec (0 , lruvec, &sc); +for_each_lruvec_id (lruvec_id) +shrink_lruvec (0 , lruvec + lruvec_id, &sc);trace_mm_vmscan_memcg_softlimit_reclaim_end (sc.nr_reclaimed);
@@ -2599,6 +2603,7 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc, int priority) {struct mem_cgroup *memcg; +int lruvec_id;if (!total_swap_pages)return ;
@@ -2607,9 +2612,12 @@ static void age_active_anon(struct zone *zone, struct scan_control *sc,do {struct lruvec *lruvec =mem_cgroup_zone_lruvec (zone, memcg); -if (inactive_anon_is_low (lruvec)) -shrink_active_list (SWAP_CLUSTER_MAX , lruvec, - sc, priority,0 ); +for_each_lruvec_id (lruvec_id) { +if (inactive_anon_is_low (lruvec + lruvec_id)) +shrink_active_list (SWAP_CLUSTER_MAX , + lruvec + lruvec_id, + sc, priority,0 ); + } memcg =mem_cgroup_iter (NULL , memcg,NULL ); }while (memcg);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 21/21] mm: zone lru vectors interleaving
mm/page_alloc.c void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec) · line 4315
Quoted 6 lines · Konstantin Khlebnikov
mm/page_alloc.c +1
-1@@ -4312,7 +4312,7 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec) static void __paginginit free_area_init_core(struct pglist_data *pgdat, unsigned long *zones_size, unsigned long *zholes_size) { - enum zone_type j; + enum zone_type j, lruvec_id;
Like other cases in the patch,
int lruvec_id;
looks clearer
mm/page_alloc.c static void __paginginit free_area_init_core(struct pglist_data *pgdat, · line 4381
Quoted 13 lines
mm/page_alloc.c +2
-1int nid = pgdat->node_id; unsigned long zone_start_pfn = pgdat->node_start_pfn; int ret;@@ -4374,7 +4374,8 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat, zone->zone_pgdat = pgdat; zone_pcp_init(zone); - init_zone_lruvec(zone, &zone->lruvec); + for_each_lruvec_id(lruvec_id) + init_zone_lruvec(zone, &zone->lruvec[lruvec_id]); zap_zone_vm_stats(zone); zone->flags = 0; if (!size)
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Re: [PATCH v3 21/21] mm: zone lru vectors interleaving
In reply to Konstantin Khlebnikovmm/Kconfig config SPLIT_PTLOCK_CPUS · line 193
Quoted 8 lines
mm/Kconfig +8
-0+config PAGE_LRU_SPLIT + int "Memory lru lists per zone" + default 4 if EXPERIMENTAL && SPARSEMEM_VMEMMAP + default 1 + help + The number of lru lists in each memory zone for interleaving. + Allows to redeuce lru_lock contention, but adds some overhead. + Without SPARSEMEM_VMEMMAP might be costly. "1" means no split.
Could you turn those two numbers into a boot option? Compile time parameters are nasty to use.
I suppose it's ok to have an upper limit.
mm/Kconfig config SPLIT_PTLOCK_CPUS · line 201
Quoted 8 lines
mm/Kconfig +8
-0+ +config PAGE_LRU_INTERLEAVING + int "Memory lru lists interleaving page-order" + default 12 + help + Page order for lru lists interleaving. By default 12 (16Mb). + Must be greater than huge-page order. + With CONFIG_PAGE_LRU_SPLIT=1 has no effect.
-Andi
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 1/2 mm: configure lruvec split by boot options
This patch adds boot options:
lruvec_split=%u by default 1, limited by CONFIG_PAGE_LRU_SPLIT lruvec_interleaving=%u by default CONFIG_PAGE_LRU_INTERLEAVING
--- include/linux/mm.h | 5 ++++- mm/internal.h | 2 +- mm/page_alloc.c | 29 +++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-)
include/linux/mm.h +4 -1
@@ -737,12 +737,15 @@ static inline int page_lruvec_id(struct page *page)#else /* CONFIG_PAGE_LRU_SPLIT */ +extern unsigned lruvec_split; +extern unsigned lruvec_interleaving; +static inline int page_lruvec_id (struct page *page) {unsigned long pfn =page_to_pfn (page); -return (pfn >>CONFIG_PAGE_LRU_INTERLEAVING ) %CONFIG_PAGE_LRU_SPLIT ; +return (pfn >> lruvec_interleaving) % lruvec_split; }#endif /* CONFIG_PAGE_LRU_SPLIT */
mm/internal.h +1 -1
@@ -17,7 +17,7 @@ for ( zone_id = 0 ; zone_id < MAX_NR_ZONES ; zone_id++ )#define for_each_lruvec_id (lruvec_id) \ - for ( lruvec_id = 0 ; lruvec_id < CONFIG_PAGE_LRU_SPLIT ; lruvec_id++ ) + for ( lruvec_id = 0 ; lruvec_id < lruvec_split ; lruvec_id++ )#define for_each_zone_and_lruvec_id (zone_id, lruvec_id) \ for_each_zone_id(zone_id) for_each_lruvec_id(lruvec_id)
mm/page_alloc.c +29 -0
@@ -4303,6 +4303,35 @@ void init_zone_lruvec(struct zone *zone, struct lruvec *lruvec)#endif } +#if CONFIG_PAGE_LRU_SPLIT !=1 + +unsigned lruvec_split =1 ; +unsigned lruvec_interleaving =CONFIG_PAGE_LRU_INTERLEAVING ; + +static int __init set_lruvec_split (char *arg) +{ +if (!kstrtouint (arg,0 , &lruvec_split) && + lruvec_split >=1 && + lruvec_split <=CONFIG_PAGE_LRU_SPLIT ) +return 0 ; + lruvec_split =1 ; +return 1 ; +} +early_param ("lruvec_split" , set_lruvec_split); + +static int __init set_lruvec_interleaving (char *arg) +{ +if (!kstrtouint (arg,0 , &lruvec_interleaving) && + lruvec_interleaving >=HPAGE_PMD_ORDER && + lruvec_interleaving <=BITS_PER_LONG ) +return 0 ; + lruvec_split =1 ; +return 1 ; +} +early_param ("lruvec_interleaving" , set_lruvec_interleaving); + +#endif +/* * Set up the zone data structures: * - mark all pages reserved
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
PATCH 2/2 mm: show zone lruvec state in /proc/zoneinfo
In reply to Andi Kleen--- mm/vmstat.c | 23 +++++++++++++++++++++++ 1 files changed, 23 insertions(+), 0 deletions(-)
mm/vmstat.c +23 -0
@@ -20,6 +20,8 @@#include <linux/writeback.h> #include <linux/compaction.h> +#include "internal.h" +#ifdef CONFIG_VM_EVENT_COUNTERS DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};EXPORT_PER_CPU_SYMBOL (vm_event_states);
@@ -1020,6 +1022,27 @@ static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,"\n start_pfn: %lu" , zone->all_unreclaimable, zone->zone_start_pfn); +seq_printf (m,"\n lruvecs" ); +for_each_lruvec_id (i) { +struct lruvec *lruvec = zone->lruvec + i; +enum lru_list lru; + +seq_printf (m, +"\n lruvec: %i" , + i); +for_each_lru (lru) +seq_printf (m, +"\n %s: %lu" , + vmstat_text[NR_LRU_BASE + lru], + lruvec->pages_count[lru]); +seq_printf (m, +"\n %s: %lu" +"\n %s: %lu" , + vmstat_text[NR_ISOLATED_ANON ], + lruvec->pages_count[LRU_ISOLATED_ANON ], + vmstat_text[NR_ISOLATED_FILE ], + lruvec->pages_count[LRU_ISOLATED_FILE ]); + }seq_putc (m,'\n' ); }
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 8 lines · Konstantin Khlebnikov
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving
Quoted 5 lines
This patchset is based on next-20120210
git: https://github.com/koct9i/linux/commits/lruvec-v3
---
I am seeing an improvement of about 7% in throughput in a workload where I am doing parallel reading of files that are mmaped. The contention on lru_lock used to be 13% in the cpu profile on the __pagevec_lru_add code path. Now lock contention on this path drops to about 0.6%. I have 40 hyper-threaded enabled cpu cores running 80 mmaped file reading processes.
So initial testing of this patch set looks encouraging.
Tim
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 15 lines · Tim Chen
Quoted 8 lines · Konstantin Khlebnikov
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleavingQuoted 5 lines
This patchset is based on next-20120210
git: https://github.com/koct9i/linux/commits/lruvec-v3
---
Quoted 8 lines
I am seeing an improvement of about 7% in throughput in a workload where I am doing parallel reading of files that are mmaped. The contention on
lru_lockused to be 13% in the cpu profile on the__pagevec_lru_addcode path. Now lock contention on this path drops to about 0.6%. I have 40 hyper-threaded enabled cpu cores running 80 mmaped file reading processes.So initial testing of this patch set looks encouraging.
That's great!
Quoted 3 lines
Tim
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin Khlebnikov…
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving
Quoted 4 lines
This patchset is based on next-20120210
I wonder.... I just wonder...if we can split a lruvec in a zone into small pieces of lruvec and have splitted LRU-lock per them, do we need per-memcg-lrulock ?
It seems per-memcg-lrulock can be much bigger lock than small-lruvec-lock. (depends on configuraton) and much more complicated..and have to take care of many things.. If unit of splitting can be specified by boot option, it seems admins can split a big memcg's per-memcg-lru lock into more small pieces.
BTW, how to think of default size of splitting ? I wonder splitting lru into the number of cpus per a node can be a choice. Each cpu may have a chance to set prefered-pfn-range at page allocation with additional patches.
Thanks,
-Kame
Quoted 37 lines
---
Konstantin Khlebnikov (21):
memcg: unifyinactive_ratiocalculation
memcg: makemm_match_cgroup()hirarchical
memcg: fixpage_referenciescgroup filter on global reclaim
memcg: usevm_swappinessfrom target memory cgroup
mm: renamelruvec->listsintolruvec->pages_lru
mm: lruvec linking functions
mm: addlruvec->pages_count
mm: unifyinactive_list_is_low()
mm: addlruvec->reclaim_stat
mm: kill structmem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec intoupdate_page_reclaim_stat()
mm: push lruvecs frompagevec_lru_move_fn()to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: splitzone->lru_lock
mm: zone lru vectors interleavinginclude/linux/huge_mm.h | 3 include/linux/memcontrol.h | 75 ------ include/linux/mm.h | 66 +++++ include/linux/mm_inline.h | 19 +- include/linux/mmzone.h | 39 ++- include/linux/swap.h | 6 mm/Kconfig | 16 + mm/compaction.c | 31 +-- mm/huge_memory.c | 14 + mm/internal.h | 204 +++++++++++++++++ mm/ksm.c | 2
Quoted 12 lines
mm/memcontrol.c | 343 +++++++++++----------------- mm/migrate.c | 2 mm/page_alloc.c | 70 +----- mm/rmap.c | 2 mm/swap.c | 217 ++++++++++-------- mm/vmscan.c | 534 ++++++++++++++++++++++++-------------------- mm/vmstat.c | 6 18 files changed, 932 insertions(+), 717 deletions(-)Signature · 1 line
Signature
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 11 lines · KAMEZAWA Hiroyuki
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin…
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving
Quoted 7 lines
Quoted 4 lines
This patchset is based on next-20120210
I wonder.... I just wonder...if we can split a lruvec in a zone into small pieces of lruvec and have splitted LRU-lock per them, do we need per-memcg-lrulock ?
What per-memcg-lrulock? I don't have it.
last patch splits lruvecs in memcg with the same factor.
Quoted 5 lines
It seems per-memcg-lrulock can be much bigger lock than small-lruvec-lock. (depends on configuraton) and much more complicated..and have to take care of many things.. If unit of splitting can be specified by boot option, it seems admins can split a big memcg's per-memcg-lru lock into more small pieces.
lruvec count per memcg can be arbitrary and changeable if cgroup is empty. This is not in this patch, but it's really easy.
Quoted 4 lines
BTW, how to think of default size of splitting ? I wonder splitting lru into the number of cpus per a node can be a choice. Each cpu may have a chance to set prefered-pfn-range at page allocation with additional patches.
If we rework page to memcg linking and add direct lruvec-id into page->flags, we will able to change lruvec before inserting page to lru.
Thus each cpu will always insert pages into its own lruvec in zone. I have not thought about races yet, but this would be perfect solution.
Quoted 55 lines
Thanks,
-KameQuoted 49 lines
---
Konstantin Khlebnikov (21):
memcg: unifyinactive_ratiocalculation
memcg: makemm_match_cgroup()hirarchical
memcg: fixpage_referenciescgroup filter on global reclaim
memcg: usevm_swappinessfrom target memory cgroup
mm: renamelruvec->listsintolruvec->pages_lru
mm: lruvec linking functions
mm: addlruvec->pages_count
mm: unifyinactive_list_is_low()
mm: addlruvec->reclaim_stat
mm: kill structmem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec intoupdate_page_reclaim_stat()
mm: push lruvecs frompagevec_lru_move_fn()to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: splitzone->lru_lock
mm: zone lru vectors interleavinginclude/linux/huge_mm.h | 3 include/linux/memcontrol.h | 75 ------ include/linux/mm.h | 66 +++++ include/linux/mm_inline.h | 19 +- include/linux/mmzone.h | 39 ++- include/linux/swap.h | 6 mm/Kconfig | 16 + mm/compaction.c | 31 +-- mm/huge_memory.c | 14 + mm/internal.h | 204 +++++++++++++++++ mm/ksm.c | 2 mm/memcontrol.c | 343 +++++++++++----------------- mm/migrate.c | 2 mm/page_alloc.c | 70 +----- mm/rmap.c | 2 mm/swap.c | 217 ++++++++++-------- mm/vmscan.c | 534 ++++++++++++++++++++++++-------------------- mm/vmstat.c | 6 18 files changed, 932 insertions(+), 717 deletions(-)--
Signature
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 14 lines · On Sat, 25 Feb 2012 09:31:01 +0400 Konstantin Khlebnikov…
Quoted 13 lines · KAMEZAWA Hiroyuki
Quoted 10 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin…
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleavingThis patchset is based on next-20120210
Quoted 9 lines
Quoted 5 lines
Quoted 2 lines
I wonder.... I just wonder...if we can split a lruvec in a zone into small pieces of lruvec and have splitted LRU-lock per them, do we need per-memcg-lrulock ?
What per-memcg-lrulock? I don't have it.
last patch splits lruvecs in memcg with the same factor.
Okay, I missed it.
Thanks,
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin Khlebnikov…
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleaving
Quoted 4 lines
This patchset is based on next-20120210
I'm sorry I can't have enough review time in these days but the whole series seems good to me.
BTW, how about trying to merge patch 1/21 -> 13or14/21 first ?
This series adds many changes to various place under /mm. So, step-by-step merging will be better I think.
(Just says as this because I tend to split a long series of patch to
small sets of patches and merge them one by one to reduce my own maintainance cost.)
At final lock splitting, performance number should be in changelog.
Thanks,
-Kame
Quoted 37 lines
---
Konstantin Khlebnikov (21):
memcg: unifyinactive_ratiocalculation
memcg: makemm_match_cgroup()hirarchical
memcg: fixpage_referenciescgroup filter on global reclaim
memcg: usevm_swappinessfrom target memory cgroup
mm: renamelruvec->listsintolruvec->pages_lru
mm: lruvec linking functions
mm: addlruvec->pages_count
mm: unifyinactive_list_is_low()
mm: addlruvec->reclaim_stat
mm: kill structmem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec intoupdate_page_reclaim_stat()
mm: push lruvecs frompagevec_lru_move_fn()to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: splitzone->lru_lock
mm: zone lru vectors interleavinginclude/linux/huge_mm.h | 3 include/linux/memcontrol.h | 75 ------ include/linux/mm.h | 66 +++++ include/linux/mm_inline.h | 19 +- include/linux/mmzone.h | 39 ++- include/linux/swap.h | 6 mm/Kconfig | 16 + mm/compaction.c | 31 +-- mm/huge_memory.c | 14 + mm/internal.h | 204 +++++++++++++++++ mm/ksm.c | 2
Quoted 19 lines
mm/memcontrol.c | 343 +++++++++++----------------- mm/migrate.c | 2 mm/page_alloc.c | 70 +----- mm/rmap.c | 2 mm/swap.c | 217 ++++++++++-------- mm/vmscan.c | 534 ++++++++++++++++++++++++-------------------- mm/vmstat.c | 6 18 files changed, 932 insertions(+), 717 deletions(-)Signature · 8 lines
Signature -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/ Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>
Quoted 19 lines · KAMEZAWA Hiroyuki
Quoted 8 lines · On Thu, 23 Feb 2012 17:51:36 +0400 Konstantin…
v3 changes:
* inactive-ratio reworked again, now it always calculated from from scratch
* hierarchical pte reference bits filter in memory-cgroup reclaimer
* fixed two bugs in locking, found by Hugh Dickins
* locking functions slightly simplified
* new patch for isolated pages accounting
* new patch with lru interleavingQuoted 4 lines
This patchset is based on next-20120210
I'm sorry I can't have enough review time in these days but the whole series seems good to me.
Quoted 5 lines
BTW, how about trying to merge patch 1/21 -> 13or14/21 first ?
This series adds many changes to various place under /mm. So, step-by-step merging will be better I think.
(Just says as this because I tend to split a long series of patch to
small sets of patches and merge them one by one to reduce my own maintainance cost.)
I agree, the amount of required cleanups has exceeded the limit. I'll send them as separate patchset, maybe tomorrow.
Quoted 2 lines
At final lock splitting, performance number should be in changelog.
For sure
Quoted 61 lines
Thanks,
-KameQuoted 56 lines
---
Konstantin Khlebnikov (21):
memcg: unifyinactive_ratiocalculation
memcg: makemm_match_cgroup()hirarchical
memcg: fixpage_referenciescgroup filter on global reclaim
memcg: usevm_swappinessfrom target memory cgroup
mm: renamelruvec->listsintolruvec->pages_lru
mm: lruvec linking functions
mm: addlruvec->pages_count
mm: unifyinactive_list_is_low()
mm: addlruvec->reclaim_stat
mm: kill structmem_cgroup_zone
mm: move page-to-lruvec translation upper
mm: push lruvec intoupdate_page_reclaim_stat()
mm: push lruvecs frompagevec_lru_move_fn()to iterator
mm: introduce lruvec locking primitives
mm: handle lruvec relocks on lumpy reclaim
mm: handle lruvec relocks in compaction
mm: handle lruvec relock in memory controller
mm: add to lruvec isolated pages counters
memcg: check lru vectors emptiness in pre-destroy
mm: splitzone->lru_lock
mm: zone lru vectors interleavinginclude/linux/huge_mm.h | 3 include/linux/memcontrol.h | 75 ------ include/linux/mm.h | 66 +++++ include/linux/mm_inline.h | 19 +- include/linux/mmzone.h | 39 ++- include/linux/swap.h | 6 mm/Kconfig | 16 + mm/compaction.c | 31 +-- mm/huge_memory.c | 14 + mm/internal.h | 204 +++++++++++++++++ mm/ksm.c | 2 mm/memcontrol.c | 343 +++++++++++----------------- mm/migrate.c | 2 mm/page_alloc.c | 70 +----- mm/rmap.c | 2 mm/swap.c | 217 ++++++++++-------- mm/vmscan.c | 534 ++++++++++++++++++++++++-------------------- mm/vmstat.c | 6 18 files changed, 932 insertions(+), 717 deletions(-)--
Signature--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email:<a href=mailto:"[email protected]"> [email protected]</a>
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>