Memory controller soft limit patches (v4)

---

 include/linux/memcontrol.h |    9 ++
 include/linux/swap.h       |    5 +
 mm/memcontrol.c            |  223 +++++++++++++++++++++++++++++++++++++++++---
 mm/vmscan.c                |   26 +++++
 4 files changed, 245 insertions(+), 18 deletions(-)

include/linux/memcontrol.h +9 -0

index 18146c9..16343d0 100644
@@ -116,6 +116,9 @@ static inline bool mem_cgroup_disabled(void)
 }
 
 extern bool mem_cgroup_oom_called(struct task_struct *task);
+unsigned long
+mem_cgroup_soft_limit_reclaim(int priority, struct zone *zone, int nid,
+				gfp_t gfp_mask);
 
 #else /* CONFIG_CGROUP_MEM_RES_CTLR */
 struct mem_cgroup;

@@ -264,6 +267,12 @@ mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
 {
 }
 
+static inline unsigned long
+mem_cgroup_soft_limit_reclaim(int priority, struct zone *zone, int nid,
+				gfp_t gfp_mask)
+{
+	return 0;
+}
 #endif /* CONFIG_CGROUP_MEM_CONT */
 
 #endif /* _LINUX_MEMCONTROL_H */

include/linux/swap.h +5 -0

index 989eb53..37bc2a9 100644
@@ -217,6 +217,11 @@ extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem,
 						  gfp_t gfp_mask, bool noswap,
 						  unsigned int swappiness);
+extern unsigned long mem_cgroup_shrink_zone(struct mem_cgroup *mem,
+						struct zone *zone,
+						gfp_t gfp_mask,
+						unsigned int swappiness,
+						int priority);
 extern int __isolate_lru_page(struct page *page, int mode, int file);
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
 extern int vm_swappiness;

mm/memcontrol.c +134 -12

index d548dd2..3be1f27 100644
@@ -20,6 +20,7 @@
 #include <linux/res_counter.h>
 #include <linux/memcontrol.h>
 #include <linux/cgroup.h>
+#include <linux/completion.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/smp.h>

@@ -191,6 +192,14 @@ struct mem_cgroup {
 	unsigned long last_tree_update;		/* Last time the tree was */
 						/* updated in jiffies     */
 
+	bool on_tree;				/* Is the node on tree? */
+	struct completion wait_on_soft_reclaim;
+	/*
+	 * Set to > 0, when reclaim is initiated due to 
+	 * the soft limit being exceeded. It adds an additional atomic
+	 * operation to page fault path.
+	 */
+	int soft_limit_reclaim_count;
 	/*
 	 * statistics. This must be placed at the end of memcg.
 	 */

@@ -227,18 +236,29 @@ pcg_default_flags[NR_CHARGE_TYPE] = {
 #define MEMFILE_TYPE(val)	(((val) >> 16) & 0xffff)
 #define MEMFILE_ATTR(val)	((val) & 0xffff)
 
+/*
+ * Bits used for hierarchical reclaim bits
+ */
+#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
+#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
+#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
+#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
+#define MEM_CGROUP_RECLAIM_SOFT_BIT	0x2
+#define MEM_CGROUP_RECLAIM_SOFT		(1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
+
 static void mem_cgroup_get(struct mem_cgroup *mem);
 static void mem_cgroup_put(struct mem_cgroup *mem);
 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
 
-static void mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
+static void __mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
 {
 	struct rb_node **p = &mem_cgroup_soft_limit_tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct mem_cgroup *mem_node;
-	unsigned long flags;
 
-	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+	if (mem->on_tree)
+		return;
+
 	while (*p) {
 		parent = *p;
 		mem_node = rb_entry(parent, struct mem_cgroup, mem_cgroup_node);

@@ -255,6 +275,23 @@ static void mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
 	rb_insert_color(&mem->mem_cgroup_node,
 			&mem_cgroup_soft_limit_tree);
 	mem->last_tree_update = jiffies;
+	mem->on_tree = true;
+}
+
+static void __mem_cgroup_remove_exceeded(struct mem_cgroup *mem)
+{
+	if (!mem->on_tree)
+		return;
+	rb_erase(&mem->mem_cgroup_node, &mem_cgroup_soft_limit_tree);
+	mem->on_tree = false;
+}
+
+static void mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+	__mem_cgroup_insert_exceeded(mem);
 	spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
 }
 

@@ -262,8 +299,34 @@ static void mem_cgroup_remove_exceeded(struct mem_cgroup *mem)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
-	rb_erase(&mem->mem_cgroup_node, &mem_cgroup_soft_limit_tree);
+	__mem_cgroup_remove_exceeded(mem);
+	spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
+}
+
+static struct mem_cgroup *mem_cgroup_get_largest_soft_limit_exceeding_node(void)
+{
+	struct rb_node *rightmost = NULL;
+	struct mem_cgroup *mem = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+retry:
+	rightmost = rb_last(&mem_cgroup_soft_limit_tree);
+	if (!rightmost)
+		goto done;		/* Nothing to reclaim from */
+
+	mem = rb_entry(rightmost, struct mem_cgroup, mem_cgroup_node);
+	/*
+	 * Remove the node now but someone else can add it back,
+	 * we will to add it back at the end of reclaim to its correct
+	 * position in the tree.
+	 */
+	__mem_cgroup_remove_exceeded(mem);
+	if (!css_tryget(&mem->css) || !res_counter_soft_limit_excess(&mem->res))
+		goto retry;
+done:
 	spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
+	return mem;
 }
 
 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,

@@ -324,6 +387,27 @@ static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
 	return total;
 }
 
+static unsigned long long
+mem_cgroup_get_node_zone_usage(struct mem_cgroup *mem, struct zone *zone,
+				int nid)
+{
+	int l;
+	unsigned long long total = 0;
+	struct mem_cgroup_per_zone *mz;
+	unsigned long flags;
+
+	/*
+	 * Is holding the zone LRU lock being overly protective?
+	 * This routine is not invoked from the hot path anyway.
+	 */
+	spin_lock_irqsave(&zone->lru_lock, flags);
+	mz = mem_cgroup_zoneinfo(mem, nid, zone_idx(zone));
+	for_each_evictable_lru(l)
+		total += MEM_CGROUP_ZSTAT(mz, l);
+	spin_unlock_irqrestore(&zone->lru_lock, flags);
+	return total * PAGE_SIZE;
+}
+
 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 {
 	return container_of(cgroup_subsys_state(cont,

@@ -888,14 +972,30 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
  * If shrink==true, for avoiding to free too much, this returns immedieately.
  */
 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
-				   gfp_t gfp_mask, bool noswap, bool shrink)
+						struct zone *zone,
+						gfp_t gfp_mask,
+						unsigned long flags,
+						int priority)
 {
 	struct mem_cgroup *victim;
 	int ret, total = 0;
 	int loop = 0;
+	bool noswap = flags & MEM_CGROUP_RECLAIM_NOSWAP;
+	bool shrink = flags & MEM_CGROUP_RECLAIM_SHRINK;
+	bool check_soft = flags & MEM_CGROUP_RECLAIM_SOFT;
 
 	while (loop < 2) {
 		victim = mem_cgroup_select_victim(root_mem);
+		/*
+		 * In the first loop, don't reclaim from victims below
+		 * their soft limit
+		 */
+		if (!loop && res_counter_check_under_soft_limit(&victim->res)) {
+			if (victim == root_mem)
+				loop++;
+			css_put(&victim->css);
+			continue;
+		}
 		if (victim == root_mem)
 			loop++;
 		if (!mem_cgroup_local_usage(&victim->stat)) {

@@ -904,8 +1004,14 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
 			continue;
 		}
 		/* we use swappiness of local cgroup */
-		ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, noswap,
-						   get_swappiness(victim));
+		if (!check_soft)
+			ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
+							noswap,
+							get_swappiness(victim));
+		else
+			ret = mem_cgroup_shrink_zone(victim, zone, gfp_mask,
+							get_swappiness(victim),
+							priority);
 		css_put(&victim->css);
 		/*
 		 * At shrinking usage, we can't check we should stop here or

@@ -915,7 +1021,10 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
 		if (shrink)
 			return ret;
 		total += ret;
-		if (mem_cgroup_check_under_limit(root_mem))
+		if (check_soft) {
+			if (res_counter_check_under_soft_limit(&root_mem->res))
+				return total;
+		} else if (mem_cgroup_check_under_limit(root_mem))
 			return 1 + total;
 	}
 	return total;

@@ -1025,7 +1134,7 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 
 	while (1) {
 		int ret;
-		bool noswap = false;
+		unsigned long flags = 0;
 
 		ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res,
 						&soft_fail_res);

@@ -1038,7 +1147,7 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 				break;
 			/* mem+swap counter fails */
 			res_counter_uncharge(&mem->res, PAGE_SIZE);
-			noswap = true;
+			flags = MEM_CGROUP_RECLAIM_NOSWAP;
 			mem_over_limit = mem_cgroup_from_res_counter(fail_res,
 									memsw);
 		} else

@@ -1049,8 +1158,8 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 		if (!(gfp_mask & __GFP_WAIT))
 			goto nomem;
 
-		ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, gfp_mask,
-							noswap, false);
+		ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
+							gfp_mask, flags, 0);
 		if (ret)
 			continue;
 

@@ -1082,9 +1191,29 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 	 * soft limit
 	 */
 	if (soft_fail_res) {
+		/*
+		 * Throttle the task here, if it is undergoing soft limit
+		 * reclaim and failing soft limits
+		 */
+		unsigned long flags;
+		bool wait = false;
+
+		spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+		if (mem->soft_limit_reclaim_count) {
+			INIT_COMPLETION(mem->wait_on_soft_reclaim);
+			wait = true;
+		}
+		spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
 		mem_over_soft_limit =

mm/memcontrol.c +19 -5

 	return 0;
 nomem:

@@ -1695,8 +1824,8 @@ int mem_cgroup_shrink_usage(struct page *page,
 		return 0;
 
 	do {
-		progress = mem_cgroup_hierarchical_reclaim(mem,
-					gfp_mask, true, false);
+		progress = mem_cgroup_hierarchical_reclaim(mem, NULL,
+					gfp_mask, MEM_CGROUP_RECLAIM_NOSWAP, 0);
 		progress += mem_cgroup_check_under_limit(mem);
 	} while (!progress && --retry);
 

@@ -1750,8 +1879,9 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
 		if (!ret)
 			break;
 
-		progress = mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL,
-						   false, true);
+		progress = mem_cgroup_hierarchical_reclaim(memcg, NULL,
+						GFP_KERNEL,
+						MEM_CGROUP_RECLAIM_SHRINK, 0);
 		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
 		/* Usage is reduced ? */
   		if (curusage >= oldusage)

@@ -1799,7 +1929,9 @@ int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
 		if (!ret)
 			break;
 
-		mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL, true, true);
+		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
+						MEM_CGROUP_RECLAIM_NOSWAP |
+						MEM_CGROUP_RECLAIM_SHRINK, 0);
 		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
 		/* Usage is reduced ? */
 		if (curusage >= oldusage)

@@ -1810,6 +1942,59 @@ int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
 	return ret;
 }
 
+unsigned long
+mem_cgroup_soft_limit_reclaim(int priority, struct zone *zone, int nid,
+				gfp_t gfp_mask)
+{
+	unsigned long nr_reclaimed = 0;
+	struct mem_cgroup *mem;
+	unsigned long flags;
+	unsigned long long usage;
+
+	/*
+	 * This loop can run a while, specially if mem_cgroup's continuously

Quoted 311 lines · On Fri, 06 Mar 2009 14:53:53 +0530 Balbir Singh <[email protected]>…

---

 include/linux/memcontrol.h |    9 ++
 include/linux/swap.h       |    5 +
 mm/memcontrol.c            |  223 +++++++++++++++++++++++++++++++++++++++++---
 mm/vmscan.c                |   26 +++++
 4 files changed, 245 insertions(+), 18 deletions(-)

include/linux/memcontrol.h +9 -0

index 18146c9..16343d0 100644
@@ -116,6 +116,9 @@ static inline bool mem_cgroup_disabled(void)
 }
 
 extern bool mem_cgroup_oom_called(struct task_struct *task);
+unsigned long
+mem_cgroup_soft_limit_reclaim(int priority, struct zone *zone, int nid,
+				gfp_t gfp_mask);
 
 #else /* CONFIG_CGROUP_MEM_RES_CTLR */
 struct mem_cgroup;

@@ -264,6 +267,12 @@ mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
 {
 }
 
+static inline unsigned long
+mem_cgroup_soft_limit_reclaim(int priority, struct zone *zone, int nid,
+				gfp_t gfp_mask)
+{
+	return 0;
+}
 #endif /* CONFIG_CGROUP_MEM_CONT */
 
 #endif /* _LINUX_MEMCONTROL_H */

include/linux/swap.h +5 -0

index 989eb53..37bc2a9 100644
@@ -217,6 +217,11 @@ extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem,
 						  gfp_t gfp_mask, bool noswap,
 						  unsigned int swappiness);
+extern unsigned long mem_cgroup_shrink_zone(struct mem_cgroup *mem,
+						struct zone *zone,
+						gfp_t gfp_mask,
+						unsigned int swappiness,
+						int priority);
 extern int __isolate_lru_page(struct page *page, int mode, int file);
 extern unsigned long shrink_all_memory(unsigned long nr_pages);
 extern int vm_swappiness;

mm/memcontrol.c +135 -12

index d548dd2..3be1f27 100644
@@ -20,6 +20,7 @@
 #include <linux/res_counter.h>
 #include <linux/memcontrol.h>
 #include <linux/cgroup.h>
+#include <linux/completion.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/smp.h>

@@ -191,6 +192,14 @@ struct mem_cgroup {
 	unsigned long last_tree_update;		/* Last time the tree was */
 						/* updated in jiffies     */
 
+	bool on_tree;				/* Is the node on tree? */
+	struct completion wait_on_soft_reclaim;
+	/*
+	 * Set to > 0, when reclaim is initiated due to 
+	 * the soft limit being exceeded. It adds an additional atomic
+	 * operation to page fault path.
+	 */
+	int soft_limit_reclaim_count;
 	/*
 	 * statistics. This must be placed at the end of memcg.
 	 */

@@ -227,18 +236,29 @@ pcg_default_flags[NR_CHARGE_TYPE] = {
 #define MEMFILE_TYPE(val)	(((val) >> 16) & 0xffff)
 #define MEMFILE_ATTR(val)	((val) & 0xffff)
 
+/*
+ * Bits used for hierarchical reclaim bits
+ */
+#define MEM_CGROUP_RECLAIM_NOSWAP_BIT	0x0
+#define MEM_CGROUP_RECLAIM_NOSWAP	(1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
+#define MEM_CGROUP_RECLAIM_SHRINK_BIT	0x1
+#define MEM_CGROUP_RECLAIM_SHRINK	(1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
+#define MEM_CGROUP_RECLAIM_SOFT_BIT	0x2
+#define MEM_CGROUP_RECLAIM_SOFT		(1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
+
 static void mem_cgroup_get(struct mem_cgroup *mem);
 static void mem_cgroup_put(struct mem_cgroup *mem);
 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
 
-static void mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
+static void __mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
 {
 	struct rb_node **p = &mem_cgroup_soft_limit_tree.rb_node;
 	struct rb_node *parent = NULL;
 	struct mem_cgroup *mem_node;
-	unsigned long flags;
 
-	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+	if (mem->on_tree)
+		return;
+
 	while (*p) {
 		parent = *p;
 		mem_node = rb_entry(parent, struct mem_cgroup, mem_cgroup_node);

@@ -255,6 +275,23 @@ static void mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
 	rb_insert_color(&mem->mem_cgroup_node,
 			&mem_cgroup_soft_limit_tree);
 	mem->last_tree_update = jiffies;
+	mem->on_tree = true;
+}
+
+static void __mem_cgroup_remove_exceeded(struct mem_cgroup *mem)
+{
+	if (!mem->on_tree)
+		return;
+	rb_erase(&mem->mem_cgroup_node, &mem_cgroup_soft_limit_tree);
+	mem->on_tree = false;
+}
+
+static void mem_cgroup_insert_exceeded(struct mem_cgroup *mem)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+	__mem_cgroup_insert_exceeded(mem);
 	spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
 }
 

@@ -262,8 +299,34 @@ static void mem_cgroup_remove_exceeded(struct mem_cgroup *mem)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
-	rb_erase(&mem->mem_cgroup_node, &mem_cgroup_soft_limit_tree);
+	__mem_cgroup_remove_exceeded(mem);
+	spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
+}
+
+static struct mem_cgroup *mem_cgroup_get_largest_soft_limit_exceeding_node(void)
+{
+	struct rb_node *rightmost = NULL;
+	struct mem_cgroup *mem = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+retry:
+	rightmost = rb_last(&mem_cgroup_soft_limit_tree);
+	if (!rightmost)
+		goto done;		/* Nothing to reclaim from */
+
+	mem = rb_entry(rightmost, struct mem_cgroup, mem_cgroup_node);
+	/*
+	 * Remove the node now but someone else can add it back,
+	 * we will to add it back at the end of reclaim to its correct
+	 * position in the tree.
+	 */
+	__mem_cgroup_remove_exceeded(mem);
+	if (!css_tryget(&mem->css) || !res_counter_soft_limit_excess(&mem->res))
+		goto retry;
+done:
 	spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
+	return mem;
 }
 
 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,

@@ -324,6 +387,27 @@ static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
 	return total;
 }
 
+static unsigned long long
+mem_cgroup_get_node_zone_usage(struct mem_cgroup *mem, struct zone *zone,
+				int nid)
+{
+	int l;
+	unsigned long long total = 0;
+	struct mem_cgroup_per_zone *mz;
+	unsigned long flags;
+
+	/*
+	 * Is holding the zone LRU lock being overly protective?
+	 * This routine is not invoked from the hot path anyway.
+	 */
+	spin_lock_irqsave(&zone->lru_lock, flags);
+	mz = mem_cgroup_zoneinfo(mem, nid, zone_idx(zone));
+	for_each_evictable_lru(l)
+		total += MEM_CGROUP_ZSTAT(mz, l);
+	spin_unlock_irqrestore(&zone->lru_lock, flags);
+	return total * PAGE_SIZE;
+}
+
 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 {
 	return container_of(cgroup_subsys_state(cont,

@@ -888,14 +972,30 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
  * If shrink==true, for avoiding to free too much, this returns immedieately.
  */
 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
-				   gfp_t gfp_mask, bool noswap, bool shrink)
+						struct zone *zone,
+						gfp_t gfp_mask,
+						unsigned long flags,
+						int priority)
 {
 	struct mem_cgroup *victim;
 	int ret, total = 0;
 	int loop = 0;
+	bool noswap = flags & MEM_CGROUP_RECLAIM_NOSWAP;
+	bool shrink = flags & MEM_CGROUP_RECLAIM_SHRINK;
+	bool check_soft = flags & MEM_CGROUP_RECLAIM_SOFT;
 
 	while (loop < 2) {
 		victim = mem_cgroup_select_victim(root_mem);
+		/*
+		 * In the first loop, don't reclaim from victims below
+		 * their soft limit
+		 */
+		if (!loop && res_counter_check_under_soft_limit(&victim->res)) {
+			if (victim == root_mem)
+				loop++;
+			css_put(&victim->css);
+			continue;
+		}
 		if (victim == root_mem)
 			loop++;
 		if (!mem_cgroup_local_usage(&victim->stat)) {

@@ -904,8 +1004,14 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
 			continue;
 		}
 		/* we use swappiness of local cgroup */
-		ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, noswap,
-						   get_swappiness(victim));
+		if (!check_soft)
+			ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
+							noswap,
+							get_swappiness(victim));
+		else
+			ret = mem_cgroup_shrink_zone(victim, zone, gfp_mask,
+							get_swappiness(victim),
+							priority);
 		css_put(&victim->css);
 		/*
 		 * At shrinking usage, we can't check we should stop here or

@@ -915,7 +1021,10 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
 		if (shrink)
 			return ret;
 		total += ret;
-		if (mem_cgroup_check_under_limit(root_mem))
+		if (check_soft) {
+			if (res_counter_check_under_soft_limit(&root_mem->res))
+				return total;
+		} else if (mem_cgroup_check_under_limit(root_mem))
 			return 1 + total;
 	}
 	return total;

@@ -1025,7 +1134,7 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 
 	while (1) {
 		int ret;
-		bool noswap = false;
+		unsigned long flags = 0;
 
 		ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res,
 						&soft_fail_res);

@@ -1038,7 +1147,7 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 				break;
 			/* mem+swap counter fails */
 			res_counter_uncharge(&mem->res, PAGE_SIZE);
-			noswap = true;
+			flags = MEM_CGROUP_RECLAIM_NOSWAP;
 			mem_over_limit = mem_cgroup_from_res_counter(fail_res,
 									memsw);
 		} else

@@ -1049,8 +1158,8 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 		if (!(gfp_mask & __GFP_WAIT))
 			goto nomem;
 
-		ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, gfp_mask,
-							noswap, false);
+		ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
+							gfp_mask, flags, 0);
 		if (ret)
 			continue;
 

@@ -1082,9 +1191,29 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
 	 * soft limit
 	 */
 	if (soft_fail_res) {
+		/*
+		 * Throttle the task here, if it is undergoing soft limit
+		 * reclaim and failing soft limits
+		 */
+		unsigned long flags;
+		bool wait = false;
+
+		spin_lock_irqsave(&memcg_soft_limit_tree_lock, flags);
+		if (mem->soft_limit_reclaim_count) {
+			INIT_COMPLETION(mem->wait_on_soft_reclaim);
+			wait = true;
+		}
+		spin_unlock_irqrestore(&memcg_soft_limit_tree_lock, flags);
 		mem_over_soft_limit =
 			mem_cgroup_from_res_counter(soft_fail_res, res);
 		mem_cgroup_check_and_update_tree(mem_over_soft_limit, true);
+		/*

mm/memcontrol.c static int __mem_cgroup_try_charge(struct mm_struct *mm, · line 1217

Quoted 7 lines

mm/memcontrol.c +6 -0

+		 * We hold the mmap_sem and throttle, I don't think there
+		 * should be corner cases, but this part could use more
+		 * review
+		 */
+		if (wait)
+			wait_for_completion(&mem->wait_on_soft_reclaim);
 	}

What ???? Why we have to wait here...holding mmap->sem...This is too bad.

Quoted 58 lines

mm/memcontrol.c +24 -5

 	return 0;
 nomem:

@@ -1695,8 +1824,8 @@ int mem_cgroup_shrink_usage(struct page *page,
 		return 0;
 
 	do {
-		progress = mem_cgroup_hierarchical_reclaim(mem,
-					gfp_mask, true, false);
+		progress = mem_cgroup_hierarchical_reclaim(mem, NULL,
+					gfp_mask, MEM_CGROUP_RECLAIM_NOSWAP, 0);
 		progress += mem_cgroup_check_under_limit(mem);
 	} while (!progress && --retry);
 

@@ -1750,8 +1879,9 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
 		if (!ret)
 			break;
 
-		progress = mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL,
-						   false, true);
+		progress = mem_cgroup_hierarchical_reclaim(memcg, NULL,
+						GFP_KERNEL,
+						MEM_CGROUP_RECLAIM_SHRINK, 0);
 		curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
 		/* Usage is reduced ? */
   		if (curusage >= oldusage)

@@ -1799,7 +1929,9 @@ int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
 		if (!ret)
 			break;
 
-		mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL, true, true);
+		mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
+						MEM_CGROUP_RECLAIM_NOSWAP |
+						MEM_CGROUP_RECLAIM_SHRINK, 0);
 		curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
 		/* Usage is reduced ? */
 		if (curusage >= oldusage)

@@ -1810,6 +1942,59 @@ int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
 	return ret;
 }
 
+unsigned long
+mem_cgroup_soft_limit_reclaim(int priority, struct zone *zone, int nid,
+				gfp_t gfp_mask)
+{
+	unsigned long nr_reclaimed = 0;
+	struct mem_cgroup *mem;
+	unsigned long flags;
+	unsigned long long usage;
+
+	/*
+	 * This loop can run a while, specially if mem_cgroup's continuously
+	 * keep exceeding their soft limit and putting the system under
+	 * pressure
+	 */
+	do {
+		mem = mem_cgroup_get_largest_soft_limit_exceeding_node();

mm/memcontrol.c int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, · line 1965

Quoted 5 lines

mm/memcontrol.c +5 -0

+		if (!mem)
+			break;
+		usage = mem_cgroup_get_node_zone_usage(mem, zone, nid);
+		if (!usage)
+			goto skip_reclaim;

Why this works well ? if "mem" is the laragest, it will be inserted again as the largest. Do I miss any ?

Quoted 5 lines

mm/memcontrol.c static int __mem_cgroup_try_charge(struct mm_struct *mm, · line 1217

Quoted 3 lines

mm/memcontrol.c +2 -0

+		if (wait)
+			wait_for_completion(&mem->wait_on_soft_reclaim);
 	}

What ???? Why we have to wait here...holding mmap->sem...This is too bad.

Since mmap_sem is no longer used for pthread_mutex*, I was not sure. That is why I added the comment asking for more review and see what people think about it. We get here only when

1. The memcg is over its soft limit
2. Tasks/threads belonging to memcg are faulting in more pages

The idea is to throttle them. If we did reclaim inline, like we do for hard limits, we can still end up holding mmap_sem for a long time.

Quoted 7 lines

Quoted 7 lines

mm/memcontrol.c +7 -0

+	/*
+	 * This loop can run a while, specially if mem_cgroup's continuously
+	 * keep exceeding their soft limit and putting the system under
+	 * pressure
+	 */
+	do {
+		mem = mem_cgroup_get_largest_soft_limit_exceeding_node();

Quoted 9 lines

mm/memcontrol.c int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, · line 1965

Quoted 5 lines

mm/memcontrol.c +5 -0

+		if (!mem)
+			break;
+		usage = mem_cgroup_get_node_zone_usage(mem, zone, nid);
+		if (!usage)
+			goto skip_reclaim;

Why this works well ? if "mem" is the laragest, it will be inserted again as the largest. Do I miss any ?

No that is correct, but when reclaim is initiated from a different zone/node combination, we still want mem to show up.

Quoted 10 lines · On Fri, 6 Mar 2009 15:31:55 +0530 Balbir Singh <[email protected]>…

Quoted 5 lines

mm/memcontrol.c static int __mem_cgroup_try_charge(struct mm_struct *mm, · line 1217

Quoted 3 lines

mm/memcontrol.c +2 -0

+		if (wait)
+			wait_for_completion(&mem->wait_on_soft_reclaim);
 	}

What ???? Why we have to wait here...holding mmap->sem...This is too bad.

Since mmap_sem is no longer used for pthread_mutex*, I was not sure. That is why I added the comment asking for more review and see what people think about it. We get here only when

Quoted 6 lines

1. The memcg is over its soft limit
2. Tasks/threads belonging to memcg are faulting in more pages

The idea is to throttle them. If we did reclaim inline, like we do for hard limits, we can still end up holding mmap_sem for a long time.

This "throttle" is hard to measuer the effect and IIUC, not implemneted in vmscan.c ...for global try_to_free_pages() yet.
Under memory shortage. before reaching here, the thread already called try_to_free_pages() or check some memory shorage conditions because it called alloc_pages(). So, waiting here is redundant and gives it too much penaly.

Quoted 19 lines

Quoted 18 lines

Quoted 4 lines

mm/memcontrol.c +2 -0

+	/*
+	 * This loop can run a while, specially if mem_cgroup's continuously

mm/memcontrol.c int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg, · line 1965

Quoted 10 lines

mm/memcontrol.c +10 -0

+	 * keep exceeding their soft limit and putting the system under
+	 * pressure
+	 */
+	do {
+		mem = mem_cgroup_get_largest_soft_limit_exceeding_node();
+		if (!mem)
+			break;
+		usage = mem_cgroup_get_node_zone_usage(mem, zone, nid);
+		if (!usage)
+			goto skip_reclaim;

Why this works well ? if "mem" is the laragest, it will be inserted again as the largest. Do I miss any ?

Quoted 2 lines

No that is correct, but when reclaim is initiated from a different zone/node combination, we still want mem to show up.

....
your logic is
==
nr_reclaimd = 0;
do {
mem = select victim.
remvoe victim from the RBtree (the largest usage one is selected) if (victim is not good)
goto skip this.
reclaimed += shirnk_zone.

skip_this:
if (mem is still exceeds soft limit)
insert RB tree again.
} while(!nr_reclaimed)
==
When this exits loop ?

Thanks,
-Kame

From: Balbir Singh <[email protected]>

New Feature: Soft limits for memory resource controller.

Changelog v4...v3
1. Adopted suggestions from Kamezawa to do a per-zone-per-node reclaim while doing soft limit reclaim. We don't record priorities while doing soft reclaim
2. Some of the overheads associated with soft limits (like calculating excess each time) is eliminated
3. The time_after(jiffies, 0) bug has been fixed
4. Tasks are throttled if the mem cgroup they belong to is being soft reclaimed and at the same time tasks are increasing the memory footprint and causing the mem cgroup to exceed its soft limit.

Changelog v3...v2
1. Implemented several review comments from Kosaki-San and Kamezawa-San Please see individual changelogs for changes

Changelog v2...v1
1. Soft limits now support hierarchies
2. Use spinlocks instead of mutexes for synchronization of the RB tree

Here is v4 of the new soft limit implementation. Soft limits is a new feature for the memory resource controller, something similar has existed in the group scheduler in the form of shares. The CPU controllers interpretation of shares is very different though.

Soft limits are the most useful feature to have for environments where the administrator wants to overcommit the system, such that only on memory contention do the limits become active. The current soft limits implementation provides a soft_limit_in_bytes interface for the memory controller and not for memory+swap controller. The implementation maintains an RB-Tree of groups that exceed their soft limit and starts reclaiming from the group that exceeds this limit by the maximum amount.

If there are no major objections to the patches, I would like to get them included in -mm.

Quoted 5 lines · On Fri, 06 Mar 2009 14:53:23 +0530 Balbir Singh <[email protected]>…

From: Balbir Singh <[email protected]>

New Feature: Soft limits for memory resource controller.

Quoted 11 lines

Changelog v4...v3
1. Adopted suggestions from Kamezawa to do a per-zone-per-node reclaim while doing soft limit reclaim. We don't record priorities while doing soft reclaim
2. Some of the overheads associated with soft limits (like calculating excess each time) is eliminated
3. The time_after(jiffies, 0) bug has been fixed
4. Tasks are throttled if the mem cgroup they belong to is being soft reclaimed and at the same time tasks are increasing the memory footprint and causing the mem cgroup to exceed its soft limit.

I don't think this "4" is necessary.

Quoted 22 lines

Changelog v3...v2
1. Implemented several review comments from Kosaki-San and Kamezawa-San Please see individual changelogs for changes

Changelog v2...v1
1. Soft limits now support hierarchies
2. Use spinlocks instead of mutexes for synchronization of the RB tree

Here is v4 of the new soft limit implementation. Soft limits is a new feature for the memory resource controller, something similar has existed in the group scheduler in the form of shares. The CPU controllers interpretation of shares is very different though.

Soft limits are the most useful feature to have for environments where the administrator wants to overcommit the system, such that only on memory contention do the limits become active. The current soft limits implementation provides a soft_limit_in_bytes interface for the memory controller and not for memory+swap controller. The implementation maintains an RB-Tree of groups that exceed their soft limit and starts reclaiming from the group that exceeds this limit by the maximum amount.

Quoted 3 lines

If there are no major objections to the patches, I would like to get them included in -mm.

You got Nack from me, again ;) And you know why.
I'll post my one later, I hope that one will be good input for you.

From: KAMEZAWA Hiroyuki <[email protected]>

Documentation for softlimit (3/3)

Signed-off-by KAMEZAWA Hiroyuki <[email protected]>

---
 Documentation/cgroups/memory.txt |   19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

Index: mmotm-2.6.29-Mar3/Documentation/cgroups/memory.txt
===================================================================

Documentation/cgroups/memory.txt +1 -0

+  victim in turn.
 
 6. Hierarchy support

Quoted 1 line · KAMEZAWA Hiroyuki

From: KAMEZAWA Hiroyuki <[email protected]>

Quoted 13 lines

Documentation for softlimit (3/3)

Signed-off-by KAMEZAWA Hiroyuki <[email protected]>

---
 Documentation/cgroups/memory.txt |   19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

Index: mmotm-2.6.29-Mar3/Documentation/cgroups/memory.txt
===================================================================

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 326

Quoted 5 lines

Documentation/cgroups/memory.txt +2 -0

@@ -322,6 +322,25 @@ will be charged as a new owner of it.
   - a cgroup which uses hierarchy and it has child cgroup.
   - a cgroup which uses hierarchy and not the root of hierarchy.
 
+5.4 softlimit
+  Memory cgroup supports softlimit and has 2 params for control.

parameters

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 328

Quoted 2 lines

Documentation/cgroups/memory.txt +2 -0

+    - memory.softlimit_in_bytes
+	- softlimit to this cgroup.

softlimit for this cgroup
(i.e., no beginning '-' and no ending '.')

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 330

Quoted 2 lines

Documentation/cgroups/memory.txt +2 -0

+    - memory.softlimit_priority.
+	- priority of this cgroup at softlimit reclaim.

priority of this cgroup at softlimit reclaim

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 331

Quoted 1 line

Documentation/cgroups/memory.txt +1 -0

+	  Allowed priority level is 3-0 and 3 is the lowest.

Not very user friendly...

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 335

Quoted 4 lines

Documentation/cgroups/memory.txt +4 -0

+	  If 0, this cgroup will not be target of softlimit.
+
+  At memory shortage of the system (or local node/zone), softlimit helps
+  kswapd(), a global memory recalim kernel thread, and inform victim cgroup

reclaim informs

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 339

Quoted 4 lines

Documentation/cgroups/memory.txt +4 -0

+  to be shrinked to kswapd.
+
+  Victim selection logic:
+  The kernel searches from the lowest priroty(3) up to the highest(1).

priority 0 ?? (from above)

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 340

Quoted 1 line

Documentation/cgroups/memory.txt +1 -0

+  If it find a cgroup witch has memory larger than softlimit, steal memory

finds which

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 342

Quoted 2 lines

Documentation/cgroups/memory.txt +2 -0

+  from it.
+  If multiple cgroups are on the same priority, each cgroup wil be a

will

Quoted 3 lines

Documentation/cgroups/memory.txt +1 -0

+  victim in turn.
 
 6. Hierarchy support

Signature · 1 line

~Randy

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 331

Quoted 1 line

Documentation/cgroups/memory.txt +1 -0

+	  Allowed priority level is 3-0 and 3 is the lowest.

Not very user friendly...

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 335

Quoted 4 lines

Documentation/cgroups/memory.txt +4 -0

+	  If 0, this cgroup will not be target of softlimit.
+
+  At memory shortage of the system (or local node/zone), softlimit helps
+  kswapd(), a global memory recalim kernel thread, and inform victim cgroup

reclaim informs

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 339

Quoted 4 lines

Documentation/cgroups/memory.txt +4 -0

+  to be shrinked to kswapd.
+
+  Victim selection logic:
+  The kernel searches from the lowest priroty(3) up to the highest(1).

priority 0 ?? (from above)

Quoted 1 line

+ If it find a cgroup witch has memory larger than softlimit, steal memory

finds which

Documentation/cgroups/memory.txt will be charged as a new owner of it. · line 342

Quoted 2 lines

Documentation/cgroups/memory.txt +2 -0

+  from it.
+  If multiple cgroups are on the same priority, each cgroup wil be a

will

Quoted 3 lines

+ victim in turn.

6. Hierarchy support

Signature · 7 lines

~Randy

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to [email protected].  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>