Introduce memory allocation speed throttle in memcg

index 679591301994..b8286c8f6704 100644
@@ -27,6 +27,14 @@ struct page_counter {
 	unsigned long watermark;
 	unsigned long failcnt;
 
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+	/* allocation speed throttle */
+	atomic_long_t total_chg;
+	atomic_long_t mem_spd_max;
+	atomic_long_t prev_spd_jifs;
+	atomic_long_t prev_total_chg;
+#endif
+
 	/*
 	 * 'parent' is placed here to be far from 'usage' to reduce
 	 * cache false sharing, as 'usage' is written mostly while

index 64ada9e650a5..7ef6aa088680 100644
@@ -1392,6 +1392,37 @@ static int memcg_page_state_unit(int item)
 	}
 }
 
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+static unsigned long mem_cgroup_mst_get_mem_spd_max(struct mem_cgroup *memcg)
+{
+	struct page_counter *c = &memcg->memory;
+	unsigned long deadline;
+
+	/* Reset speed if it is too old */
+	deadline = atomic_long_read(&c->prev_spd_jifs) + 5 * HZ;
+	if (time_after(jiffies, deadline))
+		atomic_long_set(&c->mem_spd_max, 0);
+
+	/* Clear after read */
+	return atomic_long_xchg(&c->mem_spd_max, 0) << PAGE_SHIFT;
+}
+
+static void mem_cgroup_mst_show_mem_spd_max(struct mem_cgroup *memcg,
+					    struct seq_file *m)
+{
+	if (mem_cgroup_is_root(memcg))
+		return;
+
+	seq_printf(m, "mst_mem_spd_max %lu\n",
+		   mem_cgroup_mst_get_mem_spd_max(memcg));
+}
+#else /* CONFIG_MEM_SPEED_THROTTLE */
+static void mem_cgroup_mst_show_mem_spd_max(struct mem_cgroup *memcg,
+					    struct seq_file *m)
+{
+}
+#endif
+
 static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
 						    int item)
 {

@@ -1462,6 +1493,11 @@ static char *memory_stat_format(struct mem_cgroup *memcg)
 		       memcg_events(memcg, THP_COLLAPSE_ALLOC));
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+	seq_buf_printf(&s, "mst_mem_spd_max %lu\n",
+		mem_cgroup_mst_get_mem_spd_max(memcg));
+#endif
+
 	/* The above should easily fit into one page */
 	WARN_ON_ONCE(seq_buf_has_overflowed(&s));
 

@@ -3909,6 +3945,8 @@ static int memcg_stat_show(struct seq_file *m, void *v)
 	}
 #endif
 
+	mem_cgroup_mst_show_mem_spd_max(memcg, m);
+
 	return 0;
 }
 

index 7d83641eb86b..d5e2cf4daf4f 100644
@@ -61,6 +61,39 @@ void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages)
 	propagate_protected_usage(counter, new);
 }
 
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+static void mem_speed_update(struct page_counter *c, unsigned long nr_pages)
+{
+	unsigned long prev_jifs, chg, spd_max;
+	long delta_jifs, delta_chg;
+
+	chg = atomic_long_add_return(nr_pages, &c->total_chg);
+	prev_jifs = atomic_long_read(&c->prev_spd_jifs);
+
+	/* First time we get here? */
+	if (!prev_jifs)
+		goto done;
+
+	delta_jifs = jiffies - prev_jifs;
+	delta_chg = chg - atomic_long_read(&c->prev_total_chg);
+
+	if (delta_jifs <= HZ || delta_chg <= 0)
+		return;
+
+	spd_max = max(delta_chg * HZ / delta_jifs,
+			atomic_long_read(&c->mem_spd_max));
+	atomic_long_set(&c->mem_spd_max, spd_max);
+
+done:
+	atomic_long_set(&c->prev_spd_jifs, jiffies);
+	atomic_long_set(&c->prev_total_chg, chg);
+}
+#else
+static void mem_speed_update(struct page_counter *c, unsigned long nr_pages)
+{
+}
+#endif
+
 /**
  * page_counter_charge - hierarchically charge pages
  * @counter: counter

@@ -83,6 +116,9 @@ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages)
 		 */
 		if (new > READ_ONCE(c->watermark))
 			WRITE_ONCE(c->watermark, new);
+
+		/* Update mem speed max, maybe inaccurate due to races */
+		mem_speed_update(c, nr_pages);
 	}
 }
 

@@ -137,6 +173,9 @@ bool page_counter_try_charge(struct page_counter *counter,
 		 */
 		if (new > READ_ONCE(c->watermark))
 			WRITE_ONCE(c->watermark, new);
+
+		/* Update mem speed max, maybe inaccurate due to races */
+		mem_speed_update(c, nr_pages);
 	}
 	return true;
 

index c193be760709..4938397f321d 100644
@@ -230,6 +230,12 @@ struct obj_cgroup {
 	};
 };
 
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+struct mem_spd_ctl {
+	unsigned long mem_spd_lmt;
+};
+#endif
+
 /*
  * The memory controller data structure. The memory controller controls both
  * page cache and RSS per cgroup. We would eventually like to provide

@@ -349,6 +355,10 @@ struct mem_cgroup {
 	struct deferred_split deferred_split_queue;
 #endif
 
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+	struct mem_spd_ctl msc;
+#endif
+
 	struct mem_cgroup_per_node *nodeinfo[0];
 	/* WARNING: nodeinfo must be the last member here */
 };

index 7ef6aa088680..d59abe15d89d 100644
@@ -1393,6 +1393,28 @@ static int memcg_page_state_unit(int item)
 }
 
 #ifdef CONFIG_MEM_SPEED_THROTTLE
+static u64 mem_cgroup_mem_spd_lmt_read(struct cgroup_subsys_state *css,
+				       struct cftype *cft)
+{
+	struct mem_cgroup *memcg = mem_cgroup_from_css(css);
+
+	return (u64)(memcg->msc.mem_spd_lmt << PAGE_SHIFT);
+}
+
+static int mem_cgroup_mem_spd_lmt_write(struct cgroup_subsys_state *css,
+					struct cftype *cft, u64 val)
+{
+	struct mem_cgroup *memcg;
+	unsigned long lmt;
+
+	memcg = mem_cgroup_from_css(css);
+	lmt = val >> PAGE_SHIFT;
+
+	memcg->msc.mem_spd_lmt = lmt;
+
+	return 0;
+}
+
 static unsigned long mem_cgroup_mst_get_mem_spd_max(struct mem_cgroup *memcg)
 {
 	struct page_counter *c = &memcg->memory;

@@ -4805,6 +4827,14 @@ static struct cftype mem_cgroup_legacy_files[] = {
 		.write_u64 = mem_cgroup_hierarchy_write,
 		.read_u64 = mem_cgroup_hierarchy_read,
 	},
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+	{
+		.name = "alloc_bps",
+		.flags = CFTYPE_NOT_ON_ROOT,
+		.read_u64 = mem_cgroup_mem_spd_lmt_read,
+		.write_u64 = mem_cgroup_mem_spd_lmt_write,
+	},
+#endif
 	{
 		.name = "cgroup.event_control",		/* XXX: for compat */
 		.write = memcg_write_event_control,

@@ -6358,6 +6388,14 @@ static struct cftype memory_files[] = {
 		.seq_show = memory_oom_group_show,
 		.write = memory_oom_group_write,
 	},
+#ifdef CONFIG_MEM_SPEED_THROTTLE
+	{
+		.name = "alloc_bps",
+		.write_u64 = mem_cgroup_mem_spd_lmt_write,
+		.read_u64 = mem_cgroup_mem_spd_lmt_read,
+		.flags = CFTYPE_NOT_ON_ROOT,
+	},
+#endif
 	{ }	/* terminate */
 };
 

index 4938397f321d..435515b1a709 100644
@@ -233,6 +233,7 @@ struct obj_cgroup {
 #ifdef CONFIG_MEM_SPEED_THROTTLE
 struct mem_spd_ctl {
 	unsigned long mem_spd_lmt;
+	int has_lmt;
 };
 #endif
 

86 lines not on this page · raw message

index 435515b1a709..230acdc635b5 100644
@@ -231,8 +231,12 @@ struct obj_cgroup {
 };
 
 #ifdef CONFIG_MEM_SPEED_THROTTLE
+#define MST_SLICE		(HZ/10)
 struct mem_spd_ctl {
 	unsigned long mem_spd_lmt;
+	unsigned long slice_lmt;
+	unsigned long prev_thl_jifs;
+	unsigned long prev_chg;
 	int has_lmt;
 };
 #endif

index 9d7a86f7f51c..05bf4ba0da15 100644
@@ -1403,6 +1403,9 @@ static void mem_cgroup_mst_msc_reset(struct mem_cgroup *memcg)
 	msc = &memcg->msc;
 	msc->has_lmt = 0;
 	msc->mem_spd_lmt = 0;
+	msc->slice_lmt = 0;
+	msc->prev_chg = 0;
+	msc->prev_thl_jifs = 0;
 }
 
 static void mem_cgroup_mst_has_lmt_init(struct mem_cgroup *memcg)

@@ -1440,6 +1443,7 @@ static int mem_cgroup_mem_spd_lmt_write(struct cgroup_subsys_state *css,
 	lmt = val >> PAGE_SHIFT;
 
 	memcg->msc.mem_spd_lmt = lmt;
+	memcg->msc.slice_lmt = lmt * MST_SLICE / HZ;
 
 	/* Sync with mst_has_lmt_init*/
 	synchronize_rcu();

@@ -1452,6 +1456,27 @@ static int mem_cgroup_mem_spd_lmt_write(struct cgroup_subsys_state *css,
 	return 0;
 }
 
+/*
+ * Update the memory speed throttle slice window.
+ * Return 0 when we still in the previous slice window
+ * Return 1 when we start a new slice window
+ */
+static int mem_cgroup_update_mst_slice(struct mem_cgroup *memcg)
+{
+	unsigned long total_charge;
+	struct mem_spd_ctl *msc = &memcg->msc;
+
+	if (msc->prev_thl_jifs &&
+	    time_before(jiffies, (msc->prev_thl_jifs + MST_SLICE)))
+		return 0;
+
+	total_charge = atomic_long_read(&memcg->memory.total_chg);
+	msc->prev_chg = total_charge;
+	msc->prev_thl_jifs = jiffies;
+
+	return 1;
+}
+
 static unsigned long mem_cgroup_mst_get_mem_spd_max(struct mem_cgroup *memcg)
 {
 	struct page_counter *c = &memcg->memory;

index 230acdc635b5..b20820f7fdbf 100644
@@ -238,6 +238,7 @@ struct mem_spd_ctl {
 	unsigned long prev_thl_jifs;
 	unsigned long prev_chg;
 	int has_lmt;
+	atomic_t updating;
 };
 #endif
 

140 lines not on this page · raw message

index b20820f7fdbf..59e6cb78a07a 100644
@@ -239,6 +239,7 @@ struct mem_spd_ctl {
 	unsigned long prev_chg;
 	int has_lmt;
 	atomic_t updating;
+	atomic_long_t nr_throttled;
 };
 #endif
 

index a1e33a9e6594..ca39974403a3 100644
@@ -1401,11 +1401,19 @@ static void mem_cgroup_mst_msc_reset(struct mem_cgroup *memcg)
 		return;
 
 	msc = &memcg->msc;
-	msc->has_lmt = 0;
-	msc->mem_spd_lmt = 0;
-	msc->slice_lmt = 0;
-	msc->prev_chg = 0;
-	msc->prev_thl_jifs = 0;
+	while (msc->prev_thl_jifs ||
+	    msc->prev_chg ||
+	    msc->slice_lmt ||
+	    msc->mem_spd_lmt ||
+	    msc->has_lmt ||
+	    atomic_long_read(&msc->nr_throttled)) {
+		atomic_long_set(&msc->nr_throttled, 0);
+		msc->has_lmt = 0;
+		msc->mem_spd_lmt = 0;
+		msc->slice_lmt = 0;
+		msc->prev_chg = 0;
+		msc->prev_thl_jifs = 0;
+	}
 }
 
 static void mem_cgroup_mst_has_lmt_init(struct mem_cgroup *memcg)

@@ -1552,6 +1560,8 @@ static void mem_cgroup_mst_spd_throttle(struct mem_cgroup *memcg)
 	if (!ret)
 		return;
 
+	atomic_long_inc(&msc->nr_throttled);
+
 	/*
 	 * Throttle the allocation for amount of jiffies according to
 	 * the fraction between the actual memory usage and allowed

@@ -1586,11 +1596,22 @@ static void mem_cgroup_mst_show_mem_spd_max(struct mem_cgroup *memcg,
 	seq_printf(m, "mst_mem_spd_max %lu\n",
 		   mem_cgroup_mst_get_mem_spd_max(memcg));
 }
+
+static void mem_cgroup_mst_show_nr_throttled(struct mem_cgroup *memcg,
+					     struct seq_file *m)
+{
+	seq_printf(m, "mst_nr_throttled %lu\n",
+		   atomic_long_read(&memcg->msc.nr_throttled));
+}
 #else /* CONFIG_MEM_SPEED_THROTTLE */
 static void mem_cgroup_mst_spd_throttle(struct mem_cgroup *memcg)
 {
 }
 
+static void mem_cgroup_mst_spd_throttle(struct mem_cgroup *memcg)
+{
+}
+
 static void mem_cgroup_mst_has_lmt_init(struct mem_cgroup *memcg)
 {
 }

@@ -1678,6 +1699,8 @@ static char *memory_stat_format(struct mem_cgroup *memcg)
 #ifdef CONFIG_MEM_SPEED_THROTTLE
 	seq_buf_printf(&s, "mst_mem_spd_max %lu\n",
 		mem_cgroup_mst_get_mem_spd_max(memcg));
+	seq_buf_printf(&s, "mst_nr_throttled %lu\n",
+		atomic_long_read(&memcg->msc.nr_throttled));
 #endif
 
 	/* The above should easily fit into one page */

@@ -4131,6 +4154,7 @@ static int memcg_stat_show(struct seq_file *m, void *v)
 #endif
 
 	mem_cgroup_mst_show_mem_spd_max(memcg, m);
+	mem_cgroup_mst_show_nr_throttled(memcg, m);
 
 	return 0;
 }

index 59e6cb78a07a..011d1426a924 100644
@@ -241,6 +241,12 @@ struct mem_spd_ctl {
 	atomic_t updating;
 	atomic_long_t nr_throttled;
 };
+
+enum mst_wmark_stat {
+	WMARK_OK = 0,
+	WMARK_REACH_LOW,
+	WMARK_REACH_MIN,
+};
 #endif
 
 /*

index ca39974403a3..b1111dc8e585 100644
@@ -1546,18 +1546,47 @@ static int mem_cgroup_mst_overspd_tree(struct mem_cgroup *memcg)
 	return ret;
 }
 
+static enum mst_wmark_stat mem_cgroup_mst_wmark_ok(struct mem_cgroup *memcg)
+{
+	int nid;
+
+	for_each_online_node(nid) {
+		unsigned long free, low, min;
+		struct zone *zone;
+
+		zone = &NODE_DATA(nid)->node_zones[ZONE_NORMAL];
+		free = zone_page_state(zone, NR_FREE_PAGES);
+		low = low_wmark_pages(zone);
+		min = min_wmark_pages(zone);
+		min += (low - min) >> 2;
+
+		if (free <= min)
+			return WMARK_REACH_MIN;
+
+		if (free <= low)
+			return WMARK_REACH_LOW;
+
+	}
+	return WMARK_OK;
+}
+
 static void mem_cgroup_mst_spd_throttle(struct mem_cgroup *memcg)
 {
 	struct mem_spd_ctl *msc = &memcg->msc;
 	long timeout;
 	int ret = 0;
+	enum mst_wmark_stat stat;
 
 	if (!memcg->msc.has_lmt || in_interrupt() || in_atomic() ||
 		irqs_disabled() || oops_in_progress)
 		return;
 
+	stat = mem_cgroup_mst_wmark_ok(memcg);
+	if (stat == WMARK_OK)
+		return;
+
 	ret = mem_cgroup_mst_overspd_tree(memcg);
-	if (!ret)
+	if (stat == WMARK_REACH_LOW && !ret)
 		return;
 
 	atomic_long_inc(&msc->nr_throttled);