cgroups: block: cfq: I/O bandwidth controlling subsystem for CGroups based on CFQ

@@ -0,0 +1,103 @@
+#ifndef _LINUX_CFQ_IOSCHED_H
+#define _LINUX_CFQ_IOSCHED_H
+
+/*
+ * Most of our rbtree usage is for sorting with min extraction, so
+ * if we cache the leftmost node we don't have to walk down the tree
+ * to find it. Idea borrowed from Ingo Molnars CFS scheduler. We should
+ * move this into the elevator for the rq sorting as well.
+ */
+struct cfq_rb_root {
+	struct rb_root rb;
+	struct rb_node *left;
+};
+#define CFQ_RB_ROOT	(struct cfq_rb_root) { RB_ROOT, NULL, }
+
+/*
+ * Per block device queue structure
+ */
+struct cfq_data {
+	struct request_queue *queue;
+
+	/*
+	 * rr list of queues with requests and the count of them
+	 */
+	struct cfq_rb_root service_tree;
+	unsigned int busy_queues;
+
+	int rq_in_driver;
+	int sync_flight;
+	int hw_tag;
+
+	/*
+	 * idle window management
+	 */
+	struct timer_list idle_slice_timer;
+	struct work_struct unplug_work;
+
+	struct cfq_queue *active_queue;
+	struct cfq_io_context *active_cic;
+
+	/*
+	 * async queue for each priority case
+	 */
+	struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR];
+	struct cfq_queue *async_idle_cfqq;
+
+	sector_t last_position;
+	unsigned long last_end_request;
+
+	/*
+	 * tunables, see top of file
+	 */
+	unsigned int cfq_quantum;
+	unsigned int cfq_fifo_expire[2];
+	unsigned int cfq_back_penalty;
+	unsigned int cfq_back_max;
+	unsigned int cfq_slice[2];
+	unsigned int cfq_slice_async_rq;
+	unsigned int cfq_slice_idle;
+
+	struct list_head cic_list;
+};
+
+/*
+ * Per process-grouping structure
+ */
+struct cfq_queue {
+	/* reference count */
+	atomic_t ref;
+	/* parent cfq_data */
+	struct cfq_data *cfqd;
+	/* service_tree member */
+	struct rb_node rb_node;
+	/* service_tree key */
+	unsigned long rb_key;
+	/* sorted list of pending requests */
+	struct rb_root sort_list;
+	/* if fifo isn't expired, next request to serve */
+	struct request *next_rq;
+	/* requests queued in sort_list */
+	int queued[2];
+	/* currently allocated requests */
+	int allocated[2];
+	/* pending metadata requests */
+	int meta_pending;
+	/* fifo list of requests in sort_list */
+	struct list_head fifo;
+
+	unsigned long slice_end;
+	long slice_resid;
+
+	/* number of requests that are on the dispatch list or inside driver */
+	int dispatched;
+
+	/* io prio of this group */
+	unsigned short ioprio, org_ioprio;
+	unsigned short ioprio_class, org_ioprio_class;
+
+	/* various state flags, see below */
+	unsigned int flags;
+};
+
+#endif /* _LINUX_CFQ_IOSCHED_H */

112 lines not on this page · raw message

@@ -0,0 +1,26 @@
+/*
+ * include/linux/cfqio-cgroup.h
+ *
+ * cfqio_subsys: CGroup subsystem that allows CFQ recognize
+ * cgroups and perform scheduling according to this.
+ *
+ * Copyright (C) 2008 OpenVZ http://openvz.org
+ *
+ * Author: Vasily Tarasov <[email protected]>
+ *
+ */
+
+#ifndef _LINUX_CFQIO_CGROUP_H
+#define _LINUX_CFQIO_CGROUP_H
+
+#define CFQIO_SS_IOPRIO_DEF	4
+#define CFQIO_SS_IOPRIO_MAX	7
+#define CFQIO_SS_IOPRIO_MIN	0
+
+/* cfqio_subsys's per cgroup state holder */
+struct cfqio_ss_css {
+	struct cgroup_subsys_state	css;
+	unsigned int			ioprio;
+};
+
+#endif /* _LINUX_CFQIO_CGROUP_H */

@@ -42,3 +42,9 @@ SUBSYS(mem_cgroup)
 #endif
 
 /* */
+
+#ifdef CONFIG_CGROUP_CFQIO
+SUBSYS(cfqio)
+#endif
+
+/* */

97 lines not on this page · raw message

@@ -13,6 +13,10 @@
 #ifndef _LINUX_CFQIO_CGROUP_H
 #define _LINUX_CFQIO_CGROUP_H
 
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/cfq-iosched.h>
+
 #define CFQIO_SS_IOPRIO_DEF	4
 #define CFQIO_SS_IOPRIO_MAX	7
 #define CFQIO_SS_IOPRIO_MIN	0

@@ -21,6 +25,31 @@
 struct cfqio_ss_css {
 	struct cgroup_subsys_state	css;
 	unsigned int			ioprio;
+	struct list_head		cfqio_cgrp_head;
+	/* this lock protects the list above */
+	rwlock_t			cfqio_cgrp_lock;
+	/* list of all such objects, anchored at cfqio_ss_css_list */
+	struct list_head		list;
 };
 
+#ifdef CONFIG_CGROUP_CFQIO
+extern struct cfqio_cgroup_data *
+cfqio_cgrp_findcreate(struct cfqio_ss_css *, struct cfq_data *, gfp_t gfp_mask);
+extern void cfqio_ss_exit_queue(struct cfq_data *);
+#else
+static inline struct cfqio_cgroup_data *
+cfqio_cgrp_findcreate(struct cfqio_ss_css *cfqio_ss,
+			struct cfq_data *cfqd, gfp_t gfp_mask)
+{
+	return &cfqd->cfqio_cgroup;
+}
+
+extern void cfqio_ss_exit_queue(struct cfq_data *cfqd) { ; }
+#endif /* CONFIG_CGROUP_CFQIO */
+
+static inline void cfqio_init_cfqio_cgroup(struct cfqio_cgroup_data *cfqio_cgrp)
+{
+	cfqio_cgrp->service_tree = CFQ_RB_ROOT;
+}
+
 #endif /* _LINUX_CFQIO_CGROUP_H */

81 lines not on this page · raw message

127 lines not on this page · raw message

@@ -36,6 +36,12 @@ struct cfqio_ss_css {
 extern struct cfqio_cgroup_data *
 cfqio_cgrp_findcreate(struct cfqio_ss_css *, struct cfq_data *, gfp_t gfp_mask);
 extern void cfqio_ss_exit_queue(struct cfq_data *);
+
+static inline struct cfqio_ss_css *cfqio_css_by_tsk(struct task_struct *tsk)
+{
+	return container_of(tsk->cgroups->subsys[cfqio_subsys_id],
+						struct cfqio_ss_css, css);
+}
 #else
 static inline struct cfqio_cgroup_data *
 cfqio_cgrp_findcreate(struct cfqio_ss_css *cfqio_ss,

@@ -45,6 +51,11 @@ cfqio_cgrp_findcreate(struct cfqio_ss_cs
 }
 
 extern void cfqio_ss_exit_queue(struct cfq_data *cfqd) { ; }
+
+static inline struct cfqio_ss_css *cfqio_css_by_tsk(struct task_struct *tsk)
+{
+	return NULL;
+}
 #endif /* CONFIG_CGROUP_CFQIO */
 
 static inline void cfqio_init_cfqio_cgroup(struct cfqio_cgroup_data *cfqio_cgrp)

@@ -12,6 +12,8 @@
 #include <linux/rbtree.h>
 #include <linux/ioprio.h>
 #include <linux/cfq-iosched.h>
+#include <linux/cgroup.h>
+#include <linux/cfqio-cgroup.h>
 
 /*
  * tunables

@@ -1256,11 +1258,14 @@ cfq_find_alloc_queue(struct cfq_data *cf
 {
 	struct cfq_queue *cfqq, *new_cfqq = NULL;
 	struct cfq_io_context *cic;
+	struct cfqio_cgroup_data *cfqio_cgrp = NULL;
+	struct cfqio_ss_css *cfqio_css;
 
 retry:
 	cic = cfq_cic_lookup(cfqd, ioc);
 	/* cic always exists here */
 	cfqq = cic_to_cfqq(cic, is_sync);
+	cfqio_css = cfqio_css_by_tsk(current);
 
 	if (!cfqq) {
 		if (new_cfqq) {

@@ -1277,6 +1282,14 @@ retry:
 			new_cfqq = kmem_cache_alloc_node(cfq_pool,
 					gfp_mask | __GFP_NOFAIL | __GFP_ZERO,
 					cfqd->queue->node);
+			if (new_cfqq) {
+				cfqio_cgrp = cfqio_cgrp_findcreate(cfqio_css,
+								cfqd, gfp_mask);
+				if (!cfqio_cgrp) {
+					kmem_cache_free(cfq_pool, new_cfqq);
+					new_cfqq = NULL;
+				}
+			}
 			spin_lock_irq(cfqd->queue->queue_lock);
 			goto retry;
 		} else {

@@ -1285,6 +1298,13 @@ retry:
 					cfqd->queue->node);
 			if (!cfqq)
 				goto out;
+
+			cfqio_cgrp = cfqio_cgrp_findcreate(cfqio_css,
+							cfqd, gfp_mask);
+			if (!cfqio_cgrp) {
+				kmem_cache_free(cfq_pool, cfqq);
+				cfqq = NULL;
+			}
 		}
 
 		RB_CLEAR_NODE(&cfqq->rb_node);

@@ -1298,6 +1318,8 @@ retry:
 
 		cfq_init_prio_data(cfqq, ioc);
 
+		cfqq->cfqio_cgrp = cfqio_cgrp;
+
 		if (is_sync) {
 			if (!cfq_class_idle(cfqq))
 				cfq_mark_cfqq_idle_window(cfqq);

@@ -1990,6 +2012,8 @@ static void cfq_exit_queue(elevator_t *e
 
 	cfq_shutdown_timer_wq(cfqd);
 
+	cfqio_ss_exit_queue(cfqd);
+
 	kfree(cfqd);
 }
 

@@ -2002,6 +2026,9 @@ static void *cfq_init_queue(struct reque
 		return NULL;
 
 	cfqd->service_tree = CFQ_RB_ROOT;
+#ifndef CONFIG_CGROUP_CFQIO
+	cfqio_init_cfqio_cgroup(&cfqd->cfqio_cgroup);
+#endif
 	INIT_LIST_HEAD(&cfqd->cic_list);
 
 	cfqd->queue = q;

@@ -2022,6 +2049,7 @@ static void *cfq_init_queue(struct reque
 	cfqd->cfq_slice[1] = cfq_slice_sync;
 	cfqd->cfq_slice_async_rq = cfq_slice_async_rq;
 	cfqd->cfq_slice_idle = cfq_slice_idle;
+	INIT_LIST_HEAD(&cfqd->act_cfqio_cgrp_head);
 
 	return cfqd;
 }

@@ -70,9 +70,8 @@ struct cfq_data {
 	struct cfqio_cgroup_data cfqio_cgroup;
 #endif
 	/*
-	 * rr list of queues with requests and the count of them
+	 * count of queues
 	 */
-	struct cfq_rb_root service_tree;
 	unsigned int busy_queues;
 
 	int rq_in_driver;

113 lines not on this page · raw message

@@ -42,6 +42,10 @@ static inline struct cfqio_ss_css *cfqio
 	return container_of(tsk->cgroups->subsys[cfqio_subsys_id],
 						struct cfqio_ss_css, css);
 }
+extern int cfqio_cgrp_expired(struct cfq_data *);
+extern void cfqio_cgrp_schedule_active(struct cfq_data *);
+extern void  cfqio_cgrp_inc_rqnum(struct cfq_queue *);
+extern void cfqio_cgrp_dec_rqnum(struct cfq_queue *);
 #else
 static inline struct cfqio_cgroup_data *
 cfqio_cgrp_findcreate(struct cfqio_ss_css *cfqio_ss,

@@ -56,6 +60,16 @@ static inline struct cfqio_ss_css *cfqio
 {
 	return NULL;
 }
+
+static inline int cfqio_cgrp_expired(struct cfq_data *cfqd) { return 0; }
+
+static inline void cfqio_cgrp_schedule_active(struct cfq_data *cfqd)
+{
+	cfqd->active_cfqio_cgroup = &cfqd->cfqio_cgroup;
+}
+
+static inline void cfqio_cgrp_inc_rqnum(struct cfq_queue *cfqq) { ; }
+static inline void cfqio_cgrp_dec_rqnum(struct cfq_queue *cfqq) { ; }
 #endif /* CONFIG_CGROUP_CFQIO */
 
 static inline void cfqio_init_cfqio_cgroup(struct cfqio_cgroup_data *cfqio_cgrp)

@@ -56,6 +56,7 @@ struct cfqio_cgroup_data {
 	struct cfqio_ss_css	*cfqio_css;
 	/* rr list of queues with requests */
 	struct cfq_rb_root	service_tree;
+	unsigned long		rqnum;
 };
 
 /*

@@ -113,6 +114,8 @@ struct cfq_data {
 	struct list_head act_cfqio_cgrp_head;
 	/* cgroup that owns a timeslice at the moment */
 	struct cfqio_cgroup_data *active_cfqio_cgroup;
+	unsigned int cfqio_cgrp_slice;
+	unsigned long cfqio_slice_end;
 };
 
 /*

65 lines not on this page · raw message

@@ -29,6 +29,7 @@ static const int cfq_slice_sync = HZ / 1
 static int cfq_slice_async = HZ / 25;
 static const int cfq_slice_async_rq = 2;
 static int cfq_slice_idle = HZ / 125;
+static int cfqio_cgrp_slice = HZ / 2;
 
 /*
  * offset from end of service tree

@@ -185,6 +186,8 @@ static inline int cfq_slice_used(struct 
 {
 	if (cfq_cfqq_slice_new(cfqq))
 		return 0;
+	if (cfqio_cgrp_expired(cfqq->cfqd))
+		return 1;
 	if (time_before(jiffies, cfqq->slice_end))
 		return 0;
 

@@ -447,6 +450,7 @@ static void cfq_add_cfqq_rr(struct cfq_d
 	BUG_ON(cfq_cfqq_on_rr(cfqq));
 	cfq_mark_cfqq_on_rr(cfqq);
 	cfqd->busy_queues++;
+	cfqio_cgrp_inc_rqnum(cfqq);
 
 	cfq_resort_rr_list(cfqd, cfqq);
 }

@@ -466,6 +470,7 @@ static void cfq_del_cfqq_rr(struct cfq_d
 
 	BUG_ON(!cfqd->busy_queues);
 	cfqd->busy_queues--;
+	cfqio_cgrp_dec_rqnum(cfqq);
 }
 
 /*

@@ -708,6 +713,8 @@ static struct cfq_queue *cfq_get_next_qu
 {
 	struct cfqio_cgroup_data *cfqio_cgrp;
 
+	cfqio_cgrp_schedule_active(cfqd);
+
 	cfqio_cgrp = cfqd->active_cfqio_cgroup;
 	if (!cfqio_cgrp)
 		return NULL;

@@ -2076,6 +2083,7 @@ static void *cfq_init_queue(struct reque
 	cfqd->cfq_slice_async_rq = cfq_slice_async_rq;
 	cfqd->cfq_slice_idle = cfq_slice_idle;
 	INIT_LIST_HEAD(&cfqd->act_cfqio_cgrp_head);
+	cfqd->cfqio_cgrp_slice = cfqio_cgrp_slice;
 
 	return cfqd;
 }

@@ -57,6 +57,7 @@ struct cfqio_cgroup_data {
 	/* rr list of queues with requests */
 	struct cfq_rb_root	service_tree;
 	unsigned long		rqnum;
+	unsigned long		on_dispatch;
 };
 
 /*

@@ -823,6 +823,7 @@ static void cfq_dispatch_insert(struct r
 
 	cfq_remove_request(rq);
 	cfqq->dispatched++;
+	cfqq->cfqio_cgrp->on_dispatch++;
 	elv_dispatch_sort(q, rq);
 
 	if (cfq_cfqq_sync(cfqq))

@@ -1775,6 +1776,7 @@ static void cfq_completed_request(struct
 	WARN_ON(!cfqq->dispatched);
 	cfqd->rq_in_driver--;
 	cfqq->dispatched--;
+	cfqq->cfqio_cgrp->on_dispatch--;
 
 	if (cfq_cfqq_sync(cfqq))
 		cfqd->sync_flight--;

@@ -54,7 +54,8 @@ static inline void set_active_cgrp(struc
 void cfqio_cgrp_schedule_active(struct cfq_data *cfqd)
 {
 	if (cfqio_cgrp_expired(cfqd) || !cfqd->active_cfqio_cgroup ||
-				!cfqd->active_cfqio_cgroup->rqnum)
+		(!cfqd->active_cfqio_cgroup->rqnum &&
+			!cfqd->active_cfqio_cgroup->on_dispatch))
 		set_active_cgrp(cfqd);
 }
 

@@ -2150,6 +2150,7 @@ SHOW_FUNCTION(cfq_slice_idle_show, cfqd-
 SHOW_FUNCTION(cfq_slice_sync_show, cfqd->cfq_slice[1], 1);
 SHOW_FUNCTION(cfq_slice_async_show, cfqd->cfq_slice[0], 1);
 SHOW_FUNCTION(cfq_slice_async_rq_show, cfqd->cfq_slice_async_rq, 0);
+SHOW_FUNCTION(cfq_cgrp_slice_show, cfqd->cfqio_cgrp_slice, 1);
 #undef SHOW_FUNCTION
 
 #define STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, __CONV)			\

@@ -2181,6 +2182,7 @@ STORE_FUNCTION(cfq_slice_sync_store, &cf
 STORE_FUNCTION(cfq_slice_async_store, &cfqd->cfq_slice[0], 1, UINT_MAX, 1);
 STORE_FUNCTION(cfq_slice_async_rq_store, &cfqd->cfq_slice_async_rq, 1,
 		UINT_MAX, 0);
+STORE_FUNCTION(cfq_cgrp_slice_store, &cfqd->cfqio_cgrp_slice, 1, UINT_MAX, 1);
 #undef STORE_FUNCTION
 
 #define CFQ_ATTR(name) \

@@ -2196,6 +2198,7 @@ static struct elv_fs_entry cfq_attrs[] =
 	CFQ_ATTR(slice_async),
 	CFQ_ATTR(slice_async_rq),
 	CFQ_ATTR(slice_idle),
+	CFQ_ATTR(cgrp_slice),
 	__ATTR_NULL
 };