cgroup: Support cgroup-v2 weight and BFQ weight

PATCH RESEND In review Andreas Herrmann 2 participants 6 messages Oct 30, 2020 – Nov 22, 2020

Share & tools

Andreas Herrmann

author

2020-10-30 11:11 UTC #0 lore raw

v3 cover cgroup: Support cgroup-v2 weight and BFQ weight

These are changes to support weight with cgroup-v2, and also BFQ's weight attribute. In recent kernels with cgroup v1 only blkio.bfq.weight is available to set weight.

v3: - rebased on fio-3.23-28-g7064f894

      (now rebased on fio-3.23-48-ga0b72421 w/o changes)

v2: - bumped FIO_SERVER_VER
- regrouped patches

Andreas Herrmann (4):
cgroup: Fix handling when cgroup blkio and cgroup-v2 are mounted
cgroup: Allow to use weights with cgroup-v2
thread_options: Introduce cgroup_use_bfq
cgroup: Support BFQ's weight attributes

 HOWTO            |   6 +++
 cconv.c          |   2 +
 cgroup.c         | 110 ++++++++++++++++++++++++++++++++++++-----------
 fio.1            |   5 +++
 options.c        |  11 +++++
 server.h         |   2 +-
 thread_options.h |   7 ++-
 7 files changed, 116 insertions(+), 27 deletions(-)

Signature · 1 line

2.29.0

Andreas Herrmann

author

2020-10-30 11:11 UTC #1 lore raw

PATCH RESEND v3 1/4 cgroup: Fix handling when cgroup blkio and cgroup-v2 are mounted

On systems that have a mixed setup of cgroup and cgroup-v2, blkio controller might be active. I think in this case there is no point in using cgroup-v2. One needs to boot with cgroup_no_v1=blkio to be able to use cgroup-v2 IO controller.

Signed-off-by Andreas Herrmann <[email protected]>

---
 cgroup.c | 58 ++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 18 deletions(-)

cgroup.c +40 ~~-18~~

index 77e31a4d..fc31d4cd 100644
@@ -20,11 +20,10 @@ struct cgroup_member {
 
 static struct cgroup_mnt *find_cgroup_mnt(struct thread_data *td)
 {
-	struct cgroup_mnt *cgroup_mnt = NULL;
+	struct cgroup_mnt *cgroup_mnt, *cg1_mnt = NULL, *cg2_mnt = NULL;
 	struct mntent *mnt, dummy;
 	char buf[256] = {0};
 	FILE *f;
-	bool cgroup2 = false;
 
 	f = setmntent("/proc/mounts", "r");
 	if (!f) {

@@ -34,27 +33,50 @@ static struct cgroup_mnt *find_cgroup_mnt(struct thread_data *td)
 
 	while ((mnt = getmntent_r(f, &dummy, buf, sizeof(buf))) != NULL) {
 		if (!strcmp(mnt->mnt_type, "cgroup") &&
-		    strstr(mnt->mnt_opts, "blkio"))
-			break;
-		if (!strcmp(mnt->mnt_type, "cgroup2")) {
-			cgroup2 = true;
+			strstr(mnt->mnt_opts, "blkio")) {
+			cg1_mnt = smalloc(sizeof(*cg1_mnt));
+			if (cg1_mnt) {
+				cg1_mnt->path = smalloc_strdup(mnt->mnt_dir);
+				if (!cg1_mnt->path) {
+					sfree(cg1_mnt);
+					cg1_mnt = NULL;
+					log_err("fio: could not allocate memory\n");
+				}
+			}
+			/*
+			 * blkio controller has precedence
+			 */
 			break;
+		} if (!strcmp(mnt->mnt_type, "cgroup2")) {
+			if (cg2_mnt)
+				continue;
+			cg2_mnt = smalloc(sizeof(*cg2_mnt));
+			if (cg2_mnt) {
+				cg2_mnt->path = smalloc_strdup(mnt->mnt_dir);
+				if (!cg2_mnt->path) {
+					sfree(cg2_mnt);
+					cg2_mnt = NULL;
+					log_err("fio: could not allocate memory\n");
+				} else
+					cg2_mnt->cgroup2 = true;
+			}
+			/*
+			 * Check all mount points, there can be a mix
+			 * of cgroup-v1 and cgroup-v2
+			 */
+			continue;
 		}
 	}
 
-	if (mnt) {
-		cgroup_mnt = smalloc(sizeof(*cgroup_mnt));
-		if (cgroup_mnt) {
-			cgroup_mnt->path = smalloc_strdup(mnt->mnt_dir);
-			if (!cgroup_mnt->path) {
-				sfree(cgroup_mnt);
-				log_err("fio: could not allocate memory\n");
-			} else {
-				cgroup_mnt->cgroup2 = cgroup2;
-			}
-		}
+	cgroup_mnt = NULL;
+	if (cg1_mnt) {
+		cgroup_mnt = cg1_mnt;
+		if (cg2_mnt)
+			sfree(cg2_mnt);
+	} else if (cg2_mnt) {
+		cgroup_mnt = cg2_mnt;
 	} else {
-		log_err("fio: cgroup blkio does not appear to be mounted\n");
+		log_err("fio: cgroup blkio or cgroup2 do not appear to be mounted\n");
 	}
 
 	endmntent(f);

Signature · 1 line

2.29.0

Andreas Herrmann

author

2020-10-30 11:11 UTC #2 lore raw

PATCH RESEND v3 2/4 cgroup: Allow to use weights with cgroup-v2

In reply to Andreas Herrmann

Signed-off-by Andreas Herrmann <[email protected]>

---
 cgroup.c | 38 ++++++++++++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 6 deletions(-)

cgroup.c +32 -6

index fc31d4cd..c6369f54 100644
@@ -165,6 +165,26 @@ static int write_int_to_file(struct thread_data *td, const char *path,
 
 }
 
+static int write_str_to_file(struct thread_data *td, const char *path,
+			     const char *filename, const char *str,
+			     const char *onerr)
+{
+	char tmp[256];
+	FILE *f;
+
+	sprintf(tmp, "%s/%s", path, filename);
+	f = fopen(tmp, "w");
+	if (!f) {
+		td_verror(td, errno, onerr);
+		return 1;
+	}
+
+	fprintf(f, "%s", str);
+	fclose(f);
+	return 0;
+
+}
+
 static int cgroup_write_pid(struct thread_data *td, char *path, bool cgroup2)
 {
 	unsigned int val = td->pid;

@@ -213,13 +233,19 @@ int cgroup_setup(struct thread_data *td, struct flist_head *clist, struct cgroup
 
 	if (td->o.cgroup_weight) {
 		if ((*mnt)->cgroup2) {
-			log_err("fio: cgroup weit doesn't work with cgroup2\n");
-			goto err;
+			if (write_str_to_file(td, (*mnt)->path, "cgroup.subtree_control",
+						"+io", "cgroup open cgroup.subtree_control"))
+				goto err;
+			if (write_int_to_file(td, root, "io.weight",
+						td->o.cgroup_weight,
+						"cgroup open io.weight"))
+				goto err;
+		} else {
+			if (write_int_to_file(td, root, "blkio.weight",
+						td->o.cgroup_weight,
+						"cgroup open blkio.weight"))
+				goto err;
 		}
-		if (write_int_to_file(td, root, "blkio.weight",
-					td->o.cgroup_weight,
-					"cgroup open weight"))
-			goto err;
 	}
 
 	if (!cgroup_write_pid(td, root, (*mnt)->cgroup2)) {

Signature · 1 line

2.29.0

Andreas Herrmann

author

2020-10-30 11:11 UTC #3 lore raw

PATCH RESEND v3 3/4 thread_options: Introduce cgroup_use_bfq

In reply to Andreas Herrmann

Signed-off-by Andreas Herrmann <[email protected]>

---
 HOWTO            |  6 ++++++
 cconv.c          |  2 ++
 fio.1            |  5 +++++
 options.c        | 11 +++++++++++
 server.h         |  2 +-
 thread_options.h |  7 +++++--
 6 files changed, 30 insertions(+), 3 deletions(-)

HOWTO +6 -0

index 386fd12a..2e94d793 100644
@@ -2857,6 +2857,12 @@ Threads, processes and job synchronization
 	job completion, set ``cgroup_nodelete=1``.  This can be useful if one wants
 	to inspect various cgroup files after job completion. Default: false.
 
+.. option:: cgroup_use_bfq=bool
+
+	Normally fio will use generic sysfs attributes to set cgroup_weight.
+	To use attributes specific to BFQ IO scheduler set ``cgroup_use_bfq=1``.
+	Default: false.
+
 .. option:: flow_id=int
 
 	The ID of the flow. If not specified, it defaults to being a global

cconv.c +2 -0

index 488dd799..ef5b6a97 100644
@@ -279,6 +279,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
 	o->continue_on_error = le32_to_cpu(top->continue_on_error);
 	o->cgroup_weight = le32_to_cpu(top->cgroup_weight);
 	o->cgroup_nodelete = le32_to_cpu(top->cgroup_nodelete);
+	o->cgroup_use_bfq = le32_to_cpu(top->cgroup_use_bfq);
 	o->uid = le32_to_cpu(top->uid);
 	o->gid = le32_to_cpu(top->gid);
 	o->flow_id = __le32_to_cpu(top->flow_id);

@@ -479,6 +480,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->continue_on_error = cpu_to_le32(o->continue_on_error);
 	top->cgroup_weight = cpu_to_le32(o->cgroup_weight);
 	top->cgroup_nodelete = cpu_to_le32(o->cgroup_nodelete);
+	top->cgroup_use_bfq = cpu_to_le32(o->cgroup_use_bfq);
 	top->uid = cpu_to_le32(o->uid);
 	top->gid = cpu_to_le32(o->gid);
 	top->flow_id = __cpu_to_le32(o->flow_id);

fio.1 +5 -0

index 48119325..39b40772 100644
@@ -2562,6 +2562,11 @@ completion. To override this behavior and to leave cgroups around after the
 job completion, set `cgroup_nodelete=1'. This can be useful if one wants
 to inspect various cgroup files after job completion. Default: false.
 .TP
+.BI cgroup_use_bfq \fR=\fPbool
+Normally fio will use generic sysfs attributes to set cgroup_weight.
+To use attributes specific to BFQ IO scheduler set ``cgroup_use_bfq=1``.
+Default: false.
+.TP
 .BI flow_id \fR=\fPint
 The ID of the flow. If not specified, it defaults to being a global
 flow. See \fBflow\fR.

options.c +11 -0

index 1e91b3e9..8136f262 100644
@@ -4643,6 +4643,17 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_GENERAL,
 		.group	= FIO_OPT_G_CGROUP,
 	},
+	{
+		.name	= "cgroup_use_bfq",
+		.lname	= "Cgroup use-bfq",
+		.type	= FIO_OPT_BOOL,
+		.off1	= offsetof(struct thread_options, cgroup_use_bfq),
+		.help	= "Use cgroup sysfs attributes specific to BFQ",
+		.def	= "0",
+		.parent	= "cgroup",
+		.category = FIO_OPT_C_GENERAL,
+		.group	= FIO_OPT_G_CGROUP,
+	},
 	{
 		.name	= "uid",
 		.lname	= "User ID",

server.h +1 -1

index 6d444749..9256d44c 100644
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 86,
+	FIO_SERVER_VER			= 87,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,

thread_options.h +5 -2

index 97c400fe..f4d296c6 100644
@@ -303,11 +303,12 @@ struct thread_options {
 	char *profile;
 
 	/*
-	 * blkio cgroup support
+	 * (blk)io cgroup support
 	 */
 	char *cgroup;
 	unsigned int cgroup_weight;
 	unsigned int cgroup_nodelete;
+	unsigned int cgroup_use_bfq;
 
 	unsigned int uid;
 	unsigned int gid;

@@ -599,11 +600,13 @@ struct thread_options_pack {
 	uint8_t profile[FIO_TOP_STR_MAX];
 
 	/*
-	 * blkio cgroup support
+	 * (blk)io cgroup support
 	 */
 	uint8_t cgroup[FIO_TOP_STR_MAX];
 	uint32_t cgroup_weight;
 	uint32_t cgroup_nodelete;
+	uint32_t cgroup_use_bfq;
+	uint32_t pad3;
 
 	uint32_t uid;
 	uint32_t gid;

Signature · 1 line

2.29.0

Andreas Herrmann

author

2020-10-30 11:11 UTC #4 lore raw

PATCH RESEND v3 4/4 cgroup: Support BFQ's weight attributes

In reply to Andreas Herrmann

Signed-off-by Andreas Herrmann <[email protected]>

---
 cgroup.c | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

cgroup.c +22 -8

index c6369f54..903c569e 100644
@@ -236,15 +236,29 @@ int cgroup_setup(struct thread_data *td, struct flist_head *clist, struct cgroup
 			if (write_str_to_file(td, (*mnt)->path, "cgroup.subtree_control",
 						"+io", "cgroup open cgroup.subtree_control"))
 				goto err;
-			if (write_int_to_file(td, root, "io.weight",
-						td->o.cgroup_weight,
-						"cgroup open io.weight"))
-				goto err;
+			if (td->o.cgroup_use_bfq) {
+				if (write_int_to_file(td, root, "io.bfq.weight",
+							td->o.cgroup_weight,
+							"cgroup open io.bfq.weight"))
+					goto err;
+			} else {
+				if (write_int_to_file(td, root, "io.weight",
+							td->o.cgroup_weight,
+							"cgroup open io.weight"))
+					goto err;
+			}
 		} else {
-			if (write_int_to_file(td, root, "blkio.weight",
-						td->o.cgroup_weight,
-						"cgroup open blkio.weight"))
-				goto err;
+			if (td->o.cgroup_use_bfq) {
+				if (write_int_to_file(td, root, "blkio.bfq.weight",
+							td->o.cgroup_weight,
+							"cgroup open blkio.bfq.weight"))
+					goto err;
+			} else {
+				if (write_int_to_file(td, root, "blkio.weight",
+							td->o.cgroup_weight,
+							"cgroup open blkio.weight"))
+					goto err;
+			}
 		}
 	}

Signature · 1 line

2.29.0

Sitsofe Wheeler

2020-11-22 14:52 UTC #5 lore raw

In reply to Andreas Herrmann

Hi Jens,

I've just looked over this series and it looks OK to me.

Quoted 26 lines · Andreas Herrmann

These are changes to support weight with cgroup-v2, and also BFQ's weight attribute. In recent kernels with cgroup v1 only blkio.bfq.weight is available to set weight.
v3: - rebased on fio-3.23-28-g7064f894
      (now rebased on fio-3.23-48-ga0b72421 w/o changes)
v2: - bumped FIO_SERVER_VER
- regrouped patches
Andreas Herrmann (4):
cgroup: Fix handling when cgroup blkio and cgroup-v2 are mounted
cgroup: Allow to use weights with cgroup-v2
thread_options: Introduce cgroup_use_bfq
cgroup: Support BFQ's weight attributes
 HOWTO            |   6 +++
 cconv.c          |   2 +
 cgroup.c         | 110 ++++++++++++++++++++++++++++++++++++-----------
 fio.1            |   5 +++
 options.c        |  11 +++++
 server.h         |   2 +-
 thread_options.h |   7 ++-
 7 files changed, 116 insertions(+), 27 deletions(-)

Quoted 3 lines

--
2.29.0

--
Sitsofe | http://sucs.org/~sits/