Skip to contentSkip to conversation

fio: support io_size=N% (N <= 100)

PATCH Applied Alexey Dobriyan 3 participants 5 messages Sep 3, 2020 – Sep 7, 2020

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 fio: support io_size=N% (N <= 100)

Parse "io_size=N%".

Semantics is "multiply whatever size= calculations result in".

Example #1:

	size=50%
	io_size=50%

will do 25% of a file.

Example #2:

	size=1G
	io_size=50%

will do 512M I/O.

As side effect, fix a bug with essentially infinite loop if both size=N% and io_size=M% are given: io_size is set to 2^64-... in this case (a lot!).

Note: only values under 100% work currently.
Going for io_size=150% requires resetting workload generator state which is whole separate endeavour.

---

 cconv.c          |    2 ++
 filesetup.c      |   11 ++++++++++-
 options.c        |   17 +++++++++++++++++
 server.h         |    2 +-
 thread_options.h |    4 ++++
 5 files changed, 34 insertions(+), 2 deletions(-)
cconv.c +2 -0
@@ -102,6 +102,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
 	o->size = le64_to_cpu(top->size);
 	o->io_size = le64_to_cpu(top->io_size);
 	o->size_percent = le32_to_cpu(top->size_percent);
+	o->io_size_percent = le32_to_cpu(top->io_size_percent);
 	o->fill_device = le32_to_cpu(top->fill_device);
 	o->file_append = le32_to_cpu(top->file_append);
 	o->file_size_low = le64_to_cpu(top->file_size_low);
@@ -367,6 +368,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->iodepth_batch_complete_max = cpu_to_le32(o->iodepth_batch_complete_max);
 	top->serialize_overlap = cpu_to_le32(o->serialize_overlap);
 	top->size_percent = cpu_to_le32(o->size_percent);
+	top->io_size_percent = cpu_to_le32(o->io_size_percent);
 	top->fill_device = cpu_to_le32(o->fill_device);
 	top->file_append = cpu_to_le32(o->file_append);
 	top->ratecycle = cpu_to_le32(o->ratecycle);
filesetup.c +10 -1
@@ -1139,6 +1139,8 @@ int setup_files(struct thread_data *td)
 		if (f->io_size == -1ULL)
 			total_size = -1ULL;
 		else {
+			uint64_t io_size;
+
                         if (o->size_percent && o->size_percent != 100) {
 				uint64_t file_size;
 
@@ -1150,7 +1152,14 @@ int setup_files(struct thread_data *td)
 
 				f->io_size -= (f->io_size % td_min_bs(td));
 			}
-			total_size += f->io_size;
+
+			io_size = f->io_size;
+			if (o->io_size_percent && o->io_size_percent != 100) {
+				io_size *= o->io_size_percent;
+				io_size /= 100;
+			}
+
+			total_size += io_size;
 		}
 
 		if (f->filetype == FIO_TYPE_FILE &&
options.c +17 -0
@@ -1476,6 +1476,22 @@ static int str_size_cb(void *data, unsigned long long *__val)
 	return 0;
 }
 
+static int str_io_size_cb(void *data, unsigned long long *__val)
+{
+	struct thread_data *td = cb_data_to_td(data);
+	unsigned long long v = *__val;
+
+	if (parse_is_percent(v)) {
+		td->o.io_size = 0;
+		td->o.io_size_percent = -1ULL - v;
+		dprint(FD_PARSE, "SET io_size_percent %d\n",
+					td->o.io_size_percent);
+	} else
+		td->o.io_size = v;
+
+	return 0;
+}
+
 static int str_write_bw_log_cb(void *data, const char *str)
 {
 	struct thread_data *td = cb_data_to_td(data);
@@ -2043,6 +2059,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.alias	= "io_limit",
 		.lname	= "IO Size",
 		.type	= FIO_OPT_STR_VAL,
+		.cb	= str_io_size_cb,
 		.off1	= offsetof(struct thread_options, io_size),
 		.help	= "Total size of I/O to be performed",
 		.interval = 1024 * 1024,
server.h +1 -1
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 85,
+	FIO_SERVER_VER			= 86,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
thread_options.h +4 -0
@@ -83,6 +83,7 @@ struct thread_options {
 	unsigned long long size;
 	unsigned long long io_size;
 	unsigned int size_percent;
+	unsigned int io_size_percent;
 	unsigned int fill_device;
 	unsigned int file_append;
 	unsigned long long file_size_low;
@@ -381,6 +382,7 @@ struct thread_options_pack {
 	uint64_t size;
 	uint64_t io_size;
 	uint32_t size_percent;
+	uint32_t io_size_percent;
 	uint32_t fill_device;
 	uint32_t file_append;
 	uint32_t unique_filename;
@@ -460,6 +462,8 @@ struct thread_options_pack {
 	struct zone_split zone_split[DDIR_RWDIR_CNT][ZONESPLIT_MAX];
 	uint32_t zone_split_nr[DDIR_RWDIR_CNT];
 
+	uint8_t pad1[4];
+
 	fio_fp64_t zipf_theta;
 	fio_fp64_t pareto_h;
 	fio_fp64_t gauss_dev;
#1 lore raw
Quoted 17 lines · Alexey Dobriyan

Parse "io_size=N%".

Semantics is "multiply whatever size= calculations result in".

Example #1:

        size=50%
        io_size=50%

will do 25% of a file.

Example #2:

        size=1G
        io_size=50%
Quoted 8 lines

will do 512M I/O.

As side effect, fix a bug with essentially infinite loop if both size=N% and io_size=M% are given: io_size is set to 2^64-... in this case (a lot!).

Note: only values under 100% work currently.
Going for io_size=150% requires resetting workload generator state which is whole separate endeavour.

It might be worth adding some error output if someone does go for more more than 100% otherwise people will think
https://github.com/axboe/fio/issues/419 has been fixed...

Quoted 127 lines

---

 cconv.c          |    2 ++
 filesetup.c      |   11 ++++++++++-
 options.c        |   17 +++++++++++++++++
 server.h         |    2 +-
 thread_options.h |    4 ++++
 5 files changed, 34 insertions(+), 2 deletions(-)
cconv.c +2 -0
@@ -102,6 +102,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->size = le64_to_cpu(top->size);
        o->io_size = le64_to_cpu(top->io_size);
        o->size_percent = le32_to_cpu(top->size_percent);
+       o->io_size_percent = le32_to_cpu(top->io_size_percent);
        o->fill_device = le32_to_cpu(top->fill_device);
        o->file_append = le32_to_cpu(top->file_append);
        o->file_size_low = le64_to_cpu(top->file_size_low);
@@ -367,6 +368,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->iodepth_batch_complete_max = cpu_to_le32(o->iodepth_batch_complete_max);
        top->serialize_overlap = cpu_to_le32(o->serialize_overlap);
        top->size_percent = cpu_to_le32(o->size_percent);
+       top->io_size_percent = cpu_to_le32(o->io_size_percent);
        top->fill_device = cpu_to_le32(o->fill_device);
        top->file_append = cpu_to_le32(o->file_append);
        top->ratecycle = cpu_to_le32(o->ratecycle);
filesetup.c +10 -1
@@ -1139,6 +1139,8 @@ int setup_files(struct thread_data *td)
                if (f->io_size == -1ULL)
                        total_size = -1ULL;
                else {
+                       uint64_t io_size;
+
                         if (o->size_percent && o->size_percent != 100) {
                                uint64_t file_size;
 
@@ -1150,7 +1152,14 @@ int setup_files(struct thread_data *td)
 
                                f->io_size -= (f->io_size % td_min_bs(td));
                        }
-                       total_size += f->io_size;
+
+                       io_size = f->io_size;
+                       if (o->io_size_percent && o->io_size_percent != 100) {
+                               io_size *= o->io_size_percent;
+                               io_size /= 100;
+                       }
+
+                       total_size += io_size;
                }
 
                if (f->filetype == FIO_TYPE_FILE &&
options.c +17 -0
@@ -1476,6 +1476,22 @@ static int str_size_cb(void *data, unsigned long long *__val)
        return 0;
 }
 
+static int str_io_size_cb(void *data, unsigned long long *__val)
+{
+       struct thread_data *td = cb_data_to_td(data);
+       unsigned long long v = *__val;
+
+       if (parse_is_percent(v)) {
+               td->o.io_size = 0;
+               td->o.io_size_percent = -1ULL - v;
+               dprint(FD_PARSE, "SET io_size_percent %d\n",
+                                       td->o.io_size_percent);
+       } else
+               td->o.io_size = v;
+
+       return 0;
+}
+
 static int str_write_bw_log_cb(void *data, const char *str)
 {
        struct thread_data *td = cb_data_to_td(data);
@@ -2043,6 +2059,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .alias  = "io_limit",
                .lname  = "IO Size",
                .type   = FIO_OPT_STR_VAL,
+               .cb     = str_io_size_cb,
                .off1   = offsetof(struct thread_options, io_size),
                .help   = "Total size of I/O to be performed",
                .interval = 1024 * 1024,
server.h +1 -1
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-       FIO_SERVER_VER                  = 85,
+       FIO_SERVER_VER                  = 86,
 
        FIO_SERVER_MAX_FRAGMENT_PDU     = 1024,
        FIO_SERVER_MAX_CMD_MB           = 2048,
thread_options.h +4 -0
@@ -83,6 +83,7 @@ struct thread_options {
        unsigned long long size;
        unsigned long long io_size;
        unsigned int size_percent;
+       unsigned int io_size_percent;
        unsigned int fill_device;
        unsigned int file_append;
        unsigned long long file_size_low;
@@ -381,6 +382,7 @@ struct thread_options_pack {
        uint64_t size;
        uint64_t io_size;
        uint32_t size_percent;
+       uint32_t io_size_percent;
        uint32_t fill_device;
        uint32_t file_append;
        uint32_t unique_filename;
@@ -460,6 +462,8 @@ struct thread_options_pack {
        struct zone_split zone_split[DDIR_RWDIR_CNT][ZONESPLIT_MAX];
        uint32_t zone_split_nr[DDIR_RWDIR_CNT];
 
+       uint8_t pad1[4];
+
        fio_fp64_t zipf_theta;
        fio_fp64_t pareto_h;
        fio_fp64_t gauss_dev;
Signature · 1 line
Sitsofe | http://sucs.org/~sits/
#2 lore raw

PATCH 1/1 fio: cap io_size=N% at 100%, update man page

Passing io_size > 100% results in semi-infinite loop, anyway.

---

 fio.1     |    3 ++-
 options.c |    6 +++++-
 parse.h   |    5 +++++
 3 files changed, 12 insertions(+), 2 deletions(-)
fio.1 +2 -1
@@ -1561,7 +1561,8 @@ if \fBsize\fR is set to 20GiB and \fBio_size\fR is set to 5GiB, fio
 will perform I/O within the first 20GiB but exit when 5GiB have been
 done. The opposite is also possible \-\- if \fBsize\fR is set to 20GiB,
 and \fBio_size\fR is set to 40GiB, then fio will do 40GiB of I/O within
-the 0..20GiB region.
+the 0..20GiB region. Value can be set as percentage: \fBio_size\fR=N%.
+In this case \fBio_size\fR multiplies \fBsize\fR= value.
 .TP
 .BI filesize \fR=\fPirange(int)
 Individual file sizes. May be a range, in which case fio will select sizes
options.c +5 -1
@@ -1481,9 +1481,13 @@ static int str_io_size_cb(void *data, unsigned long long *__val)
 	struct thread_data *td = cb_data_to_td(data);
 	unsigned long long v = *__val;
 
-	if (parse_is_percent(v)) {
+	if (parse_is_percent_uncapped(v)) {
 		td->o.io_size = 0;
 		td->o.io_size_percent = -1ULL - v;
+		if (td->o.io_size_percent > 100) {
+			log_err("fio: io_size values greater than 100%% aren't supported\n");
+			return 1;
+		}
 		dprint(FD_PARSE, "SET io_size_percent %d\n",
 					td->o.io_size_percent);
 	} else
parse.h +5 -0
@@ -133,6 +133,11 @@ static inline int parse_is_percent(unsigned long long val)
 	return val <= -1ULL && val >= (-1ULL - 100ULL);
 }
 
+static inline int parse_is_percent_uncapped(unsigned long long val)
+{
+	return (long long)val <= -1;
+}
+
 struct print_option {
 	struct flist_head list;
 	char *name;
Applied
#3 lore raw

Re: [PATCH] fio: cap io_size=N% at 100%, update man page

Quoted 1 line · Alexey Dobriyan

Passing io_size > 100% results in semi-infinite loop, anyway.

Applied, thanks.

Signature · 1 line
Jens Axboe
Applied
#4 lore raw
In reply to Alexey Dobriyan
Quoted 16 lines · Alexey Dobriyan

Parse "io_size=N%".

Semantics is "multiply whatever size= calculations result in".

Example #1:

	size=50%
	io_size=50%

will do 25% of a file.

Example #2:

	size=1G
	io_size=50%
Quoted 8 lines

will do 512M I/O.

As side effect, fix a bug with essentially infinite loop if both size=N% and io_size=M% are given: io_size is set to 2^64-... in this case (a lot!).

Note: only values under 100% work currently.
Going for io_size=150% requires resetting workload generator state which is whole separate endeavour.

Applied, thanks.

Signature · 1 line
Jens Axboe