Skip to contentSkip to conversation

Add a new parameter and fix minimal rate calculation.

PATCH Superseded Hongwei Qin 2 participants 8 messages Jan 18, 2021 – Jan 20, 2021

v3 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

v3 cover Add a new parameter and fix minimal rate calculation.

The first patch adds a new parameter thinktime_blocks_type to control the behavior of thinktime_blocks. It can be either complete or issue.
If it is complete (default), fio triggers thinktime when thinktime_blocks number of blocks are **completed**.
If it is issue, fio triggers thinktime when thinktime_blocks number of blocks are **issued**

The second patch updates the compare time if handle_thinktime sleeps or spin.

I bring the test results from the first patch to here for your convenience:

Tests:
jobfile1:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
```

jobfile2:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
thinktime_blocks_type=issue
```

Results:
Current HEAD:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4002msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

This patch:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4001msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

fio jobfile2:
write: IOPS=1, BW=8191B/s (8191B/s)(98.3kB/12001msec); 0 zone resets

- issue 4 requests
- sleep 2s
***
- issue 4 requests
- sleep 2s
- finish

Hongwei Qin (2):
Add a new parameter.
Calculate min_rate with the consideration of thinktime

 HOWTO            |  7 +++++++
 backend.c        | 22 ++++++++++++++++------
 cconv.c          |  2 ++
 engines/cpu.c    |  1 +
 fio.h            |  5 +++++
 options.c        | 22 ++++++++++++++++++++++
 thread_options.h |  2 ++
 7 files changed, 55 insertions(+), 6 deletions(-)
Signature · 1 line
1.8.3.1
#1 lore raw

PATCH v3 1/2 Add a new parameter.

This patch adds a new parameter thinktime_blocks_type to control the behavior of thinktime_blocks. It can be either complete or issue.
If it is complete (default), fio triggers thinktime when thinktime_blocks number of blocks are **completed**. If it is issue, fio triggers thinktime when thinktime_blocks number of blocks are **issued**

Tests:
jobfile1:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
```

jobfile2:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
thinktime_blocks_type=issue
```

Results:
Current HEAD:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4002msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

This patch:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4001msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

fio jobfile2:
write: IOPS=1, BW=8191B/s (8191B/s)(98.3kB/12001msec); 0 zone resets

- issue 4 requests
- sleep 2s
***
- issue 4 requests
- sleep 2s
- finish

---
 HOWTO            |  7 +++++++
 backend.c        | 16 +++++++++++-----
 cconv.c          |  2 ++
 engines/cpu.c    |  1 +
 fio.h            |  5 +++++
 options.c        | 22 ++++++++++++++++++++++
 thread_options.h |  2 ++
 7 files changed, 50 insertions(+), 5 deletions(-)
HOWTO +7 -0
index 372f268..803bef4 100644
@@ -2562,6 +2562,13 @@ I/O rate
 	before we have to complete it and do our :option:`thinktime`. In other words, this
 	setting effectively caps the queue depth if the latter is larger.
 
+.. option:: thinktime_blocks_type=str
+
+	This option controls how thinktime_blocks triggers. The default is
+	`complete`, which triggers thinktime when fio completes thinktime_blocks
+	blocks. If this is set to `issue`, then the trigger happens at the
+	issue side.
+
 .. option:: rate=int[,int][,int]
 
 	Cap the bandwidth used by this job. The number is in bytes/sec, the normal
backend.c +11 -5
index e20a2e0..874e193 100644
@@ -864,8 +864,8 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
 	uint64_t total;
 	int left;
 
-	b = ddir_rw_sum(td->io_blocks);
-	if (b % td->o.thinktime_blocks)
+	b = ddir_rw_sum(td->thinktime_blocks_counter);
+	if (b % td->o.thinktime_blocks || !b)
 		return;
 
 	io_u_quiesce(td);
@@ -1076,6 +1076,10 @@ reap:
 		}
 		if (ret < 0)
 			break;
+
+		if (ddir_rw(ddir) && td->o.thinktime)
+			handle_thinktime(td, ddir);
+
 		if (!ddir_rw_sum(td->bytes_done) &&
 		    !td_ioengine_flagged(td, FIO_NOIO))
 			continue;
@@ -1090,9 +1094,6 @@ reap:
 		}
 		if (!in_ramp_time(td) && td->o.latency_target)
 			lat_target_check(td);
-
-		if (ddir_rw(ddir) && td->o.thinktime)
-			handle_thinktime(td, ddir);
 	}
 
 	check_update_rusage(td);
@@ -1744,6 +1745,11 @@ static void *thread_main(void *data)
 	if (rate_submit_init(td, sk_out))
 		goto err;
 
+	if (td->o.thinktime_blocks_type == THINKTIME_BLOCKS_TYPE_COMPLETE)
+		td->thinktime_blocks_counter = td->io_blocks;
+	else
+		td->thinktime_blocks_counter = td->io_issues;
+
 	set_epoch_time(td, o->log_unix_epoch);
 	fio_getrusage(&td->ru_start);
 	memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
cconv.c +2 -0
index 62c2fc2..0d47f9c 100644
@@ -210,6 +210,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
 	o->thinktime = le32_to_cpu(top->thinktime);
 	o->thinktime_spin = le32_to_cpu(top->thinktime_spin);
 	o->thinktime_blocks = le32_to_cpu(top->thinktime_blocks);
+	o->thinktime_blocks_type = le64_to_cpu(top->thinktime_blocks_type);
 	o->fsync_blocks = le32_to_cpu(top->fsync_blocks);
 	o->fdatasync_blocks = le32_to_cpu(top->fdatasync_blocks);
 	o->barrier_blocks = le32_to_cpu(top->barrier_blocks);
@@ -431,6 +432,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->thinktime = cpu_to_le32(o->thinktime);
 	top->thinktime_spin = cpu_to_le32(o->thinktime_spin);
 	top->thinktime_blocks = cpu_to_le32(o->thinktime_blocks);
+	top->thinktime_blocks_type = __cpu_to_le64(o->thinktime_blocks_type);
 	top->fsync_blocks = cpu_to_le32(o->fsync_blocks);
 	top->fdatasync_blocks = cpu_to_le32(o->fdatasync_blocks);
 	top->barrier_blocks = cpu_to_le32(o->barrier_blocks);
engines/cpu.c +1 -0
index ccbfe00..ce74dbc 100644
@@ -268,6 +268,7 @@ static int fio_cpuio_init(struct thread_data *td)
 	 * set thinktime_sleep and thinktime_spin appropriately
 	 */
 	o->thinktime_blocks = 1;
+	o->thinktime_blocks_type = THINKTIME_BLOCKS_TYPE_COMPLETE;
 	o->thinktime_spin = 0;
 	o->thinktime = ((unsigned long long) co->cpucycle *
 				(100 - co->cpuload)) / co->cpuload;
fio.h +5 -0
index ee582a7..ae6ac76 100644
@@ -149,6 +149,9 @@ enum {
 
 	RATE_PROCESS_LINEAR = 0,
 	RATE_PROCESS_POISSON = 1,
+
+	THINKTIME_BLOCKS_TYPE_COMPLETE = 0,
+	THINKTIME_BLOCKS_TYPE_ISSUE = 1,
 };
 
 enum {
@@ -355,6 +358,8 @@ struct thread_data {
 	struct fio_sem *sem;
 	uint64_t bytes_done[DDIR_RWDIR_CNT];
 
+	uint64_t *thinktime_blocks_counter;
+
 	/*
 	 * State for random io, a bitmap of blocks done vs not done
 	 */
options.c +22 -0
index 955bf95..2898850 100644
@@ -3609,6 +3609,28 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.group	= FIO_OPT_G_THINKTIME,
 	},
 	{
+		.name	= "thinktime_blocks_type",
+		.lname	= "Thinktime blocks type",
+		.type	= FIO_OPT_STR,
+		.off1	= offsetof(struct thread_options, thinktime_blocks_type),
+		.help	= "How thinktime_blocks takes effect",
+		.def	= "complete",
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_THINKTIME,
+		.posval = {
+			  { .ival = "complete",
+			    .oval = THINKTIME_BLOCKS_TYPE_COMPLETE,
+			    .help = "Thinktime_blocks takes effect at the completion side",
+			  },
+			  {
+			    .ival = "issue",
+			    .oval = THINKTIME_BLOCKS_TYPE_ISSUE,
+			    .help = "Thinktime_blocks takes effect at the issue side",
+			  },
+		},
+		.parent = "thinktime",
+	},
+	{
 		.name	= "rate",
 		.lname	= "I/O rate",
 		.type	= FIO_OPT_ULL,
thread_options.h +2 -0
index 0a03343..0bd7452 100644
@@ -177,6 +177,7 @@ struct thread_options {
 	unsigned int thinktime;
 	unsigned int thinktime_spin;
 	unsigned int thinktime_blocks;
+	unsigned int thinktime_blocks_type;
 	unsigned int fsync_blocks;
 	unsigned int fdatasync_blocks;
 	unsigned int barrier_blocks;
@@ -479,6 +480,7 @@ struct thread_options_pack {
 	uint32_t thinktime;
 	uint32_t thinktime_spin;
 	uint32_t thinktime_blocks;
+	uint64_t thinktime_blocks_type;
 	uint32_t fsync_blocks;
 	uint32_t fdatasync_blocks;
 	uint32_t barrier_blocks;
Signature · 1 line
1.8.3.1
#2 lore raw

Re: [PATCH v3 1/2] Add a new parameter.

The reason I use uint64 is that using u32 or u8
causes alignment issues and the compilation check
will fail.
I currently don't know how to address it.
Any idea?

Best,
Hongwei

Quoted 272 lines · Hongwei Qin

This patch adds a new parameter thinktime_blocks_type to control the behavior of thinktime_blocks. It can be either complete or issue.
If it is complete (default), fio triggers thinktime when thinktime_blocks number of blocks are **completed**. If it is issue, fio triggers thinktime when thinktime_blocks number of blocks are **issued**

Tests:
jobfile1:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
```

jobfile2:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
thinktime_blocks_type=issue
```

Results:
Current HEAD:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4002msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

This patch:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4001msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

fio jobfile2:
write: IOPS=1, BW=8191B/s (8191B/s)(98.3kB/12001msec); 0 zone resets

- issue 4 requests
- sleep 2s
***
- issue 4 requests
- sleep 2s
- finish

---
 HOWTO            |  7 +++++++
 backend.c        | 16 +++++++++++-----
 cconv.c          |  2 ++
 engines/cpu.c    |  1 +
 fio.h            |  5 +++++
 options.c        | 22 ++++++++++++++++++++++
 thread_options.h |  2 ++
 7 files changed, 50 insertions(+), 5 deletions(-)
HOWTO +7 -0
index 372f268..803bef4 100644
@@ -2562,6 +2562,13 @@ I/O rate
        before we have to complete it and do our :option:`thinktime`. In other words, this
        setting effectively caps the queue depth if the latter is larger.
 
+.. option:: thinktime_blocks_type=str
+
+       This option controls how thinktime_blocks triggers. The default is
+       `complete`, which triggers thinktime when fio completes thinktime_blocks
+       blocks. If this is set to `issue`, then the trigger happens at the
+       issue side.
+
 .. option:: rate=int[,int][,int]
 
        Cap the bandwidth used by this job. The number is in bytes/sec, the normal
backend.c +11 -5
index e20a2e0..874e193 100644
@@ -864,8 +864,8 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
        uint64_t total;
        int left;
 
-       b = ddir_rw_sum(td->io_blocks);
-       if (b % td->o.thinktime_blocks)
+       b = ddir_rw_sum(td->thinktime_blocks_counter);
+       if (b % td->o.thinktime_blocks || !b)
                return;
 
        io_u_quiesce(td);
@@ -1076,6 +1076,10 @@ reap:
                }
                if (ret < 0)
                        break;
+
+               if (ddir_rw(ddir) && td->o.thinktime)
+                       handle_thinktime(td, ddir);
+
                if (!ddir_rw_sum(td->bytes_done) &&
                    !td_ioengine_flagged(td, FIO_NOIO))
                        continue;
@@ -1090,9 +1094,6 @@ reap:
                }
                if (!in_ramp_time(td) && td->o.latency_target)
                        lat_target_check(td);
-
-               if (ddir_rw(ddir) && td->o.thinktime)
-                       handle_thinktime(td, ddir);
        }
 
        check_update_rusage(td);
@@ -1744,6 +1745,11 @@ static void *thread_main(void *data)
        if (rate_submit_init(td, sk_out))
                goto err;
 
+       if (td->o.thinktime_blocks_type == THINKTIME_BLOCKS_TYPE_COMPLETE)
+               td->thinktime_blocks_counter = td->io_blocks;
+       else
+               td->thinktime_blocks_counter = td->io_issues;
+
        set_epoch_time(td, o->log_unix_epoch);
        fio_getrusage(&td->ru_start);
        memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
cconv.c +2 -0
index 62c2fc2..0d47f9c 100644
@@ -210,6 +210,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->thinktime = le32_to_cpu(top->thinktime);
        o->thinktime_spin = le32_to_cpu(top->thinktime_spin);
        o->thinktime_blocks = le32_to_cpu(top->thinktime_blocks);
+       o->thinktime_blocks_type = le64_to_cpu(top->thinktime_blocks_type);
        o->fsync_blocks = le32_to_cpu(top->fsync_blocks);
        o->fdatasync_blocks = le32_to_cpu(top->fdatasync_blocks);
        o->barrier_blocks = le32_to_cpu(top->barrier_blocks);
@@ -431,6 +432,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->thinktime = cpu_to_le32(o->thinktime);
        top->thinktime_spin = cpu_to_le32(o->thinktime_spin);
        top->thinktime_blocks = cpu_to_le32(o->thinktime_blocks);
+       top->thinktime_blocks_type = __cpu_to_le64(o->thinktime_blocks_type);
        top->fsync_blocks = cpu_to_le32(o->fsync_blocks);
        top->fdatasync_blocks = cpu_to_le32(o->fdatasync_blocks);
        top->barrier_blocks = cpu_to_le32(o->barrier_blocks);
engines/cpu.c +1 -0
index ccbfe00..ce74dbc 100644
@@ -268,6 +268,7 @@ static int fio_cpuio_init(struct thread_data *td)
         * set thinktime_sleep and thinktime_spin appropriately
         */
        o->thinktime_blocks = 1;
+       o->thinktime_blocks_type = THINKTIME_BLOCKS_TYPE_COMPLETE;
        o->thinktime_spin = 0;
        o->thinktime = ((unsigned long long) co->cpucycle *
                                (100 - co->cpuload)) / co->cpuload;
fio.h +5 -0
index ee582a7..ae6ac76 100644
@@ -149,6 +149,9 @@ enum {
 
        RATE_PROCESS_LINEAR = 0,
        RATE_PROCESS_POISSON = 1,
+
+       THINKTIME_BLOCKS_TYPE_COMPLETE = 0,
+       THINKTIME_BLOCKS_TYPE_ISSUE = 1,
 };
 
 enum {
@@ -355,6 +358,8 @@ struct thread_data {
        struct fio_sem *sem;
        uint64_t bytes_done[DDIR_RWDIR_CNT];
 
+       uint64_t *thinktime_blocks_counter;
+
        /*
         * State for random io, a bitmap of blocks done vs not done
         */
options.c +22 -0
index 955bf95..2898850 100644
@@ -3609,6 +3609,28 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .group  = FIO_OPT_G_THINKTIME,
        },
        {
+               .name   = "thinktime_blocks_type",
+               .lname  = "Thinktime blocks type",
+               .type   = FIO_OPT_STR,
+               .off1   = offsetof(struct thread_options, thinktime_blocks_type),
+               .help   = "How thinktime_blocks takes effect",
+               .def    = "complete",
+               .category = FIO_OPT_C_IO,
+               .group  = FIO_OPT_G_THINKTIME,
+               .posval = {
+                         { .ival = "complete",
+                           .oval = THINKTIME_BLOCKS_TYPE_COMPLETE,
+                           .help = "Thinktime_blocks takes effect at the completion side",
+                         },
+                         {
+                           .ival = "issue",
+                           .oval = THINKTIME_BLOCKS_TYPE_ISSUE,
+                           .help = "Thinktime_blocks takes effect at the issue side",
+                         },
+               },
+               .parent = "thinktime",
+       },
+       {
                .name   = "rate",
                .lname  = "I/O rate",
                .type   = FIO_OPT_ULL,
thread_options.h +2 -1
index 0a03343..0bd7452 100644
@@ -177,6 +177,7 @@ struct thread_options {
        unsigned int thinktime;
        unsigned int thinktime_spin;
        unsigned int thinktime_blocks;
+       unsigned int thinktime_blocks_type;
        unsigned int fsync_blocks;
        unsigned int fdatasync_blocks;
        unsigned int barrier_blocks;
@@ -479,6 +480,7 @@ struct thread_options_pack {
        uint32_t thinktime;
        uint32_t thinktime_spin;
        uint32_t thinktime_blocks;
+       uint64_t thinktime_blocks_type;
        uint32_t fsync_blocks;
        uint32_t fdatasync_blocks;
        uint32_t barrier_blocks;
--

1.8.3.1

#3 lore raw

Re: [PATCH v3 1/2] Add a new parameter.

Quoted 6 lines · Hongwei Qin

The reason I use uint64 is that using u32 or u8
causes alignment issues and the compilation check
will fail.
I currently don't know how to address it.
Any idea?

Hongwei,
Usually the best approach is to introduce some padding (e.g. as in
https://github.com/axboe/fio/commit/8b38f40138047975ea7a59b9ce06681ffdd2c31d
). Could you also check client/server mode still works?

Jens,
As we've changed FIO_SERVER_VER since the last release (see a87c90fd72823d5438d724e5a57ced8d1f7bed3f ) should we skip changing it again for this change?

Quoted 278 lines

Best,
Hongwei

Quoted 272 lines · Hongwei Qin

This patch adds a new parameter thinktime_blocks_type to control the behavior of thinktime_blocks. It can be either complete or issue.
If it is complete (default), fio triggers thinktime when thinktime_blocks number of blocks are **completed**. If it is issue, fio triggers thinktime when thinktime_blocks number of blocks are **issued**

Tests:
jobfile1:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
```

jobfile2:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
thinktime_blocks_type=issue
```

Results:
Current HEAD:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4002msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

This patch:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4001msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

fio jobfile2:
write: IOPS=1, BW=8191B/s (8191B/s)(98.3kB/12001msec); 0 zone resets

- issue 4 requests
- sleep 2s
***
- issue 4 requests
- sleep 2s
- finish

---
 HOWTO            |  7 +++++++
 backend.c        | 16 +++++++++++-----
 cconv.c          |  2 ++
 engines/cpu.c    |  1 +
 fio.h            |  5 +++++
 options.c        | 22 ++++++++++++++++++++++
 thread_options.h |  2 ++
 7 files changed, 50 insertions(+), 5 deletions(-)
HOWTO +7 -0
index 372f268..803bef4 100644
@@ -2562,6 +2562,13 @@ I/O rate
        before we have to complete it and do our :option:`thinktime`. In other words, this
        setting effectively caps the queue depth if the latter is larger.
 
+.. option:: thinktime_blocks_type=str
+
+       This option controls how thinktime_blocks triggers. The default is
+       `complete`, which triggers thinktime when fio completes thinktime_blocks
+       blocks. If this is set to `issue`, then the trigger happens at the
+       issue side.
+
 .. option:: rate=int[,int][,int]
 
        Cap the bandwidth used by this job. The number is in bytes/sec, the normal
backend.c +11 -5
index e20a2e0..874e193 100644
@@ -864,8 +864,8 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
        uint64_t total;
        int left;
 
-       b = ddir_rw_sum(td->io_blocks);
-       if (b % td->o.thinktime_blocks)
+       b = ddir_rw_sum(td->thinktime_blocks_counter);
+       if (b % td->o.thinktime_blocks || !b)
                return;
 
        io_u_quiesce(td);
@@ -1076,6 +1076,10 @@ reap:
                }
                if (ret < 0)
                        break;
+
+               if (ddir_rw(ddir) && td->o.thinktime)
+                       handle_thinktime(td, ddir);
+
                if (!ddir_rw_sum(td->bytes_done) &&
                    !td_ioengine_flagged(td, FIO_NOIO))
                        continue;
@@ -1090,9 +1094,6 @@ reap:
                }
                if (!in_ramp_time(td) && td->o.latency_target)
                        lat_target_check(td);
-
-               if (ddir_rw(ddir) && td->o.thinktime)
-                       handle_thinktime(td, ddir);
        }
 
        check_update_rusage(td);
@@ -1744,6 +1745,11 @@ static void *thread_main(void *data)
        if (rate_submit_init(td, sk_out))
                goto err;
 
+       if (td->o.thinktime_blocks_type == THINKTIME_BLOCKS_TYPE_COMPLETE)
+               td->thinktime_blocks_counter = td->io_blocks;
+       else
+               td->thinktime_blocks_counter = td->io_issues;
+
        set_epoch_time(td, o->log_unix_epoch);
        fio_getrusage(&td->ru_start);
        memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
cconv.c +2 -0
index 62c2fc2..0d47f9c 100644
@@ -210,6 +210,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->thinktime = le32_to_cpu(top->thinktime);
        o->thinktime_spin = le32_to_cpu(top->thinktime_spin);
        o->thinktime_blocks = le32_to_cpu(top->thinktime_blocks);
+       o->thinktime_blocks_type = le64_to_cpu(top->thinktime_blocks_type);
        o->fsync_blocks = le32_to_cpu(top->fsync_blocks);
        o->fdatasync_blocks = le32_to_cpu(top->fdatasync_blocks);
        o->barrier_blocks = le32_to_cpu(top->barrier_blocks);
@@ -431,6 +432,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->thinktime = cpu_to_le32(o->thinktime);
        top->thinktime_spin = cpu_to_le32(o->thinktime_spin);
        top->thinktime_blocks = cpu_to_le32(o->thinktime_blocks);
+       top->thinktime_blocks_type = __cpu_to_le64(o->thinktime_blocks_type);
        top->fsync_blocks = cpu_to_le32(o->fsync_blocks);
        top->fdatasync_blocks = cpu_to_le32(o->fdatasync_blocks);
        top->barrier_blocks = cpu_to_le32(o->barrier_blocks);
engines/cpu.c +1 -0
index ccbfe00..ce74dbc 100644
@@ -268,6 +268,7 @@ static int fio_cpuio_init(struct thread_data *td)
         * set thinktime_sleep and thinktime_spin appropriately
         */
        o->thinktime_blocks = 1;
+       o->thinktime_blocks_type = THINKTIME_BLOCKS_TYPE_COMPLETE;
        o->thinktime_spin = 0;
        o->thinktime = ((unsigned long long) co->cpucycle *
                                (100 - co->cpuload)) / co->cpuload;
fio.h +5 -0
index ee582a7..ae6ac76 100644
@@ -149,6 +149,9 @@ enum {
 
        RATE_PROCESS_LINEAR = 0,
        RATE_PROCESS_POISSON = 1,
+
+       THINKTIME_BLOCKS_TYPE_COMPLETE = 0,
+       THINKTIME_BLOCKS_TYPE_ISSUE = 1,
 };
 
 enum {
@@ -355,6 +358,8 @@ struct thread_data {
        struct fio_sem *sem;
        uint64_t bytes_done[DDIR_RWDIR_CNT];
 
+       uint64_t *thinktime_blocks_counter;
+
        /*
         * State for random io, a bitmap of blocks done vs not done
         */
options.c +22 -0
index 955bf95..2898850 100644
@@ -3609,6 +3609,28 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .group  = FIO_OPT_G_THINKTIME,
        },
        {
+               .name   = "thinktime_blocks_type",
+               .lname  = "Thinktime blocks type",
+               .type   = FIO_OPT_STR,
+               .off1   = offsetof(struct thread_options, thinktime_blocks_type),
+               .help   = "How thinktime_blocks takes effect",
+               .def    = "complete",
+               .category = FIO_OPT_C_IO,
+               .group  = FIO_OPT_G_THINKTIME,
+               .posval = {
+                         { .ival = "complete",
+                           .oval = THINKTIME_BLOCKS_TYPE_COMPLETE,
+                           .help = "Thinktime_blocks takes effect at the completion side",
+                         },
+                         {
+                           .ival = "issue",
+                           .oval = THINKTIME_BLOCKS_TYPE_ISSUE,
+                           .help = "Thinktime_blocks takes effect at the issue side",
+                         },
+               },
+               .parent = "thinktime",
+       },
+       {
                .name   = "rate",
                .lname  = "I/O rate",
                .type   = FIO_OPT_ULL,
thread_options.h +2 -1
index 0a03343..0bd7452 100644
@@ -177,6 +177,7 @@ struct thread_options {
        unsigned int thinktime;
        unsigned int thinktime_spin;
        unsigned int thinktime_blocks;
+       unsigned int thinktime_blocks_type;
        unsigned int fsync_blocks;
        unsigned int fdatasync_blocks;
        unsigned int barrier_blocks;
@@ -479,6 +480,7 @@ struct thread_options_pack {
        uint32_t thinktime;
        uint32_t thinktime_spin;
        uint32_t thinktime_blocks;
+       uint64_t thinktime_blocks_type;
        uint32_t fsync_blocks;
        uint32_t fdatasync_blocks;
        uint32_t barrier_blocks;
--

1.8.3.1

Signature · 1 line
Sitsofe
#4 lore raw

Re: [PATCH v3 1/2] Add a new parameter.

Hi Sitsofe,
Thanks for the introduction of the padding method. Maybe a new patch later on?

The results of C/S mode:

```
Server:
    fio --server=192.168.1.172,8765

Client (On the same machine):
    fio --client=192.168.1.172,8765 jobfile1
        write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4002msec); 0 zone resets

    blktrace:
        11 reqs -- 2s -- 8 reqs -- 2s -- 5 reqs -- end

    fio --client=192.168.1.172,8765 jobfile2
        write: IOPS=1, BW=8189B/s (8189B/s)(98.3kB/12003msec); 0 zone resets

    blktrace:
        4 reqs -- 2s -- 4 reqs ... -- 4 reqs -- 2s -- end
```
Quoted 302 lines · Sitsofe Wheeler
Quoted 6 lines · Hongwei Qin

The reason I use uint64 is that using u32 or u8
causes alignment issues and the compilation check
will fail.
I currently don't know how to address it.
Any idea?

Hongwei,
Usually the best approach is to introduce some padding (e.g. as in
https://github.com/axboe/fio/commit/8b38f40138047975ea7a59b9ce06681ffdd2c31d
). Could you also check client/server mode still works?

Jens,
As we've changed FIO_SERVER_VER since the last release (see a87c90fd72823d5438d724e5a57ced8d1f7bed3f ) should we skip changing it again for this change?

Quoted 270 lines

Best,
Hongwei

Quoted 264 lines · Hongwei Qin

This patch adds a new parameter thinktime_blocks_type to control the behavior of thinktime_blocks. It can be either complete or issue.
If it is complete (default), fio triggers thinktime when thinktime_blocks number of blocks are **completed**. If it is issue, fio triggers thinktime when thinktime_blocks number of blocks are **issued**

Tests:
jobfile1:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
```

jobfile2:

```
[global]
thread
kb_base=1000
direct=1
size=1GiB
group_reporting
io_size=96KiB
ioengine=libaio
iodepth=8
bs=4096
filename=/dev/qblkdev
rw=randwrite

[fio_randwrite]
thinktime=2s
thinktime_blocks=4
thinktime_blocks_type=issue
```

Results:
Current HEAD:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4002msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

This patch:
fio jobfile1:
write: IOPS=5, BW=24.6kB/s (24.0KiB/s)(98.3kB/4001msec); 0 zone resets

- issue 11 requests
- sleep 2s
- issue 8 requests
- sleep 2s
- issue 5 requests
- finish

fio jobfile2:
write: IOPS=1, BW=8191B/s (8191B/s)(98.3kB/12001msec); 0 zone resets

- issue 4 requests
- sleep 2s
***
- issue 4 requests
- sleep 2s
- finish

---
 HOWTO            |  7 +++++++
 backend.c        | 16 +++++++++++-----
 cconv.c          |  2 ++
 engines/cpu.c    |  1 +
 fio.h            |  5 +++++
 options.c        | 22 ++++++++++++++++++++++
 thread_options.h |  2 ++
 7 files changed, 50 insertions(+), 5 deletions(-)
HOWTO +7 -0
index 372f268..803bef4 100644
@@ -2562,6 +2562,13 @@ I/O rate
        before we have to complete it and do our :option:`thinktime`. In other words, this
        setting effectively caps the queue depth if the latter is larger.
 
+.. option:: thinktime_blocks_type=str
+
+       This option controls how thinktime_blocks triggers. The default is
+       `complete`, which triggers thinktime when fio completes thinktime_blocks
+       blocks. If this is set to `issue`, then the trigger happens at the
+       issue side.
+
 .. option:: rate=int[,int][,int]
 
        Cap the bandwidth used by this job. The number is in bytes/sec, the normal
backend.c +11 -5
index e20a2e0..874e193 100644
@@ -864,8 +864,8 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
        uint64_t total;
        int left;
 
-       b = ddir_rw_sum(td->io_blocks);
-       if (b % td->o.thinktime_blocks)
+       b = ddir_rw_sum(td->thinktime_blocks_counter);
+       if (b % td->o.thinktime_blocks || !b)
                return;
 
        io_u_quiesce(td);
@@ -1076,6 +1076,10 @@ reap:
                }
                if (ret < 0)
                        break;
+
+               if (ddir_rw(ddir) && td->o.thinktime)
+                       handle_thinktime(td, ddir);
+
                if (!ddir_rw_sum(td->bytes_done) &&
                    !td_ioengine_flagged(td, FIO_NOIO))
                        continue;
@@ -1090,9 +1094,6 @@ reap:
                }
                if (!in_ramp_time(td) && td->o.latency_target)
                        lat_target_check(td);
-
-               if (ddir_rw(ddir) && td->o.thinktime)
-                       handle_thinktime(td, ddir);
        }
 
        check_update_rusage(td);
@@ -1744,6 +1745,11 @@ static void *thread_main(void *data)
        if (rate_submit_init(td, sk_out))
                goto err;
 
+       if (td->o.thinktime_blocks_type == THINKTIME_BLOCKS_TYPE_COMPLETE)
+               td->thinktime_blocks_counter = td->io_blocks;
+       else
+               td->thinktime_blocks_counter = td->io_issues;
+
        set_epoch_time(td, o->log_unix_epoch);
        fio_getrusage(&td->ru_start);
        memcpy(&td->bw_sample_time, &td->epoch, sizeof(td->epoch));
cconv.c +2 -0
index 62c2fc2..0d47f9c 100644
@@ -210,6 +210,7 @@ void convert_thread_options_to_cpu(struct thread_options *o,
        o->thinktime = le32_to_cpu(top->thinktime);
        o->thinktime_spin = le32_to_cpu(top->thinktime_spin);
        o->thinktime_blocks = le32_to_cpu(top->thinktime_blocks);
+       o->thinktime_blocks_type = le64_to_cpu(top->thinktime_blocks_type);
        o->fsync_blocks = le32_to_cpu(top->fsync_blocks);
        o->fdatasync_blocks = le32_to_cpu(top->fdatasync_blocks);
        o->barrier_blocks = le32_to_cpu(top->barrier_blocks);
@@ -431,6 +432,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
        top->thinktime = cpu_to_le32(o->thinktime);
        top->thinktime_spin = cpu_to_le32(o->thinktime_spin);
        top->thinktime_blocks = cpu_to_le32(o->thinktime_blocks);
+       top->thinktime_blocks_type = __cpu_to_le64(o->thinktime_blocks_type);
        top->fsync_blocks = cpu_to_le32(o->fsync_blocks);
        top->fdatasync_blocks = cpu_to_le32(o->fdatasync_blocks);
        top->barrier_blocks = cpu_to_le32(o->barrier_blocks);
engines/cpu.c +1 -0
index ccbfe00..ce74dbc 100644
@@ -268,6 +268,7 @@ static int fio_cpuio_init(struct thread_data *td)
         * set thinktime_sleep and thinktime_spin appropriately
         */
        o->thinktime_blocks = 1;
+       o->thinktime_blocks_type = THINKTIME_BLOCKS_TYPE_COMPLETE;
        o->thinktime_spin = 0;
        o->thinktime = ((unsigned long long) co->cpucycle *
                                (100 - co->cpuload)) / co->cpuload;
fio.h +5 -0
index ee582a7..ae6ac76 100644
@@ -149,6 +149,9 @@ enum {
 
        RATE_PROCESS_LINEAR = 0,
        RATE_PROCESS_POISSON = 1,
+
+       THINKTIME_BLOCKS_TYPE_COMPLETE = 0,
+       THINKTIME_BLOCKS_TYPE_ISSUE = 1,
 };
 
 enum {
@@ -355,6 +358,8 @@ struct thread_data {
        struct fio_sem *sem;
        uint64_t bytes_done[DDIR_RWDIR_CNT];
 
+       uint64_t *thinktime_blocks_counter;
+
        /*
         * State for random io, a bitmap of blocks done vs not done
         */
options.c +22 -0
index 955bf95..2898850 100644
@@ -3609,6 +3609,28 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
                .group  = FIO_OPT_G_THINKTIME,
        },
        {
+               .name   = "thinktime_blocks_type",
+               .lname  = "Thinktime blocks type",
+               .type   = FIO_OPT_STR,
+               .off1   = offsetof(struct thread_options, thinktime_blocks_type),
+               .help   = "How thinktime_blocks takes effect",
+               .def    = "complete",
+               .category = FIO_OPT_C_IO,
+               .group  = FIO_OPT_G_THINKTIME,
+               .posval = {
+                         { .ival = "complete",
+                           .oval = THINKTIME_BLOCKS_TYPE_COMPLETE,
+                           .help = "Thinktime_blocks takes effect at the completion side",
+                         },
+                         {
+                           .ival = "issue",
+                           .oval = THINKTIME_BLOCKS_TYPE_ISSUE,
+                           .help = "Thinktime_blocks takes effect at the issue side",
+                         },
+               },
+               .parent = "thinktime",
+       },
+       {
                .name   = "rate",
                .lname  = "I/O rate",
                .type   = FIO_OPT_ULL,
thread_options.h +1 -0
index 0a03343..0bd7452 100644
@@ -177,6 +177,7 @@ struct thread_options {
        unsigned int thinktime;
        unsigned int thinktime_spin;
        unsigned int thinktime_blocks;
+       unsigned int thinktime_blocks_type;
        unsigned int fsync_blocks;
        unsigned int fdatasync_blocks;
        unsigned int barrier_blocks;
@@ -479,6 +480,7 @@ struct thread_options_pack {
        uint32_t thinktime;
        uint32_t thinktime_spin;
Quoted 8 lines
Quoted 8 lines
thread_options.h +1 -1
        uint32_t thinktime_blocks;
+       uint64_t thinktime_blocks_type;
        uint32_t fsync_blocks;
        uint32_t fdatasync_blocks;
        uint32_t barrier_blocks;
--

1.8.3.1

--
Sitsofe

Signature · 1 line
Hongwei
#5 lore raw

Re: [PATCH v3 1/2] Add a new parameter.

Quoted 4 lines · Hongwei Qin

Hi Sitsofe,
Thanks for the introduction of the padding method. Maybe a new patch later on?

I'd prefer we roll it into this patch as otherwise it means another protocol change later.

Signature · 1 line
Sitsofe
#6 lore raw

PATCH v3 2/2 Calculate min_rate with the consideration of thinktime

In reply to Hongwei Qin

This patch updates the compare time if handle_thinktime sleeps or spin.

---
 backend.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
backend.c +6 -2
index 874e193..f2efddd 100644
@@ -858,7 +858,8 @@ static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir)
 	return 0;
 }
 
-static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
+static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir,
+			     struct timespec *time)
 {
 	unsigned long long b;
 	uint64_t total;
@@ -898,6 +899,9 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
 		/* adjust for rate_process=poisson */
 		td->last_usec[ddir] += total;
 	}
+
+	if (time && should_check_rate(td))
+		fio_gettime(time, NULL);
 }
 
 /*
@@ -1078,7 +1082,7 @@ reap:
 			break;
 
 		if (ddir_rw(ddir) && td->o.thinktime)
-			handle_thinktime(td, ddir);
+			handle_thinktime(td, ddir, &comp_time);
 
 		if (!ddir_rw_sum(td->bytes_done) &&
 		    !td_ioengine_flagged(td, FIO_NOIO))
Signature · 1 line
1.8.3.1
#7 lore raw

Re: [PATCH v3 2/2] Calculate min_rate with the consideration of thinktime

Quoted 5 lines · Hongwei Qin

This patch updates the compare time if handle_thinktime sleeps or spin.

Quoted 40 lines
---
 backend.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
backend.c +6 -3
index 874e193..f2efddd 100644
@@ -858,7 +858,8 @@ static long long usec_for_io(struct thread_data *td, enum fio_ddir ddir)
        return 0;
 }
 
-static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
+static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir,
+                            struct timespec *time)
 {
        unsigned long long b;
        uint64_t total;
@@ -898,6 +899,9 @@ static void handle_thinktime(struct thread_data *td, enum fio_ddir ddir)
                /* adjust for rate_process=poisson */
                td->last_usec[ddir] += total;
        }
+
+       if (time && should_check_rate(td))
+               fio_gettime(time, NULL);
 }
 
 /*
@@ -1078,7 +1082,7 @@ reap:
                        break;
 
                if (ddir_rw(ddir) && td->o.thinktime)
-                       handle_thinktime(td, ddir);
+                       handle_thinktime(td, ddir, &comp_time);
 
                if (!ddir_rw_sum(td->bytes_done) &&
                    !td_ioengine_flagged(td, FIO_NOIO))
--

1.8.3.1

Signature · 1 line
Sitsofe