Recent changes (master)
The following changes since commit 0e21f5c6f64a73bcede20e1a29a885845e453b8e:
Merge branch 'testing' of https://github.com/vincentkfu/fio (2020-05-20 14:19:12 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 1bb1bcade4acbd3cf69ba1c825ad01eebf0b3cdf:
zbd: make zbd_info->mutex non-recursive (2020-05-21 17:23:13 -0600)
----------------------------------------------------------------
Alexey Dobriyan (5):
verify: decouple seed generation from buffer fill
zbd: bump ZBD_MAX_OPEN_ZONES
zbd: don't lock zones outside working area
zbd: introduce per job maximum open zones limit
zbd: make zbd_info->mutex non-recursive
Jens Axboe (1):
Merge branch 'latency_run' of https://github.com/liu-song-6/fio
Song Liu (1):
Add option latency_run to continue enable latency_target
HOWTO | 7 ++++ backend.c | 6 --- cconv.c | 2 + file.h | 3 ++ fio.1 | 11 +++++- fio.h | 2 + io_u.c | 18 ++++++++- options.c | 26 +++++++++++-- server.h | 2 +- thread_options.h | 3 ++ verify.c | 20 ++++------ zbd.c | 110 ++++++++++++++++++++++++++++++++++++++----------------- zbd.h | 3 ++ zbd_types.h | 2 +- 14 files changed, 155 insertions(+), 60 deletions(-)
---
Diff of recent changes:
HOWTO +7 -0
@@ -2552,6 +2552,13 @@ I/O latency defaults to 100.0, meaning that all I/Os must be equal or below to the value set by :option:`latency_target`. +.. option:: latency_run=bool + + Used with :option:`latency_target`. If false (default), fio will find + the highest queue depth that meets :option:`latency_target` and exit. If + true, fio will continue running and try to meet :option:`latency_target` + by adjusting queue depth. + .. option:: max_latency=time If set, fio will exit the job with an ETIMEDOUT error if it exceeds this
backend.c +0 -6
@@ -1006,12 +1006,6 @@ static void do_io(struct thread_data *td, uint64_t *bytes_done)if (td->o.verify !=VERIFY_NONE && io_u->ddir ==DDIR_READ && ((io_u->flags &IO_U_F_VER_LIST ) || !td_rw (td))) { -if (!td->o.verify_pattern_bytes) { - io_u->rand_seed =__rand (&td->verify_state); -if (sizeof (int ) !=sizeof (long *)) - io_u->rand_seed *=__rand (&td->verify_state); - } -if (verify_state_should_stop (td, io_u)) {put_io_u (td, io_u);break ;
cconv.c +2 -0
@@ -288,6 +288,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->latency_window =le64_to_cpu (top->latency_window); o->max_latency =le64_to_cpu (top->max_latency); o->latency_percentile.u.f =fio_uint64_to_double (le64_to_cpu (top->latency_percentile.u.i)); + o->latency_run =le32_to_cpu (top->latency_run); o->compress_percentage =le32_to_cpu (top->compress_percentage); o->compress_chunk =le32_to_cpu (top->compress_chunk); o->dedupe_percentage =le32_to_cpu (top->dedupe_percentage);
@@ -487,6 +488,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->latency_window =__cpu_to_le64 (o->latency_window); top->max_latency =__cpu_to_le64 (o->max_latency); top->latency_percentile.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->latency_percentile.u.f)); + top->latency_run =__cpu_to_le32 (o->latency_run); top->compress_percentage =cpu_to_le32 (o->compress_percentage); top->compress_chunk =cpu_to_le32 (o->compress_chunk); top->dedupe_percentage =cpu_to_le32 (o->dedupe_percentage);
file.h +3 -0
@@ -104,6 +104,9 @@ struct fio_file {* Zoned block device information. See also zonemode=zbd. */ struct zoned_block_device_info *zbd_info; +/* zonemode=zbd working area */ +uint32_t min_zone;/* inclusive */ +uint32_t max_zone;/* exclusive */ /* * Track last end and last start of IO for a given data direction
fio.1 +10 -1
@@ -804,7 +804,11 @@ so. Default: false. When running a random write test across an entire drive many more zones will be open than in a typical application workload. Hence this command line option that allows to limit the number of open zones. The number of open zones is -defined as the number of zones to which write commands are issued. +defined as the number of zones to which write commands are issued by all +threads/processes. +.TP +.BI job_max_open_zones \fR=\fPint +Limit on the number of simultaneously opened zones per single thread/process. .TP .BI zone_reset_threshold \fR=\fPfloat A number between zero and one that indicates the ratio of logical blocks with
@@ -2276,6 +2280,11 @@ The percentage of I/Os that must fall within the criteria specified by defaults to 100.0, meaning that all I/Os must be equal or below to the value set by \fBlatency_target\fR. .TP +.BI latency_run \fR=\fPbool +Used with \fBlatency_target\fR. If false (default), fio will find the highest +queue depth that meets \fBlatency_target\fR and exit. If true, fio will continue +running and try to meet \fBlatency_target\fR by adjusting queue depth. +.TP .BI max_latency \fR=\fPtime If set, fio will exit the job with an ETIMEDOUT error if it exceeds this maximum latency. When the unit is omitted, the value is interpreted in
fio.h +2 -0
@@ -260,6 +260,7 @@ struct thread_data {struct frand_state prio_state;struct zone_split_index **zone_state_index; +unsigned int num_open_zones;unsigned int verify_batch;unsigned int trim_batch;
@@ -377,6 +378,7 @@ struct thread_data {unsigned int latency_qd_high;unsigned int latency_qd_low;unsigned int latency_failed; +unsigned int latency_stable_count;uint64_t latency_ios;int latency_end_run;
io_u.c +17 -1
@@ -1391,6 +1391,7 @@ static bool __lat_target_failed(struct thread_data *td) td->latency_qd_low--; td->latency_qd = (td->latency_qd + td->latency_qd_low) /2 ; + td->latency_stable_count =0 ;dprint (FD_RATE ,"Ramped down: %d %d %d\n" , td->latency_qd_low, td->latency_qd, td->latency_qd_high);
@@ -1440,6 +1441,21 @@ static void lat_target_success(struct thread_data *td) td->latency_qd_low = td->latency_qd; +if (td->latency_qd +1 == td->latency_qd_high) { +/* +* latency_qd will not incease on lat_target_success(), so +* called stable. If we stick with this queue depth, the +* final latency is likely lower than latency_target. Fix +* this by increasing latency_qd_high slowly. Use a naive +* heuristic here. If we get lat_target_success() 3 times +* in a row, increase latency_qd_high by 1. +*/ +if (++td->latency_stable_count >=3 ) { + td->latency_qd_high++; + td->latency_stable_count =0 ; + } + } +/* * If we haven't failed yet, we double up to a failing value instead * of bisecting from highest possible queue depth. If we have set
@@ -1459,7 +1475,7 @@ static void lat_target_success(struct thread_data *td)* Same as last one, we are done. Let it run a latency cycle, so * we get only the results from the targeted depth. */ -if (td->latency_qd == qd) { +if (!o->latency_run && td->latency_qd == qd) {if (td->latency_end_run) {dprint (FD_RATE ,"We are done\n" ); td->done =1 ;
options.c +23 -3
@@ -3360,12 +3360,22 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { }, { .name = "max_open_zones", - .lname = "Maximum number of open zones", + .lname = "Per device/file maximum number of open zones", .type = FIO_OPT_INT, .off1 = offsetof(struct thread_options, max_open_zones), .maxval = ZBD_MAX_OPEN_ZONES, - .help ="Limit random writes to SMR drives to the specified" - " number of sequential zones", + .help = "Limit on the number of simultaneously opened sequential write zones with zonemode=zbd", + .def = "0", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_INVALID, + }, + { + .name = "job_max_open_zones", + .lname = "Job maximum number of open zones", + .type = FIO_OPT_INT, + .off1 = offsetof(struct thread_options, job_max_open_zones), + .maxval = ZBD_MAX_OPEN_ZONES, + .help = "Limit on the number of simultaneously opened sequential write zones with zonemode=zbd by one thread/process", .def = "0", .category = FIO_OPT_C_IO, .group = FIO_OPT_G_INVALID,
@@ -3672,6 +3682,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .category = FIO_OPT_C_IO, .group = FIO_OPT_G_LATPROF, }, + { + .name = "latency_run", + .lname = "Latency Run", + .type = FIO_OPT_BOOL, + .off1 = offsetof(struct thread_options, latency_run), + .help = "Keep adjusting queue depth to match latency_target", + .def = "0", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_LATPROF, + }, { .name = "invalidate", .lname = "Cache invalidate",
server.h +1 -1
@@ -48,7 +48,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =82 , +FIO_SERVER_VER =83 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +3 -0
@@ -324,6 +324,7 @@ struct thread_options {unsigned long long latency_target;unsigned long long latency_window;fio_fp64_t latency_percentile; +uint32_t latency_run;unsigned int sig_figs;
@@ -342,6 +343,7 @@ struct thread_options {/* Parameters that affect zonemode=zbd */ unsigned int read_beyond_wp;int max_open_zones; +unsigned int job_max_open_zones;fio_fp64_t zrt;fio_fp64_t zrf; };
@@ -612,6 +614,7 @@ struct thread_options_pack {uint64_t latency_window;uint64_t max_latency;fio_fp64_t latency_percentile; +uint32_t latency_run;uint32_t sig_figs;
verify.c +7 -13
@@ -46,15 +46,6 @@ static void __fill_buffer(struct thread_options *o, uint64_t seed, void *p,__fill_random_buf_percentage (seed, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes); } -static uint64_t fill_buffer (struct thread_data *td,void *p, -unsigned int len) -{ -struct frand_state *fs = &td->verify_state; -struct thread_options *o = &td->o; - -return fill_random_buf_percentage (fs, p, o->compress_percentage, len, len, o->buffer_pattern, o->buffer_pattern_bytes); -} -void fill_verify_pattern (struct thread_data *td,void *p,unsigned int len,struct io_u *io_u,uint64_t seed,int use_seed) {
@@ -63,10 +54,13 @@ void fill_verify_pattern(struct thread_data *td, void *p, unsigned int len,if (!o->verify_pattern_bytes) {dprint (FD_VERIFY ,"fill random bytes len=%u\n" , len); -if (use_seed) -__fill_buffer (o, seed, p, len); -else - io_u->rand_seed =fill_buffer (td, p, len); +if (!use_seed) { + seed =__rand (&td->verify_state); +if (sizeof (int ) !=sizeof (long *)) + seed *= (unsigned long )__rand (&td->verify_state); + } + io_u->rand_seed = seed; +__fill_buffer (o, seed, p, len);return ; }
zbd.c +76 -34
295 lines not on this page · raw message
zbd.h +3 -0
@@ -45,6 +45,8 @@ struct fio_zone_info {/** * zoned_block_device_info - zoned block device characteristics * @model: Device model. +* @max_open_zones: global limit on the number of simultaneously opened +* sequential write zones. * @mutex: Protects the modifiable members in this structure (refcount and * num_open_zones). * @zone_size: size of a single zone in units of 512 bytes
@@ -65,6 +67,7 @@ struct fio_zone_info {*/ struct zoned_block_device_info {enum zbd_zoned_model model; +uint32_t max_open_zones;pthread_mutex_t mutex;uint64_t zone_size;uint64_t sectors_with_data;
zbd_types.h +1 -1
@@ -8,7 +8,7 @@#include <inttypes.h> -#define ZBD_MAX_OPEN_ZONES 128 +#define ZBD_MAX_OPEN_ZONES 4096/* * Zoned block device models.