Recent changes (master)
The following changes since commit 1201b24acd347d6daaad969e6abfe0975cb86bc8:
init: did_arg cleanup (2017-11-28 16:00:22 -0700)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 6c3fb04c80c3c241162e743a54761e5e896d4ba2:
options: correct parser type for max_latency (2017-11-29 22:27:05 -0700)
----------------------------------------------------------------
Jens Axboe (13):
options: don't quicksort zoned distribution series
Add support for absolute random zones examples/rand-zones.fio: add zoned_abs example
io_u: cleanup and simplify __get_next_rand_offset_zoned_abs() Unify max split zone support
io_u: don't do expensive int divide for buffer scramble
io_u: do nsec -> usec converison in one spot in account_io_completion()
options: make it clear that max_latency is in usecs
options: make max_latency a 64-bit variable
Change latency targets to be in nsec values internally
verify: kill unneeded forward declaration
verify: convert hdr time to sec+nsec
options: correct parser type for max_latency
Tomohiro Kusumi (1):
Revert "Avoid irrelevant "offset extend ends" error message for chrdev"
HOWTO | 24 +++++++-- cconv.c | 4 +- examples/rand-zones.fio | 8 +++ filesetup.c | 26 ++++----- fio.1 | 24 ++++++++- fio.h | 3 ++ init.c | 7 +++ io_u.c | 83 +++++++++++++++++++++++++---- libfio.c | 1 + options.c | 136 ++++++++++++++++++++++++++++-------------------- profiles/act.c | 3 +- server.h | 2 +- thread_options.h | 14 ++--- verify.c | 5 +- verify.h | 2 +- 15 files changed, 240 insertions(+), 102 deletions(-)
---
Diff of recent changes:
HOWTO +20 -4
@@ -1254,6 +1254,9 @@ I/O type **zoned** Zoned random distribution + **zoned_abs** + Zone absolute random distribution + When using a **zipf** or **pareto** distribution, an input value is also needed to define the access pattern. For **zipf**, this is the `Zipf theta`. For **pareto**, it's the `Pareto power`. Fio includes a test
@@ -1278,10 +1281,23 @@ I/O type random_distribution=zoned:60/10:30/20:8/30:2/40 - similarly to how :option:`bssplit` works for setting ranges and percentages - of block sizes. Like :option:`bssplit`, it's possible to specify separate - zones for reads, writes, and trims. If just one set is given, it'll apply to - all of them. + A **zoned_abs** distribution works exactly like the **zoned**, except + that it takes absolute sizes. For example, let's say you wanted to + define access according to the following criteria: + + * 60% of accesses should be to the first 20G + * 30% of accesses should be to the next 100G + * 10% of accesses should be to the next 500G + + we can define an absolute zoning distribution with: + + random_distribution=zoned_abs=60/20G:30/100G:10/500g + + Similarly to how :option:`bssplit` works for setting ranges and + percentages of block sizes. Like :option:`bssplit`, it's possible to + specify separate zones for reads, writes, and trims. If just one set + is given, it'll apply to all of them. This goes for both **zoned** + **zoned_abs** distributions. .. option:: percentage_random=int[,int][,int]
cconv.c +2 -2
@@ -234,7 +234,6 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->loops =le32_to_cpu (top->loops); o->mem_type =le32_to_cpu (top->mem_type); o->mem_align =le32_to_cpu (top->mem_align); - o->max_latency =le32_to_cpu (top->max_latency); o->stonewall =le32_to_cpu (top->stonewall); o->new_group =le32_to_cpu (top->new_group); o->numjobs =le32_to_cpu (top->numjobs);
@@ -283,6 +282,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->sync_file_range =le32_to_cpu (top->sync_file_range); o->latency_target =le64_to_cpu (top->latency_target); o->latency_window =le64_to_cpu (top->latency_window); + o->max_latency =le64_to_cpu (top->max_latency); o->latency_percentile.u.f =fio_uint64_to_double (le64_to_cpu (top->latency_percentile.u.i)); o->compress_percentage =le32_to_cpu (top->compress_percentage); o->compress_chunk =le32_to_cpu (top->compress_chunk);
@@ -423,7 +423,6 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->loops =cpu_to_le32 (o->loops); top->mem_type =cpu_to_le32 (o->mem_type); top->mem_align =cpu_to_le32 (o->mem_align); - top->max_latency =cpu_to_le32 (o->max_latency); top->stonewall =cpu_to_le32 (o->stonewall); top->new_group =cpu_to_le32 (o->new_group); top->numjobs =cpu_to_le32 (o->numjobs);
@@ -472,6 +471,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->sync_file_range =cpu_to_le32 (o->sync_file_range); top->latency_target =__cpu_to_le64 (o->latency_target); top->latency_window =__cpu_to_le64 (o->latency_window); + top->max_latency =__cpu_to_le64 (o->max_latency); top->latency_percentile.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->latency_percentile.u.f)); top->compress_percentage =cpu_to_le32 (o->compress_percentage); top->compress_chunk =cpu_to_le32 (o->compress_chunk);
examples/rand-zones.fio +8 -0
@@ -10,6 +10,14 @@ rw=randread norandommap random_distribution=zoned:50/5:30/15:20/ +# It's also possible to use zoned_abs to specify absolute sizes. For +# instance, if you do: +# +# random_distribution=zoned_abs:50/10G:30/100G:20/500G +# +# Then 50% of the access will be to the first 10G of the drive, 30% +# will be to the next 100G, and 20% will be to the next 500G. + # The above applies to all of reads/writes/trims. If we wanted to do # something differently for writes, let's say 50% for the first 10% # and 50% for the remaining 90%, we could do it by adding a new section
filesetup.c +9 -17
@@ -435,8 +435,12 @@ static int get_file_size(struct thread_data *td, struct fio_file *f) ret =bdev_size (td, f); else if (f->filetype == FIO_TYPE_CHAR) ret =char_size (td, f); - else - f->real_file_size = -1ULL; + else { + f->real_file_size =-1 ; +log_info ("%s: failed to get file size of %s\n" , td->o.name, + f->file_name); +return 1 ;/* avoid offset extends end error message */ + }/* * Leave ->real_file_size with 0 since it could be expectation
@@ -446,22 +450,10 @@ static int get_file_size(struct thread_data *td, struct fio_file *f)return ret;/* -* If ->real_file_size is -1, a conditional for the message -* "offset extends end" is always true, but it makes no sense, -* so just return the same value here. -*/ -if (f->real_file_size ==-1ULL ) { -log_info ("%s: failed to get file size of %s\n" , td->o.name, - f->file_name); -return 1 ; - } - -if (td->o.start_offset && f->file_offset ==0 ) -dprint (FD_FILE ,"offset of file %s not initialized yet\n" , - f->file_name); -/* * ->file_offset normally hasn't been initialized yet, so this -* is basically always false. +* is basically always false unless ->real_file_size is -1, but +* if ->real_file_size is -1 this message doesn't make sense. +* As a result, this message is basically useless. */ if (f->file_offset > f->real_file_size) { log_err("%s: offset extends end (%llu > %llu)\n", td->o.name,
fio.1 +23 -1
@@ -1033,6 +1033,8 @@ Normal (Gaussian) distribution .TP .B zoned Zoned random distribution +.B zoned_abs +Zoned absolute random distribution .RE .P When using a \fBzipf\fR or \fBpareto\fR distribution, an input value is also
@@ -1068,7 +1070,27 @@ example, the user would do: random_distribution=zoned:60/10:30/20:8/30:2/40 .RE .P -similarly to how \fBbssplit\fR works for setting ranges and percentages +A \fBzoned_abs\fR distribution works exactly like the\fBzoned\fR, except that +it takes absolute sizes. For example, let's say you wanted to define access +according to the following criteria: +.RS +.P +.PD 0 +60% of accesses should be to the first 20G +.P +30% of accesses should be to the next 100G +.P +10% of accesses should be to the next 500G +.PD +.RE +.P +we can define an absolute zoning distribution with: +.RS +.P +random_distribution=zoned:60/10:30/20:8/30:2/40 +.RE +.P +Similarly to how \fBbssplit\fR works for setting ranges and percentages of block sizes. Like \fBbssplit\fR, it's possible to specify separate zones for reads, writes, and trims. If just one set is given, it'll apply to all of them.
fio.h +3 -0
@@ -158,6 +158,8 @@ void sk_out_drop(void);struct zone_split_index {uint8_t size_perc;uint8_t size_perc_prev; +uint64_t size; +uint64_t size_prev; };/*
@@ -813,6 +815,7 @@ enum {FIO_RAND_DIST_PARETO ,FIO_RAND_DIST_GAUSS ,FIO_RAND_DIST_ZONED , + FIO_RAND_DIST_ZONED_ABS, };#define FIO_DEF_ZIPF 1.1
init.c +7 -0
@@ -925,6 +925,13 @@ static int fixup_options(struct thread_data *td) ret =1 ; } +/* +* Fix these up to be nsec internally +*/ + o->max_latency *=1000ULL ; + o->latency_target *=1000ULL ; + o->latency_window *=1000ULL ; +return ret; }
io_u.c +74 -9
130 lines not on this page · raw message
libfio.c +1 -0
@@ -366,6 +366,7 @@ int initialize_fio(char *envp[])compiletime_assert ((offsetof(struct jobs_eta , m_rate) %8 ) ==0 ,"m_rate" );compiletime_assert (__TD_F_LAST <=TD_ENG_FLAG_SHIFT ,"TD_ENG_FLAG_SHIFT" ); +compiletime_assert (BSSPLIT_MAX ==ZONESPLIT_MAX ,"bsssplit/zone max" ); err =endian_check ();if (err) {
options.c +81 -55
308 lines not on this page · raw message
profiles/act.c +2 -1
@@ -288,10 +288,11 @@ static int act_prep_cmdline(void)return 0 ; } -static int act_io_u_lat (struct thread_data *td,uint64_t usec) +static int act_io_u_lat (struct thread_data *td,uint64_t nsec) {struct act_prof_data *apd = td->prof_data;struct act_slice *slice; +uint64_t usec = nsec /1000ULL ;int i, ret =0 ;double perm;
server.h +1 -1
@@ -49,7 +49,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =66 , +FIO_SERVER_VER =67 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
thread_options.h +7 -7
@@ -36,6 +36,8 @@ struct bssplit {struct zone_split {uint8_t access_perc;uint8_t size_perc; +uint8_t pad[6 ]; +uint64_t size; };#define NR_OPTS_SZ (FIO_MAX_OPTS / (8 * sizeof(uint64_t)))
@@ -190,7 +192,7 @@ struct thread_options {enum fio_memtype mem_type;unsigned int mem_align; -unsigned int max_latency; +unsigned long long max_latency;unsigned int stonewall;unsigned int new_group;
@@ -427,7 +429,8 @@ struct thread_options_pack {uint32_t random_distribution;uint32_t exitall_error; -uint32_t pad; + +uint32_t sync_file_range;struct zone_split zone_split[DDIR_RWDIR_CNT ][ZONESPLIT_MAX ];uint32_t zone_split_nr[DDIR_RWDIR_CNT ];
@@ -467,8 +470,6 @@ struct thread_options_pack {uint32_t mem_type;uint32_t mem_align; -uint32_t max_latency; -uint32_t stonewall;uint32_t new_group;uint32_t numjobs;
@@ -519,6 +520,7 @@ struct thread_options_pack {uint64_t trim_backlog;uint32_t clat_percentiles;uint32_t percentile_precision; +uint32_t pad;fio_fp64_t percentile_list[FIO_IO_U_LIST_MAX_LEN ];uint8_t read_iolog_file[FIO_TOP_STR_MAX ];
@@ -579,11 +581,9 @@ struct thread_options_pack {uint64_t offset_increment;uint64_t number_ios; -uint32_t sync_file_range; -uint32_t pad2; -uint64_t latency_target;uint64_t latency_window; +uint64_t max_latency;fio_fp64_t latency_percentile;uint32_t sig_figs;
verify.c +1 -4
@@ -30,9 +30,6 @@static void populate_hdr (struct thread_data *td,struct io_u *io_u,struct verify_header *hdr,unsigned int header_num,unsigned int header_len); -static void fill_hdr (struct thread_data *td,struct io_u *io_u, -struct verify_header *hdr,unsigned int header_num, -unsigned int header_len,uint64_t rand_seed);static void __fill_hdr (struct thread_data *td,struct io_u *io_u,struct verify_header *hdr,unsigned int header_num,unsigned int header_len,uint64_t rand_seed);
@@ -1167,7 +1164,7 @@ static void __fill_hdr(struct thread_data *td, struct io_u *io_u, hdr->rand_seed = rand_seed; hdr->offset = io_u->offset + header_num * td->o.verify_interval; hdr->time_sec = io_u->start_time.tv_sec; - hdr->time_usec = io_u->start_time.tv_nsec /1000 ; + hdr->time_nsec = io_u->start_time.tv_nsec; hdr->thread = td->thread_number; hdr->numberio = io_u->numberio; hdr->crc32 =fio_crc32c (p, offsetof(struct verify_header , crc32));
verify.h +1 -1
@@ -43,7 +43,7 @@ struct verify_header {uint64_t rand_seed;uint64_t offset;uint32_t time_sec; -uint32_t time_usec; +uint32_t time_nsec;uint16_t thread;uint16_t numberio;uint32_t crc32;