added new type of random distribution
PATCH 1/1 added new type of random distribution
From: vadlakondaswetha <[email protected]>
This PR introduces a new generic random distribution primitive: sequence.
While fio currently provides excellent primitives for linear, pseudo-random (randread), and strided access patterns, it lacks a mechanism to simulate deterministic, local non-linearity within a repeating stride.
This access pattern has become a critical benchmark vector for modern workloads, most notably:
*LLM Inference Weights Loading*
Reading .safetensors model files which contain a list of tensor files. Even though the file is read in sequential mode, within a tensor, data will be requested out of order.
*Database Engine Log-Merging / LSM-Trees*
Scenarios where specific block indices (like parity blocks or metadata headers) must be systematically read out-of-order within every repeating chunk or block group.
Using a generated read_iolog for multi-terabyte model benchmarks is sub-optimal as the trace files become massive, unscalable, and lack dynamic flexibility across varying block sizes. The sequence distribution resolves this by calculating offsets algorithmically on the fly.
NEW OPTIONS INTRODUCED
----------------------
*random_distribution=sequence:N1,N2,N3*
Intercepts the random offset generator to utilize the pattern sequence.
*random_sequence_stride*
Boolean option to modify sequence distribution behavior.
- When disabled (0, default), the sequence loops over absolute block indices.
- When enabled (1), it switches to a Strided Block Group pattern, advancing the base block index by the sequence length after each cycle to progress through the file.
--- cconv.c | 22 ++++++++ fio.h | 1 + io_u.c | 24 ++++++++ options.c | 84 ++++++++++++++++++++++++++++ t/sequence.py | 140 +++++++++++++++++++++++++++++++++++++++++++++++ thread_options.h | 7 +++ 6 files changed, 278 insertions(+) create mode 100644 t/sequence.py
cconv.c +22 -0
@@ -56,6 +56,10 @@ static void free_thread_options_to_cpu(struct thread_options *o)free (o->bssplit[i]);free (o->zone_split[i]); } +if (o->random_sequence) { +free (o->random_sequence); + o->random_sequence =NULL ; + } }size_t thread_options_pack_size (struct thread_options *o)
@@ -239,6 +243,13 @@ int convert_thread_options_to_cpu(struct thread_options *o, o->fsync_on_close =le32_to_cpu (top->fsync_on_close); o->bs_is_seq_rand =le32_to_cpu (top->bs_is_seq_rand); o->random_distribution =le32_to_cpu (top->random_distribution); + o->random_sequence_nr =le32_to_cpu (top->random_sequence_nr); +if (o->random_sequence_nr) { + o->random_sequence =malloc (o->random_sequence_nr *sizeof (unsigned int )); +for (j =0 ; j < o->random_sequence_nr; j++) + o->random_sequence[j] =le32_to_cpu (top->random_sequence[j]); + } + o->random_sequence_stride =le32_to_cpu (top->random_sequence_stride); o->exitall_error =le32_to_cpu (top->exitall_error); o->zipf_theta.u.f =fio_uint64_to_double (le64_to_cpu (top->zipf_theta.u.i)); o->pareto_h.u.f =fio_uint64_to_double (le64_to_cpu (top->pareto_h.u.i));
@@ -495,6 +506,17 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->fsync_on_close =cpu_to_le32 (o->fsync_on_close); top->bs_is_seq_rand =cpu_to_le32 (o->bs_is_seq_rand); top->random_distribution =cpu_to_le32 (o->random_distribution); + top->random_sequence_nr =cpu_to_le32 (o->random_sequence_nr); +if (o->random_sequence_nr) { +unsigned int seq_nr = o->random_sequence_nr; +if (seq_nr >FIO_SEQ_MAX ) { +log_err ("fio: FIO_SEQ_MAX is too small\n" ); + seq_nr =FIO_SEQ_MAX ; + } +for (j =0 ; j < seq_nr; j++) + top->random_sequence[j] =cpu_to_le32 (o->random_sequence[j]); + } + top->random_sequence_stride =cpu_to_le32 (o->random_sequence_stride); top->exitall_error =cpu_to_le32 (o->exitall_error); top->zipf_theta.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->zipf_theta.u.f)); top->pareto_h.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->pareto_h.u.f));
fio.h +1 -0
@@ -957,6 +957,7 @@ enum {FIO_RAND_DIST_GAUSS ,FIO_RAND_DIST_ZONED ,FIO_RAND_DIST_ZONED_ABS , + FIO_RAND_DIST_SEQUENCE, };#define FIO_DEF_ZIPF 1.1
io_u.c +24 -0
@@ -291,6 +291,28 @@ bail:return 0 ; } +static int __get_next_rand_offset_sequence (struct thread_data *td, +struct fio_file *f,enum fio_ddir ddir, +uint64_t *b) +{ +uint64_t io_count; +unsigned int idx; + +if (!td->o.random_sequence_nr) +return 1 ; + + io_count = td->io_issues[ddir]; + idx = io_count % td->o.random_sequence_nr; + +if (td->o.random_sequence_stride) { +uint64_t group_idx = io_count / td->o.random_sequence_nr; + *b = group_idx * td->o.random_sequence_nr + td->o.random_sequence[idx]; + }else { + *b = td->o.random_sequence[idx]; + } +return 0 ; +} +static int get_next_rand_offset (struct thread_data *td,struct fio_file *f,enum fio_ddir ddir,uint64_t *b) {
@@ -314,6 +336,8 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,return __get_next_rand_offset_zoned (td, f, ddir, b); else if (td->o.random_distribution == FIO_RAND_DIST_ZONED_ABS) return __get_next_rand_offset_zoned_abs(td, f, ddir, b); + else if (td->o.random_distribution == FIO_RAND_DIST_SEQUENCE) + return __get_next_rand_offset_sequence(td, f, ddir, b);log_err ("fio: unknown random distribution: %d\n" , td->o.random_distribution);return 1 ;
options.c +84 -0
124 lines not on this page · raw message
t/sequence.py new file +140 -0
140 lines not on this page · raw message
thread_options.h +7 -0
@@ -43,6 +43,7 @@ enum dedupe_mode {#define BSSPLIT_MAX 64#define ZONESPLIT_MAX 256 +#define FIO_SEQ_MAX 64struct split {unsigned int nr;
@@ -190,6 +191,9 @@ struct thread_options {unsigned int verify_only;unsigned int random_distribution; +unsigned int *random_sequence; +unsigned int random_sequence_nr; +unsigned int random_sequence_stride;unsigned int exitall_error;struct zone_split *zone_split[DDIR_RWDIR_CNT ];
@@ -527,6 +531,9 @@ struct thread_options_pack {uint32_t bs_is_seq_rand;uint32_t random_distribution; +uint32_t random_sequence[FIO_SEQ_MAX ]; +uint32_t random_sequence_nr; +uint32_t random_sequence_stride;uint32_t exitall_error;uint32_t sync_file_range;
Signature · 1 line
2.54.0.545.g6539524ca2-goog
The result of fio's continuous integration tests was: success
For more details see https://github.com/fiotestbot/fio/actions/runs/25421974783