Recent changes (master)
Subsystems (2)
The following changes since commit ac2aa2ca02dec925fa05cb9e9d4a1cd25e78ae84:
Kill of IO engine cancelation support (2025-08-21 16:23:59 -0600)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 1fb2d4acde3d7e205cf941167c7efad49baf525c:
verify: use new buffer for threads with %o format (2025-08-22 13:57:09 -0400)
----------------------------------------------------------------
Tomas Winkler (13):
sprandom: add command line options
sprandom: add debug facility
sprandom: examples: add sprandom example file
sprandom: implement region computation and invalidation percentage
sprandom: set up LFSR random generator and disable randommap
num2str: add bytes2str_simple()
unittests: add bytes2str_simple()
sprandom: pcbuf.h add two-phase circular buffer header-only library
unittests: add pcbuf simple unit test
sprandom: initialize random state
sprandom: implement sprandom_get_next_offset()
sprandom: initialize sprandom for file
sprandom: integrate sprandom_get_next_offset() into io_u path
Vincent Fu (3):
Merge branch 'sprandom' of https://github.com/tomas-winkler-sndk/fio
sprandom: abort when invalid options specified
verify: use new buffer for threads with %o format
HOWTO.rst | 31 ++ Makefile | 4 +- cconv.c | 6 + debug.h | 1 + examples/sprandom.fio | 41 +++ file.h | 3 + filesetup.c | 13 + fio.1 | 28 ++ fio.h | 2 + init.c | 35 ++ io_u.c | 21 +- lib/num2str.c | 32 +- lib/num2str.h | 2 + options.c | 37 +++ pcbuf.h | 211 ++++++++++++ server.h | 2 +- sprandom.c | 835 ++++++++++++++++++++++++++++++++++++++++++++++++ sprandom.h | 78 +++++ thread_options.h | 6 + unittests/lib/num2str.c | 35 ++ unittests/lib/pcbuf.c | 116 +++++++ unittests/unittest.c | 1 + unittests/unittest.h | 1 + verify.c | 2 +- 24 files changed, 1538 insertions(+), 5 deletions(-) create mode 100644 examples/sprandom.fio create mode 100644 pcbuf.h create mode 100644 sprandom.c create mode 100644 sprandom.h create mode 100644 unittests/lib/pcbuf.c
---
Diff of recent changes:
HOWTO.rst +31 -0
@@ -1632,6 +1632,37 @@ I/O type space exceeds 2^32 blocks. If it does, then **tausworthe64** is selected automatically. +.. option:: sprandom=bool + + + SPRandom is a method designed to rapidly precondition SSDs for + steady-state random write workloads. It divides the device into + equally sized regions and writes the device's entire physical capacity + once, selecting offsets so that the regions have a distribution of + invalid blocks matching the distribution that occurs at steady state. + + Default: false. + + It uses **random_generator=lfsr**, which fio will set by default. + Selecting any other random generator will result in an error. + + +.. option:: spr_num_regions=int + + See :option:`sprandom`. Specifies the number of regions used for SPRandom. + For large devices it is better to use more regions, to increase precision + and reduce memory allocation. The allocation is proportional to the region size. + + Default=100 + + +.. option:: spr_op=float + + See :option:`sprandom`. Over-provisioning ratio in the range (0, 1), + as specified by the SSD manufacturer. + + Default=0.15 + Block size ~~~~~~~~~~
Makefile +3 -1
@@ -62,7 +62,8 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ workqueue.c rate-submit.c optgroup.c helper_thread.c \ - steadystate.c zone-dist.c zbd.c dedupe.c dataplacement.c + steadystate.c zone-dist.c zbd.c dedupe.c dataplacement.c \ + sprandom.c ifdef CONFIG_LIBHDFS HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
@@ -448,6 +449,7 @@ UT_OBJS = unittests/unittest.o UT_OBJS += unittests/lib/memalign.o UT_OBJS += unittests/lib/num2str.o UT_OBJS += unittests/lib/strntol.o +UT_OBJS += unittests/lib/pcbuf.o UT_OBJS += unittests/oslib/strlcat.o UT_OBJS += unittests/oslib/strndup.o UT_OBJS += unittests/oslib/strcasestr.o
cconv.c +6 -0
@@ -228,6 +228,9 @@ int convert_thread_options_to_cpu(struct thread_options *o, o->job_start_clock_id =le32_to_cpu (top->job_start_clock_id); o->norandommap =le32_to_cpu (top->norandommap); o->softrandommap =le32_to_cpu (top->softrandommap); + o->sprandom =le32_to_cpu (top->sprandom); + o->spr_num_regions =le32_to_cpu (top->spr_num_regions); + o->spr_over_provisioning.u.f =fio_uint64_to_double (le64_to_cpu (top->spr_over_provisioning.u.i)); o->bs_unaligned =le32_to_cpu (top->bs_unaligned); o->fsync_on_close =le32_to_cpu (top->fsync_on_close); o->bs_is_seq_rand =le32_to_cpu (top->bs_is_seq_rand);
@@ -476,6 +479,9 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->job_start_clock_id =cpu_to_le32 (o->job_start_clock_id); top->norandommap =cpu_to_le32 (o->norandommap); top->softrandommap =cpu_to_le32 (o->softrandommap); + top->sprandom =cpu_to_le32 (o->sprandom); + top->spr_num_regions =cpu_to_le32 (o->spr_num_regions); + top->spr_over_provisioning.u.i =__cpu_to_le64 (fio_double_to_uint64 (o->spr_over_provisioning.u.f)); top->bs_unaligned =cpu_to_le32 (o->bs_unaligned); top->fsync_on_close =cpu_to_le32 (o->fsync_on_close); top->bs_is_seq_rand =cpu_to_le32 (o->bs_is_seq_rand);
debug.h +1 -0
@@ -23,6 +23,7 @@ enum {FD_STEADYSTATE ,FD_HELPERTHREAD ,FD_ZBD , +FD_SPRANDOM , FD_DEBUG_MAX, };
examples/sprandom.fio new file +41 -0
41 lines not on this page · raw message
file.h +3 -0
@@ -113,6 +113,9 @@ struct fio_file {uint32_t min_zone;/* inclusive */ uint32_t max_zone;/* exclusive */ +/* SP Random Info */ +struct sprandom_info *spr_info; +/* * Track last end and last start of IO for a given data direction */
filesetup.c +13 -0
@@ -15,6 +15,7 @@#include "lib/axmap.h" #include "rwlock.h" #include "zbd.h" +#include "sprandom.h" #ifdef CONFIG_LINUX_FALLOCATE #include <linux/falloc.h>
@@ -1401,6 +1402,16 @@ done: goto err_out; } +if (td->o.sprandom) { +if (td->o.nr_files !=1 ) { +log_err ("fio: SPRandom supports only one file" ); + goto err_out; + } + err =sprandom_init (td, td->files[0 ]); +if (err) + goto err_out; + } +if (o->create_only) td->done =1 ;
@@ -1590,6 +1601,8 @@ void fio_file_free(struct fio_file *f)axmap_free (f->io_axmap);if (f->ruhs_info)sfree (f->ruhs_info); +if (f->spr_info) +sprandom_free (f->spr_info);if (!fio_file_smalloc (f)) {free (f->file_name);free (f);
fio.1 +28 -0
@@ -1440,6 +1440,34 @@ multiple times. The default value is \fBtausworthe\fR, unless the required space exceeds 2^32 blocks. If it does, then \fBtausworthe64\fR is selected automatically. .RE +.TP +.B sprandom=bool +SPRandom is a method designed to rapidly precondition SSDs for +steady-state random write workloads. It divides the device into +equally sized regions and writes the device's entire physical capacity +once, selecting offsets so that the regions have a distribution of +invalid blocks matching the distribution that occurs at steady state. +Default: false. + +It uses \fBrandom_generator=lfsr\fR, which fio will set by default. +Selecting any other random generator will result in an error. +.TP +.B spr_num_regions=int +See +.BR sprandom . +Specifies the number of regions used for SPRandom. Default=100 +.P +.RS +For large devices it is better to use more regions, to increase precision +and reduce memory allocation. The allocation is proportional to the region size. +.RE +.TP +.B spr_op=float +See +.BR sprandom . +Over-provisioning ratio in the range (0, 1), as specified by the SSD manufacturer. +The default is 0.15. +.RE .SS "Block size" .TP .BI blocksize \fR=\fPint[,int][,int] "\fR,\fB bs" \fR=\fPint[,int][,int]
fio.h +2 -0
@@ -157,6 +157,7 @@ enum {FIO_RAND_PRIO_CMDS ,FIO_RAND_DEDUPE_WORKING_SET_IX ,FIO_RAND_FDP_OFF , +FIO_RAND_SPRANDOM_OFF , FIO_RAND_NR_OFFS, };
@@ -286,6 +287,7 @@ struct thread_data {struct frand_state prio_state;struct frand_state dedupe_working_set_index_state;struct frand_state *dedupe_working_set_states; +struct frand_state sprandom_state;unsigned long long num_unique_pages;
init.c +35 -0
@@ -690,6 +690,36 @@ static int fixup_options(struct thread_data *td)if (o->zone_mode ==ZONE_MODE_STRIDED && !o->zone_range) o->zone_range = o->zone_size; +/* +* SPRandom Requires: random write, random_generator=lfsr, norandommap=1 +*/ +if (o->sprandom) { +if (td_write (td) &&td_random (td)) { +if (fio_option_is_set (o, random_generator)) { +if (o->random_generator !=FIO_RAND_GEN_LFSR ) { +log_err ("fio: sprandom requires random_generator=lfsr\n" ); + ret |=1 ; + } + }else { +log_info ("fio: sprandom sets random_generator=lfsr\n" ); + o->random_generator =FIO_RAND_GEN_LFSR ; + } +if (fio_option_is_set (o, norandommap)) { +if (o->norandommap ==0 ) { +log_err ("fio: sprandom requires norandommap=1\n" ); + ret |=1 ; + } +/* if == 1, OK */ + }else { +log_info ("fio: sprandom sets norandommap=1\n" ); + o->norandommap =1 ; + } + }else { +log_err ("fio: sprandom requires random write, random_generator=lfsr, norandommap=1" ); + ret |=1 ; + } + } +/* * Reads can do overwrites, we always need to pre-create the file */
@@ -1165,6 +1195,7 @@ void td_fill_rand_seeds(struct thread_data *td)frand_copy (&td->buf_state_prev, &td->buf_state);init_rand_seed (&td->fdp_state, td->rand_seeds[FIO_RAND_FDP_OFF ], use64); +init_rand_seed (&td->sprandom_state, td->rand_seeds[FIO_RAND_SPRANDOM_OFF ], false); }static int setup_random_seeds (struct thread_data *td)
@@ -2470,6 +2501,10 @@ const struct debug_level debug_levels[] = { .help = "Zoned Block Device logging", .shift = FD_ZBD, }, + { .name = "sprandom", + .help = "SPRandom logging", + .shift = FD_SPRANDOM, + }, { .name = NULL, }, };
io_u.c +20 -1
@@ -11,6 +11,7 @@#include "lib/pow2.h" #include "minmax.h" #include "zbd.h" +#include "sprandom.h" struct io_completion_data {int nr;/* input */
@@ -84,6 +85,22 @@ static uint64_t last_block(struct thread_data *td, struct fio_file *f,return max_blocks; } + +static int __get_next_rand_offset_sprandom (struct thread_data *td,struct fio_file *f, +enum fio_ddir ddir,uint64_t *b, +uint64_t lastb) +{ +assert (ddir ==DDIR_WRITE ); + +/* SP RANDOM writes all addresses once */ +if (sprandom_get_next_offset (f->spr_info, f, b)) { +dprint (FD_SPRANDOM ,"sprandom is done\n" ); + td->done =1 ; +return 1 ; + } +return 0 ; +} +static int __get_next_rand_offset (struct thread_data *td,struct fio_file *f,enum fio_ddir ddir,uint64_t *b,uint64_t lastb)
@@ -277,7 +294,9 @@ bail:static int get_next_rand_offset (struct thread_data *td,struct fio_file *f,enum fio_ddir ddir,uint64_t *b) { -if (td->o.random_distribution ==FIO_RAND_DIST_RANDOM ) { +if (td->o.sprandom && ddir ==DDIR_WRITE ) { +return __get_next_rand_offset_sprandom (td, f, ddir, b,0 ); + }else if (td->o.random_distribution ==FIO_RAND_DIST_RANDOM ) {uint64_t lastb; lastb =last_block (td, f, ddir);
lib/num2str.c +31 -1
@@ -7,6 +7,37 @@#include "../oslib/asprintf.h" #include "num2str.h" + +static const char *iecstr[] = {"" ,"Ki" ,"Mi" ,"Gi" ,"Ti" ,"Pi" ,"Ei" }; + +/** +* bytes2str_simple - Converts a byte value to a human-readable string. +* @buf: buffer to store the resulting string +* @bufsize: size of the buffer +* @bytes: number of bytes to convert +* @returns : pointer to the buf containing the formatted string. +* Converts the given byte value into a human-readable string using IEC units +* (e.g., KiB, MiB, GiB), and stores the result in the provided buffer. +* The output is formatted with two decimal places of precision. +*/ +const char *bytes2str_simple (char *buf,size_t bufsize,uint64_t bytes) +{ +int unit =0 ; +double size = (double )bytes; + + buf[0 ] ='\0' ; + +while (size >=1024.0 && unit <FIO_ARRAY_SIZE (iecstr) -1 ) { + size /=1024.0 ; + unit++; + } + +snprintf (buf, bufsize,"%.2f %sB" , size, iecstr[unit]); + +return buf; +} + +/** * num2str() - Cheesy number->string conversion, complete with carry rounding error. * @num: quantity (e.g., number of blocks, bytes or bits)
@@ -19,7 +50,6 @@char *num2str (uint64_t num,int maxlen,int base,int pow2,enum n2s_unit units) {const char *sistr[] = {"" ,"k" ,"M" ,"G" ,"T" ,"P" ,"E" }; -const char *iecstr[] = {"" ,"Ki" ,"Mi" ,"Gi" ,"Ti" ,"Pi" ,"Ei" };const char **unitprefix;static const char *const unitstr[] = { [N2S_NONE ] ="" ,
lib/num2str.h +2 -0
@@ -14,4 +14,6 @@ enum n2s_unit {extern char *num2str (uint64_t ,int ,int ,int ,enum n2s_unit ); +extern const char *bytes2str_simple (char *buf,size_t bufsize,uint64_t bytes); +#endif
options.c +37 -0
@@ -2628,6 +2628,43 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .category = FIO_OPT_C_IO, .group = FIO_OPT_G_RANDOM, }, + { + .name = "sprandom", + .lname = "Sandisk Pseudo Random Preconditioning", + .type = FIO_OPT_BOOL, + .off1 = offsetof(struct thread_options, sprandom), + .help = "Set up Sandisk Pseudo Random Preconditioning", + .parent = "rw", + .hide = 1, + .def = "0", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_RANDOM, + }, + { + .name = "spr_num_regions", + .lname = "SPRandom number of regions", + .type = FIO_OPT_INT, + .off1 = offsetof(struct thread_options, spr_num_regions), + .help = "Number of regions for sprandom", + .parent = "sprandom", + .hide = 1, + .def = "100", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_RANDOM, + }, + { + .name = "spr_op", + .lname = "SPRandom Over provisioning", + .type = FIO_OPT_FLOAT_LIST, + .off1 = offsetof(struct thread_options, spr_over_provisioning), + .help = "Over provisioning ratio for SPRandom", + .parent = "sprandom", + .maxlen = 1, + .hide = 1, + .def = "0.15", + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_RANDOM, + }, { .name = "random_generator", .lname = "Random Generator",
pcbuf.h new file +211 -0
211 lines not on this page · raw message
server.h +1 -1
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply { };enum { -FIO_SERVER_VER =112 , +FIO_SERVER_VER =113 ,FIO_SERVER_MAX_FRAGMENT_PDU =1024 ,FIO_SERVER_MAX_CMD_MB =2048 ,
sprandom.c new file +835 -0
835 lines not on this page · raw message
sprandom.h new file +78 -0
78 lines not on this page · raw message
thread_options.h +6 -0
@@ -178,6 +178,9 @@ struct thread_options {unsigned int log_alternate_epoch_clock_id;unsigned int norandommap;unsigned int softrandommap; +unsigned int sprandom; +unsigned int spr_num_regions; +fio_fp64_t spr_over_provisioning;unsigned int bs_unaligned;unsigned int fsync_on_close;unsigned int bs_is_seq_rand;
@@ -510,6 +513,9 @@ struct thread_options_pack {uint32_t log_alternate_epoch_clock_id;uint32_t norandommap;uint32_t softrandommap; +uint32_t sprandom; +uint32_t spr_num_regions; +fio_fp64_t spr_over_provisioning;uint32_t bs_unaligned;uint32_t fsync_on_close;uint32_t bs_is_seq_rand;
unittests/lib/num2str.c +35 -0
@@ -37,11 +37,46 @@ static void test_num2str(void) } } +struct bytes2str_testcase { +uint64_t bytes; +const char *expected; +}; + +static const struct bytes2str_testcase bytes2str_testcases[] = { + {0 ,"0.00 B" }, + {512 ,"512.00 B" }, + {1024 ,"1.00 KiB" }, + {1536 ,"1.50 KiB" }, + {1048576 ,"1.00 MiB" }, + {1073741824ULL ,"1.00 GiB" }, + {1099511627776ULL ,"1.00 TiB" }, + {1125899906842624ULL ,"1.00 PiB" }, + {1152921504606846976ULL ,"1.00 EiB" }, +}; + +static void test_bytes2str_simple (void ) +{ +char buf[64 ]; +int i; + +for (i =0 ; i <FIO_ARRAY_SIZE (bytes2str_testcases); ++i) { +const struct bytes2str_testcase *tc = &bytes2str_testcases[i]; +const char *result =bytes2str_simple (buf,sizeof (buf), tc->bytes); + +CU_ASSERT_PTR_EQUAL (result, buf); +CU_ASSERT_STRING_EQUAL (result, tc->expected); + } +} +static struct fio_unittest_entry tests[] = { { .name ="num2str/1" , .fn = test_num2str, }, + { + .name ="bytes2str_simple/1" , + .fn = test_bytes2str_simple, + }, { .name =NULL , },
unittests/lib/pcbuf.c new file +116 -0
116 lines not on this page · raw message
unittests/unittest.c +1 -0
@@ -50,6 +50,7 @@ int main(void)fio_unittest_register (fio_unittest_lib_memalign);fio_unittest_register (fio_unittest_lib_num2str);fio_unittest_register (fio_unittest_lib_strntol); +fio_unittest_register (fio_unittest_lib_pcbuf);fio_unittest_register (fio_unittest_oslib_strlcat);fio_unittest_register (fio_unittest_oslib_strndup);fio_unittest_register (fio_unittest_oslib_strcasestr);
unittests/unittest.h +1 -0
@@ -17,6 +17,7 @@ CU_ErrorCode fio_unittest_add_suite(const char*, CU_InitializeFunc,CU_ErrorCode fio_unittest_lib_memalign (void );CU_ErrorCode fio_unittest_lib_num2str (void );CU_ErrorCode fio_unittest_lib_strntol (void ); +CU_ErrorCode fio_unittest_lib_pcbuf (void );CU_ErrorCode fio_unittest_oslib_strlcat (void );CU_ErrorCode fio_unittest_oslib_strndup (void );CU_ErrorCode fio_unittest_oslib_strcasestr (void );
verify.c +1 -1
@@ -431,7 +431,7 @@ done:*/ static inline bool pattern_need_buffer (struct thread_data *td) { -return td->o.verify_async && +return (td->o.verify_async || td->o.use_thread) && td->o.verify_fmt_sz && td->o.verify_fmt[0 ].desc->paste == paste_blockoff; }