Skip to contentSkip to conversation

Recent changes (master)

PATCH Jens Axboe 1 participant 1 message Aug 23, 2025

Subsystems (2)
lore
Share & tools
lore

Jens Axboe

author
#0 lore raw

The following changes since commit ac2aa2ca02dec925fa05cb9e9d4a1cd25e78ae84:

Kill of IO engine cancelation support (2025-08-21 16:23:59 -0600)

are available in the Git repository at:

git://git.kernel.dk/fio.git master

for you to fetch changes up to 1fb2d4acde3d7e205cf941167c7efad49baf525c:

verify: use new buffer for threads with %o format (2025-08-22 13:57:09 -0400)

----------------------------------------------------------------
Tomas Winkler (13):
sprandom: add command line options
sprandom: add debug facility
sprandom: examples: add sprandom example file
sprandom: implement region computation and invalidation percentage
sprandom: set up LFSR random generator and disable randommap
num2str: add bytes2str_simple()
unittests: add bytes2str_simple()
sprandom: pcbuf.h add two-phase circular buffer header-only library
unittests: add pcbuf simple unit test
sprandom: initialize random state
sprandom: implement sprandom_get_next_offset()
sprandom: initialize sprandom for file
sprandom: integrate sprandom_get_next_offset() into io_u path

Vincent Fu (3):
Merge branch 'sprandom' of https://github.com/tomas-winkler-sndk/fio
sprandom: abort when invalid options specified
verify: use new buffer for threads with %o format

 HOWTO.rst               |  31 ++
 Makefile                |   4 +-
 cconv.c                 |   6 +
 debug.h                 |   1 +
 examples/sprandom.fio   |  41 +++
 file.h                  |   3 +
 filesetup.c             |  13 +
 fio.1                   |  28 ++
 fio.h                   |   2 +
 init.c                  |  35 ++
 io_u.c                  |  21 +-
 lib/num2str.c           |  32 +-
 lib/num2str.h           |   2 +
 options.c               |  37 +++
 pcbuf.h                 | 211 ++++++++++++
 server.h                |   2 +-
 sprandom.c              | 835 ++++++++++++++++++++++++++++++++++++++++++++++++
 sprandom.h              |  78 +++++
 thread_options.h        |   6 +
 unittests/lib/num2str.c |  35 ++
 unittests/lib/pcbuf.c   | 116 +++++++
 unittests/unittest.c    |   1 +
 unittests/unittest.h    |   1 +
 verify.c                |   2 +-
 24 files changed, 1538 insertions(+), 5 deletions(-)
 create mode 100644 examples/sprandom.fio
 create mode 100644 pcbuf.h
 create mode 100644 sprandom.c
 create mode 100644 sprandom.h
 create mode 100644 unittests/lib/pcbuf.c

---

Diff of recent changes:

HOWTO.rst +31 -0
index 3eb0d9fe..3a4e018f 100644
@@ -1632,6 +1632,37 @@ I/O type
 	space exceeds 2^32 blocks. If it does, then **tausworthe64** is
 	selected automatically.
 
+.. option:: sprandom=bool
+
+
+        SPRandom is a method designed to rapidly precondition SSDs for
+        steady-state random write workloads. It divides the device into
+        equally sized regions and writes the device's entire physical capacity
+        once, selecting offsets so that the regions have a distribution of
+        invalid blocks matching the distribution that occurs at steady state.
+
+        Default: false.
+
+        It uses **random_generator=lfsr**, which fio will set by default.
+        Selecting any other random generator will result in an error.
+
+
+.. option:: spr_num_regions=int
+
+	See :option:`sprandom`. Specifies the number of regions used for SPRandom.
+        For large devices it is better to use more regions, to increase precision
+        and reduce memory allocation. The allocation is proportional to the region size.
+
+        Default=100
+
+
+.. option:: spr_op=float
+
+	See :option:`sprandom`. Over-provisioning ratio in the range (0, 1),
+        as specified by the SSD manufacturer.
+
+        Default=0.15
+
 
 Block size
 ~~~~~~~~~~
Makefile +3 -1
index ec6249f3..ba7ad6dc 100644
@@ -62,7 +62,8 @@ SOURCE :=	$(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
 		gettime-thread.c helpers.c json.c idletime.c td_error.c \
 		profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \
 		workqueue.c rate-submit.c optgroup.c helper_thread.c \
-		steadystate.c zone-dist.c zbd.c dedupe.c dataplacement.c
+		steadystate.c zone-dist.c zbd.c dedupe.c dataplacement.c \
+		sprandom.c
 
 ifdef CONFIG_LIBHDFS
   HDFSFLAGS= -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/linux -I $(FIO_LIBHDFS_INCLUDE)
@@ -448,6 +449,7 @@ UT_OBJS = unittests/unittest.o
 UT_OBJS += unittests/lib/memalign.o
 UT_OBJS += unittests/lib/num2str.o
 UT_OBJS += unittests/lib/strntol.o
+UT_OBJS += unittests/lib/pcbuf.o
 UT_OBJS += unittests/oslib/strlcat.o
 UT_OBJS += unittests/oslib/strndup.o
 UT_OBJS += unittests/oslib/strcasestr.o
cconv.c +6 -0
index 4e72ae16..e7bbfc53 100644
@@ -228,6 +228,9 @@ int convert_thread_options_to_cpu(struct thread_options *o,
 	o->job_start_clock_id = le32_to_cpu(top->job_start_clock_id);
 	o->norandommap = le32_to_cpu(top->norandommap);
 	o->softrandommap = le32_to_cpu(top->softrandommap);
+	o->sprandom = le32_to_cpu(top->sprandom);
+	o->spr_num_regions = le32_to_cpu(top->spr_num_regions);
+	o->spr_over_provisioning.u.f = fio_uint64_to_double(le64_to_cpu(top->spr_over_provisioning.u.i));
 	o->bs_unaligned = le32_to_cpu(top->bs_unaligned);
 	o->fsync_on_close = le32_to_cpu(top->fsync_on_close);
 	o->bs_is_seq_rand = le32_to_cpu(top->bs_is_seq_rand);
@@ -476,6 +479,9 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
 	top->job_start_clock_id = cpu_to_le32(o->job_start_clock_id);
 	top->norandommap = cpu_to_le32(o->norandommap);
 	top->softrandommap = cpu_to_le32(o->softrandommap);
+	top->sprandom = cpu_to_le32(o->sprandom);
+	top->spr_num_regions = cpu_to_le32(o->spr_num_regions);
+	top->spr_over_provisioning.u.i = __cpu_to_le64(fio_double_to_uint64(o->spr_over_provisioning.u.f));
 	top->bs_unaligned = cpu_to_le32(o->bs_unaligned);
 	top->fsync_on_close = cpu_to_le32(o->fsync_on_close);
 	top->bs_is_seq_rand = cpu_to_le32(o->bs_is_seq_rand);
debug.h +1 -0
index 51b18de2..49a8791d 100644
@@ -23,6 +23,7 @@ enum {
 	FD_STEADYSTATE,
 	FD_HELPERTHREAD,
 	FD_ZBD,
+	FD_SPRANDOM,
 	FD_DEBUG_MAX,
 };
 
examples/sprandom.fio new file +41 -0

41 lines not on this page · raw message

file.h +3 -0
index 8fd40cdf..f400155f 100644
@@ -113,6 +113,9 @@ struct fio_file {
 	uint32_t min_zone;	/* inclusive */
 	uint32_t max_zone;	/* exclusive */
 
+	/* SP Random Info */
+	struct sprandom_info *spr_info;
+
 	/*
 	 * Track last end and last start of IO for a given data direction
 	 */
filesetup.c +13 -0
index bcbd871e..597bf4c5 100644
@@ -15,6 +15,7 @@
 #include "lib/axmap.h"
 #include "rwlock.h"
 #include "zbd.h"
+#include "sprandom.h"
 
 #ifdef CONFIG_LINUX_FALLOCATE
 #include <linux/falloc.h>
@@ -1401,6 +1402,16 @@ done:
 			goto err_out;
 	}
 
+	if (td->o.sprandom) {
+		if (td->o.nr_files != 1) {
+			 log_err("fio: SPRandom supports only one file");
+			 goto err_out;
+		}
+		err = sprandom_init(td, td->files[0]);
+		if (err)
+			goto err_out;
+	}
+
 	if (o->create_only)
 		td->done = 1;
 
@@ -1590,6 +1601,8 @@ void fio_file_free(struct fio_file *f)
 		axmap_free(f->io_axmap);
 	if (f->ruhs_info)
 		sfree(f->ruhs_info);
+	if (f->spr_info)
+		sprandom_free(f->spr_info);
 	if (!fio_file_smalloc(f)) {
 		free(f->file_name);
 		free(f);
fio.1 +28 -0
index 6aa23f7d..d9f4fd15 100644
@@ -1440,6 +1440,34 @@ multiple times. The default value is \fBtausworthe\fR, unless the required
 space exceeds 2^32 blocks. If it does, then \fBtausworthe64\fR is
 selected automatically.
 .RE
+.TP
+.B sprandom=bool
+SPRandom is a method designed to rapidly precondition SSDs for
+steady-state random write workloads. It divides the device into
+equally sized regions and writes the device's entire physical capacity
+once, selecting offsets so that the regions have a distribution of
+invalid blocks matching the distribution that occurs at steady state.
+Default: false.
+
+It uses \fBrandom_generator=lfsr\fR, which fio will set by default.
+Selecting any other random generator will result in an error.
+.TP
+.B spr_num_regions=int
+See
+.BR sprandom .
+Specifies the number of regions used for SPRandom. Default=100
+.P
+.RS
+For large devices it is better to use more regions, to increase precision
+and reduce memory allocation. The allocation is proportional to the region size.
+.RE
+.TP
+.B spr_op=float
+See
+.BR sprandom .
+Over-provisioning ratio in the range (0, 1), as specified by the SSD manufacturer.
+The default is 0.15.
+.RE
 .SS "Block size"
 .TP
 .BI blocksize \fR=\fPint[,int][,int] "\fR,\fB bs" \fR=\fPint[,int][,int]
fio.h +2 -0
index e11b9261..44788899 100644
@@ -157,6 +157,7 @@ enum {
 	FIO_RAND_PRIO_CMDS,
 	FIO_RAND_DEDUPE_WORKING_SET_IX,
 	FIO_RAND_FDP_OFF,
+	FIO_RAND_SPRANDOM_OFF,
 	FIO_RAND_NR_OFFS,
 };
 
@@ -286,6 +287,7 @@ struct thread_data {
 	struct frand_state prio_state;
 	struct frand_state dedupe_working_set_index_state;
 	struct frand_state *dedupe_working_set_states;
+	struct frand_state sprandom_state;
 
 	unsigned long long num_unique_pages;
 
init.c +35 -0
index 20f5462d..cf66ac2c 100644
@@ -690,6 +690,36 @@ static int fixup_options(struct thread_data *td)
 	if (o->zone_mode == ZONE_MODE_STRIDED && !o->zone_range)
 		o->zone_range = o->zone_size;
 
+	/*
+	 * SPRandom Requires: random write, random_generator=lfsr, norandommap=1
+	 */
+	if (o->sprandom) {
+		if (td_write(td) && td_random(td)) {
+			if (fio_option_is_set(o, random_generator)) {
+				if (o->random_generator != FIO_RAND_GEN_LFSR) {
+					log_err("fio: sprandom requires random_generator=lfsr\n");
+					ret |= 1;
+				}
+			} else {
+				log_info("fio: sprandom sets random_generator=lfsr\n");
+				o->random_generator = FIO_RAND_GEN_LFSR;
+			}
+			if (fio_option_is_set(o, norandommap)) {
+				if (o->norandommap == 0) {
+					log_err("fio: sprandom requires norandommap=1\n");
+					ret |= 1;
+				}
+				/* if == 1, OK */
+			} else {
+				log_info("fio: sprandom sets norandommap=1\n");
+				o->norandommap = 1;
+			}
+		} else {
+			log_err("fio: sprandom requires random write, random_generator=lfsr, norandommap=1");
+			ret |= 1;
+		}
+	}
+
 	/*
 	 * Reads can do overwrites, we always need to pre-create the file
 	 */
@@ -1165,6 +1195,7 @@ void td_fill_rand_seeds(struct thread_data *td)
 	frand_copy(&td->buf_state_prev, &td->buf_state);
 
 	init_rand_seed(&td->fdp_state, td->rand_seeds[FIO_RAND_FDP_OFF], use64);
+	init_rand_seed(&td->sprandom_state, td->rand_seeds[FIO_RAND_SPRANDOM_OFF], false);
 }
 
 static int setup_random_seeds(struct thread_data *td)
@@ -2470,6 +2501,10 @@ const struct debug_level debug_levels[] = {
 	  .help = "Zoned Block Device logging",
 	  .shift = FD_ZBD,
 	},
+	{ .name = "sprandom",
+	  .help = "SPRandom logging",
+	  .shift = FD_SPRANDOM,
+	},
 	{ .name = NULL, },
 };
 
io_u.c +20 -1
index 78dbac9c..ab1bf7c9 100644
@@ -11,6 +11,7 @@
 #include "lib/pow2.h"
 #include "minmax.h"
 #include "zbd.h"
+#include "sprandom.h"
 
 struct io_completion_data {
 	int nr;				/* input */
@@ -84,6 +85,22 @@ static uint64_t last_block(struct thread_data *td, struct fio_file *f,
 	return max_blocks;
 }
 
+
+static int __get_next_rand_offset_sprandom(struct thread_data *td, struct fio_file *f,
+					   enum fio_ddir ddir, uint64_t *b,
+					   uint64_t lastb)
+{
+	assert(ddir == DDIR_WRITE);
+
+	/* SP RANDOM writes all addresses once */
+	if (sprandom_get_next_offset(f->spr_info, f, b)) {
+		dprint(FD_SPRANDOM, "sprandom is done\n");
+		td->done = 1;
+		return 1;
+	}
+	return 0;
+}
+
 static int __get_next_rand_offset(struct thread_data *td, struct fio_file *f,
 				  enum fio_ddir ddir, uint64_t *b,
 				  uint64_t lastb)
@@ -277,7 +294,9 @@ bail:
 static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
 				enum fio_ddir ddir, uint64_t *b)
 {
-	if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) {
+	if (td->o.sprandom && ddir == DDIR_WRITE) {
+		return __get_next_rand_offset_sprandom(td, f, ddir, b, 0);
+	} else if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) {
 		uint64_t lastb;
 
 		lastb = last_block(td, f, ddir);
lib/num2str.c +31 -1
index cd89a0e5..e48a483c 100644
@@ -7,6 +7,37 @@
 #include "../oslib/asprintf.h"
 #include "num2str.h"
 
+
+static const char *iecstr[] = { "", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei" };
+
+/**
+ * bytes2str_simple - Converts a byte value to a human-readable string.
+ * @buf:      buffer to store the resulting string
+ * @bufsize:  size of the buffer
+ * @bytes:    number of bytes to convert
+ * @returns : pointer to the buf containing the formatted string.
+ * Converts the given byte value into a human-readable string using IEC units
+ * (e.g., KiB, MiB, GiB), and stores the result in the provided buffer.
+ * The output is formatted with two decimal places of precision.
+ */
+const char *bytes2str_simple(char *buf, size_t bufsize, uint64_t bytes)
+{
+	int unit = 0;
+	double size = (double)bytes;
+
+	buf[0] = '\0';
+
+	while (size >= 1024.0 && unit < FIO_ARRAY_SIZE(iecstr) - 1) {
+		size /= 1024.0;
+		unit++;
+	}
+
+	snprintf(buf, bufsize, "%.2f %sB", size, iecstr[unit]);
+
+	return buf;
+}
+
+
 /**
  * num2str() - Cheesy number->string conversion, complete with carry rounding error.
  * @num: quantity (e.g., number of blocks, bytes or bits)
@@ -19,7 +50,6 @@
 char *num2str(uint64_t num, int maxlen, int base, int pow2, enum n2s_unit units)
 {
 	const char *sistr[] = { "", "k", "M", "G", "T", "P", "E" };
-	const char *iecstr[] = { "", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei" };
 	const char **unitprefix;
 	static const char *const unitstr[] = {
 		[N2S_NONE]	= "",
lib/num2str.h +2 -0
index 797288b5..8ca2050a 100644
@@ -14,4 +14,6 @@ enum n2s_unit {
 
 extern char *num2str(uint64_t, int, int, int, enum n2s_unit);
 
+extern const char *bytes2str_simple(char *buf, size_t bufsize, uint64_t bytes);
+
 #endif
options.c +37 -0
index 6295a616..337a3a52 100644
@@ -2628,6 +2628,43 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
 		.category = FIO_OPT_C_IO,
 		.group	= FIO_OPT_G_RANDOM,
 	},
+	{
+		.name	= "sprandom",
+		.lname	= "Sandisk Pseudo Random Preconditioning",
+		.type	= FIO_OPT_BOOL,
+		.off1	= offsetof(struct thread_options, sprandom),
+		.help	= "Set up Sandisk Pseudo Random Preconditioning",
+		.parent	= "rw",
+		.hide	= 1,
+		.def	= "0",
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_RANDOM,
+	},
+	{
+		.name	= "spr_num_regions",
+		.lname	= "SPRandom number of regions",
+		.type	= FIO_OPT_INT,
+		.off1	= offsetof(struct thread_options, spr_num_regions),
+		.help	= "Number of regions for sprandom",
+		.parent	= "sprandom",
+		.hide	= 1,
+		.def    = "100",
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_RANDOM,
+	},
+	{
+		.name	= "spr_op",
+		.lname	= "SPRandom Over provisioning",
+		.type	= FIO_OPT_FLOAT_LIST,
+		.off1	= offsetof(struct thread_options, spr_over_provisioning),
+		.help	= "Over provisioning ratio for SPRandom",
+		.parent	= "sprandom",
+		.maxlen = 1,
+		.hide	= 1,
+		.def    = "0.15",
+		.category = FIO_OPT_C_IO,
+		.group	= FIO_OPT_G_RANDOM,
+	},
 	{
 		.name	= "random_generator",
 		.lname	= "Random Generator",
pcbuf.h new file +211 -0

211 lines not on this page · raw message

server.h +1 -1
index f0b15a22..5246af5c 100644
@@ -51,7 +51,7 @@ struct fio_net_cmd_reply {
 };
 
 enum {
-	FIO_SERVER_VER			= 112,
+	FIO_SERVER_VER			= 113,
 
 	FIO_SERVER_MAX_FRAGMENT_PDU	= 1024,
 	FIO_SERVER_MAX_CMD_MB		= 2048,
sprandom.c new file +835 -0

835 lines not on this page · raw message

sprandom.h new file +78 -0

78 lines not on this page · raw message

thread_options.h +6 -0
index 1b26ab58..3abce731 100644
@@ -178,6 +178,9 @@ struct thread_options {
 	unsigned int log_alternate_epoch_clock_id;
 	unsigned int norandommap;
 	unsigned int softrandommap;
+	unsigned int sprandom;
+	unsigned int spr_num_regions;
+	fio_fp64_t spr_over_provisioning;
 	unsigned int bs_unaligned;
 	unsigned int fsync_on_close;
 	unsigned int bs_is_seq_rand;
@@ -510,6 +513,9 @@ struct thread_options_pack {
 	uint32_t log_alternate_epoch_clock_id;
 	uint32_t norandommap;
 	uint32_t softrandommap;
+	uint32_t sprandom;
+	uint32_t spr_num_regions;
+	fio_fp64_t spr_over_provisioning;
 	uint32_t bs_unaligned;
 	uint32_t fsync_on_close;
 	uint32_t bs_is_seq_rand;
unittests/lib/num2str.c +35 -0
index 8f12cf83..49e80346 100644
@@ -37,11 +37,46 @@ static void test_num2str(void)
 	}
 }
 
+struct bytes2str_testcase {
+        uint64_t bytes;
+        const char *expected;
+};
+
+static const struct bytes2str_testcase bytes2str_testcases[] = {
+        { 0, "0.00 B" },
+        { 512, "512.00 B" },
+        { 1024, "1.00 KiB" },
+        { 1536, "1.50 KiB" },
+        { 1048576, "1.00 MiB" },
+        { 1073741824ULL, "1.00 GiB" },
+        { 1099511627776ULL, "1.00 TiB" },
+        { 1125899906842624ULL, "1.00 PiB" },
+        { 1152921504606846976ULL, "1.00 EiB" },
+};
+
+static void test_bytes2str_simple(void)
+{
+        char buf[64];
+        int i;
+
+        for (i = 0; i < FIO_ARRAY_SIZE(bytes2str_testcases); ++i) {
+                const struct bytes2str_testcase *tc = &bytes2str_testcases[i];
+                const char *result = bytes2str_simple(buf, sizeof(buf), tc->bytes);
+
+                CU_ASSERT_PTR_EQUAL(result, buf);
+                CU_ASSERT_STRING_EQUAL(result, tc->expected);
+        }
+}
+
 static struct fio_unittest_entry tests[] = {
 	{
 		.name	= "num2str/1",
 		.fn	= test_num2str,
 	},
+	{
+                .name   = "bytes2str_simple/1",
+                .fn     = test_bytes2str_simple,
+        },
 	{
 		.name	= NULL,
 	},
unittests/lib/pcbuf.c new file +116 -0

116 lines not on this page · raw message

unittests/unittest.c +1 -0
index f490b485..4a034b40 100644
@@ -50,6 +50,7 @@ int main(void)
 	fio_unittest_register(fio_unittest_lib_memalign);
 	fio_unittest_register(fio_unittest_lib_num2str);
 	fio_unittest_register(fio_unittest_lib_strntol);
+	fio_unittest_register(fio_unittest_lib_pcbuf);
 	fio_unittest_register(fio_unittest_oslib_strlcat);
 	fio_unittest_register(fio_unittest_oslib_strndup);
 	fio_unittest_register(fio_unittest_oslib_strcasestr);
unittests/unittest.h +1 -0
index ecb7d124..0f45bfbd 100644
@@ -17,6 +17,7 @@ CU_ErrorCode fio_unittest_add_suite(const char*, CU_InitializeFunc,
 CU_ErrorCode fio_unittest_lib_memalign(void);
 CU_ErrorCode fio_unittest_lib_num2str(void);
 CU_ErrorCode fio_unittest_lib_strntol(void);
+CU_ErrorCode fio_unittest_lib_pcbuf(void);
 CU_ErrorCode fio_unittest_oslib_strlcat(void);
 CU_ErrorCode fio_unittest_oslib_strndup(void);
 CU_ErrorCode fio_unittest_oslib_strcasestr(void);
verify.c +1 -1
index c7f43c06..20c49a94 100644
@@ -431,7 +431,7 @@ done:
  */
 static inline bool pattern_need_buffer(struct thread_data *td)
 {
-	return td->o.verify_async &&
+	return (td->o.verify_async || td->o.use_thread) &&
 		td->o.verify_fmt_sz &&
 		td->o.verify_fmt[0].desc->paste == paste_blockoff;
 }