Skip to contentSkip to conversation

Protection information support for io_uring passthrough

PATCH Superseded Ankit Kumar 1 participant 11 messages Aug 11, 2023 1 message not in the archive

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): CGME20230811060633epcas5p2afd7e23260a8bebbe91c4ae87b25f6c7@epcas5p2.samsung.com

#1 lore raw

v2 cover Protection information support for io_uring passthrough

This patch series enables protection information support with io_uring passthrough (io_uring_cmd) ioengine. This support is added for both DIF (extended logical blocks) and DIX (separate metadata buffer) cases.

The initial patch adds support for metadata buffer. As we are unaware of metadata size during metadata buffer allocation, we provide an option md_per_io_size. This must be used to specify metadata size for single IO, if namespace is formatted with separate metadata buffer.

This then enables the protection information support with and without PRACT bit set for both DIF and DIX cases. The support covers 16-bit and 64-bit guard protection information format, without storage tags. As 32-bit guard protection information format mandates minimum 16 bits for storage tags, it is not considered.

For the sake of consistency metadata and protection information specific options are the same as the ones used by SPDK's external ioengine.

This series also borrows relevant kernel crc16-t10 and NVMe CRC64 generation files, along with 48-bit accessors for reference tags.

The NVMe CRC64 table is not generated at build time which is what kernel does. This will be done in future.

Changes from v1:
- Fixed build issue on windows and macos.
- Changed name of CRC64 tables and algorithm as per NVMe command set specification (Keith's suggestion).
- Addressed review comments from Vincent.

Ankit Kumar (10):
engines:io_uring: add missing error during open file engines:io_uring: update arguments to fetch nvme data engines:io_uring: enable support for separate metadata buffer engines:io_uring: uring_cmd add support for protection info
io_u: move engine data out of union
crc: pull required crc16-t10 files from linux kernel engines:io_uring: generate and verify pi for 16b guard
crc: pull required crc64 nvme apis from linux kernel engines:nvme: pull required 48 bit accessors from linux kernel engines:io_uring: generate and verify pi for 64b guard

 HOWTO.rst              |  39 ++++
 crc/crc-t10dif.h       |  10 +
 crc/crc64.c            |  32 +++
 crc/crc64.h            |   3 +
 crc/crc64table.h       | 130 ++++++++++++
 crc/crct10dif_common.c |  78 +++++++
 engines/io_uring.c     | 225 +++++++++++++++++---
 engines/nvme.c         | 466 +++++++++++++++++++++++++++++++++++++++--
 engines/nvme.h         | 230 +++++++++++++++++++-
 fio.1                  |  38 ++++
 io_u.h                 |   2 +-
 11 files changed, 1200 insertions(+), 53 deletions(-)
 create mode 100644 crc/crc-t10dif.h
 create mode 100644 crc/crc64table.h
 create mode 100644 crc/crct10dif_common.c
Signature · 1 line
2.25.1
#2 lore raw

PATCH v2 1/10 engines:io_uring: add missing error during open file

This change ensures the error is propogated to upper layers to make fio exit with a non-zero return code.
Add filename for errors when block size is not a multiple of logical blocks.

---
 engines/io_uring.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
engines/io_uring.c +5 -3
index b361e6a5..09065463 100644
@@ -1115,10 +1115,12 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
 			if (td->o.min_bs[ddir] % lba_size ||
 				td->o.max_bs[ddir] % lba_size) {
 				if (data->lba_ext)
-					log_err("block size must be a multiple of "
-						"(LBA data size + Metadata size)\n");
+					log_err("%s: block size must be a multiple of (LBA data size + Metadata size)\n",
+						f->file_name);
 				else
-					log_err("block size must be a multiple of LBA data size\n");
+					log_err("%s: block size must be a multiple of LBA data size\n",
+						f->file_name);
+				td_verror(td, EINVAL, "fio_ioring_cmd_open_file");
 				return 1;
 			}
                 }
Signature · 1 line
2.25.1
#3 lore raw

PATCH v2 2/10 engines:io_uring: update arguments to fetch nvme data

In reply to Ankit Kumar

This is a prep patch to keep number of arguments for fio_nvme_get_info in check. The follow up patches will enable metadata, protection info.

---
 engines/io_uring.c | 38 +++++++++++++-------------------------
 engines/nvme.c     | 16 ++++++++++------
 engines/nvme.h     |  5 +++--
 3 files changed, 26 insertions(+), 33 deletions(-)
engines/io_uring.c +13 -25
index 09065463..30d9ccd7 100644
@@ -1086,30 +1086,24 @@ static int fio_ioring_cmd_open_file(struct thread_data *td, struct fio_file *f)
 
 	if (o->cmd_type == FIO_URING_CMD_NVME) {
 		struct nvme_data *data = NULL;
-		unsigned int nsid, lba_size = 0;
-		__u32 ms = 0;
+		unsigned int lba_size = 0;
 		__u64 nlba = 0;
 		int ret;
 
 		/* Store the namespace-id and lba size. */
 		data = FILE_ENG_DATA(f);
 		if (data == NULL) {
-			ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba);
-			if (ret)
-				return ret;
-
 			data = calloc(1, sizeof(struct nvme_data));
-			data->nsid = nsid;
-			if (ms)
-				data->lba_ext = lba_size + ms;
-			else
-				data->lba_shift = ilog2(lba_size);
+			ret = fio_nvme_get_info(f, &nlba, data);
+			if (ret) {
+				free(data);
+				return ret;
+			}
 
 			FILE_SET_ENG_DATA(f, data);
 		}
 
-		assert(data->lba_shift < 32);
-		lba_size = data->lba_ext ? data->lba_ext : (1U << data->lba_shift);
+		lba_size = data->lba_ext ? data->lba_ext : data->lba_size;
 
 		for_each_rw_ddir(ddir) {
 			if (td->o.min_bs[ddir] % lba_size ||
@@ -1173,23 +1167,17 @@ static int fio_ioring_cmd_get_file_size(struct thread_data *td,
 
 	if (o->cmd_type == FIO_URING_CMD_NVME) {
 		struct nvme_data *data = NULL;
-		unsigned int nsid, lba_size = 0;
-		__u32 ms = 0;
 		__u64 nlba = 0;
 		int ret;
 
-		ret = fio_nvme_get_info(f, &nsid, &lba_size, &ms, &nlba);
-		if (ret)
-			return ret;
-
 		data = calloc(1, sizeof(struct nvme_data));
-		data->nsid = nsid;
-		if (ms)
-			data->lba_ext = lba_size + ms;
-		else
-			data->lba_shift = ilog2(lba_size);
+		ret = fio_nvme_get_info(f, &nlba, data);
+		if (ret) {
+			free(data);
+			return ret;
+		}
 
-		f->real_file_size = lba_size * nlba;
+		f->real_file_size = data->lba_size * nlba;
 		fio_file_set_size_known(f);
 
 		FILE_SET_ENG_DATA(f, data);
engines/nvme.c +10 -6
index b18ad4c2..7e891eed 100644
@@ -99,8 +99,7 @@ static int nvme_identify(int fd, __u32 nsid, enum nvme_identify_cns cns,
 	return ioctl(fd, NVME_IOCTL_ADMIN_CMD, &cmd);
 }
 
-int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
-		      __u32 *ms, __u64 *nlba)
+int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, struct nvme_data *data)
 {
 	struct nvme_id_ns ns;
 	int namespace_id;
@@ -137,7 +136,7 @@ int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
 		return err;
 	}
 
-	*nsid = namespace_id;
+	data->nsid = namespace_id;
 
 	/*
 	 * 16 or 64 as maximum number of supported LBA formats.
@@ -149,21 +148,26 @@ int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
 	else
 		format_idx = (ns.flbas & 0xf) + (((ns.flbas >> 5) & 0x3) << 4);
 
-	*lba_sz = 1 << ns.lbaf[format_idx].ds;
+	data->lba_size = 1 << ns.lbaf[format_idx].ds;
 
 	/*
 	 * Only extended LBA can be supported.
 	 * Bit 4 for flbas indicates if metadata is transferred at the end of
 	 * logical block creating an extended LBA.
 	 */
-	*ms = le16_to_cpu(ns.lbaf[format_idx].ms);
-	if (*ms && !((ns.flbas >> 4) & 0x1)) {
+	data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);
+	if (data->ms && !((ns.flbas >> 4) & 0x1)) {
 		log_err("%s: only extended logical block can be supported\n",
 			f->file_name);
 		err = -ENOTSUP;
 		goto out;
 	}
 
+	if (data->ms)
+		data->lba_ext = data->lba_size + data->ms;
+	else
+		data->lba_shift = ilog2(data->lba_size);
+
 	/* Check for end to end data protection support */
 	if (ns.dps & 0x3) {
 		log_err("%s: end to end data protection not supported\n",
engines/nvme.h +3 -2
index 238471dd..e742b14f 100644
@@ -88,7 +88,9 @@ enum nvme_zns_zs {
 struct nvme_data {
 	__u32 nsid;
 	__u32 lba_shift;
+	__u32 lba_size;
 	__u32 lba_ext;
+	__u16 ms;
 };
 
 struct nvme_lbaf {
@@ -219,8 +221,7 @@ struct nvme_dsm_range {
 int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
 			 struct nvme_fdp_ruh_status *ruhs, __u32 bytes);
 
-int fio_nvme_get_info(struct fio_file *f, __u32 *nsid, __u32 *lba_sz,
-		      __u32 *ms, __u64 *nlba);
+int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, struct nvme_data *data);
 
 int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 			    struct iovec *iov, struct nvme_dsm_range *dsm);
Signature · 1 line
2.25.1
#4 lore raw

PATCH v2 3/10 engines:io_uring: enable support for separate metadata buffer

In reply to Ankit Kumar

This patch enables support for separate metadata buffer with io_uring_cmd ioengine. As we are unaware of metadata size during buffer allocation, we provide an option md_per_io_size. This option must be used to specify metadata buffer size for single IO, if namespace is formatted with a separate metadata buffer.

For the sake of consistency this is the same option as used by SPDK's external ioengine.

---
 HOWTO.rst          |  4 ++++
 engines/io_uring.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++
 engines/nvme.c     | 16 ++++++--------
 fio.1              |  3 +++
 4 files changed, 67 insertions(+), 10 deletions(-)
HOWTO.rst +4 -0
index ac8314f3..6e0677f2 100644
@@ -2487,6 +2487,10 @@ with the caveat that when used on the command line, they must come after the
         want fio to use placement identifier only at indices 0, 2 and 5 specify
         ``fdp_pli=0,2,5``.
 
+.. option:: md_per_io_size=int : [io_uring_cmd]
+
+	Size in bytes for separate metadata buffer per IO. Default: 0.
+
 .. option:: cpuload=int : [cpuio]
 
 	Attempt to use the specified percentage of CPU cycles. This is a mandatory
engines/io_uring.c +54 -0

104 lines not on this page · raw message

engines/nvme.c +6 -10
index 7e891eed..65725e3c 100644
@@ -79,6 +79,10 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		cmd->addr = (__u64)(uintptr_t)io_u->xfer_buf;
 		cmd->data_len = io_u->xfer_buflen;
 	}
+	if (data->lba_shift && data->ms) {
+		cmd->metadata = (__u64)(uintptr_t)io_u->mmap_data;
+		cmd->metadata_len = (nlb + 1) * data->ms;
+	}
 	cmd->nsid = data->nsid;
 	return 0;
 }
@@ -149,21 +153,13 @@ int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, struct nvme_data *data)
 		format_idx = (ns.flbas & 0xf) + (((ns.flbas >> 5) & 0x3) << 4);
 
 	data->lba_size = 1 << ns.lbaf[format_idx].ds;
+	data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);
 
 	/*
-	 * Only extended LBA can be supported.
 	 * Bit 4 for flbas indicates if metadata is transferred at the end of
 	 * logical block creating an extended LBA.
 	 */
-	data->ms = le16_to_cpu(ns.lbaf[format_idx].ms);
-	if (data->ms && !((ns.flbas >> 4) & 0x1)) {
-		log_err("%s: only extended logical block can be supported\n",
-			f->file_name);
-		err = -ENOTSUP;
-		goto out;
-	}
-
-	if (data->ms)
+	if (data->ms && ((ns.flbas >> 4) & 0x1))
 		data->lba_ext = data->lba_size + data->ms;
 	else
 		data->lba_shift = ilog2(data->lba_size);
fio.1 +3 -0
index f62617e7..6b49a747 100644
@@ -2247,6 +2247,9 @@ By default, the job will cycle through all available Placement IDs, so use this
 to isolate these identifiers to specific jobs. If you want fio to use placement
 identifier only at indices 0, 2 and 5 specify, you would set `fdp_pli=0,2,5`.
 .TP
+.BI (io_uring_cmd)md_per_io_size \fR=\fPint
+Size in bytes for separate metadata buffer per IO. Default: 0.
+.TP
 .BI (cpuio)cpuload \fR=\fPint
 Attempt to use the specified percentage of CPU cycles. This is a mandatory
 option when using cpuio I/O engine.
Signature · 1 line
2.25.1
#5 lore raw

PATCH v2 4/10 engines:io_uring: uring_cmd add support for protection info

In reply to Ankit Kumar

This patch enables support for protection information to nvme command backend of io_uring_cmd ioengine. The patch only supports protection information action bit set to 1, for read and write operation.

This adds 4 new ioengine specific options
* pi_act - Protection information action. Default: 1
* pi_chk - Can be set to GUARD, APPTAG or REFTAG
* apptag - Sets apptag field of command dword 15
* apptag_mask - Sets apptag_mask field of command dword 15

For the sake of consistency these options are the same as the ones used by SPDK's external ioengine.

For pi_act=1, if namespace is formatted with metadata size equal to protection information size, the nvme controller inserts and removes protection information for write and read command respectively. Added a check so that fio doesn't send metadata for such cases.

Storage tag support is not present, so return an error for that.

---
 HOWTO.rst          |  35 +++++++++
 engines/io_uring.c |  95 +++++++++++++++++++++-
 engines/nvme.c     | 112 +++++++++++++++++++++++---
 engines/nvme.h     | 191 ++++++++++++++++++++++++++++++++++++++++++++-
 fio.1              |  35 +++++++++
 5 files changed, 454 insertions(+), 14 deletions(-)
HOWTO.rst +35 -0
index 6e0677f2..89032941 100644
@@ -2491,6 +2491,41 @@ with the caveat that when used on the command line, they must come after the
 
 	Size in bytes for separate metadata buffer per IO. Default: 0.
 
+.. option:: pi_act=int : [io_uring_cmd]
+
+	Action to take when nvme namespace is formatted with protection
+	information. If this is set to 1 and namespace is formatted with
+	metadata size equal to protection information size, fio won't use
+	separate metadata buffer or extended logical block. If this is set to
+	1 and namespace is formatted with metadata size greater than protection
+	information size, fio will not generate or verify the protection
+	information portion of metadata for write or read case respectively.
+	If this is set to 0, fio generates protection information for
+	write case and verifies for read case. Default: 1.
+
+.. option:: pi_chk=str[,str][,str] : [io_uring_cmd]
+
+	Controls the protection information check. This can take one or more
+	of these values. Default: none.
+
+	**GUARD**
+		Enables protection information checking of guard field.
+	**REFTAG**
+		Enables protection information checking of logical block
+		reference tag field.
+	**APPTAG**
+		Enables protection information checking of application tag field.
+
+.. option:: apptag=int : [io_uring_cmd]
+
+	Specifies logical block application tag value, if namespace is
+	formatted to use end to end protection information. Default: 0x1234.
+
+.. option:: apptag_mask=int : [io_uring_cmd]
+
+	Specifies logical block application tag mask value, if namespace is
+	formatted to use end to end protection information. Default: 0xffff.
+
 .. option:: cpuload=int : [cpuio]
 
 	Attempt to use the specified percentage of CPU cycles. This is a mandatory
engines/io_uring.c +93 -2

149 lines not on this page · raw message

engines/nvme.c +101 -11

152 lines not on this page · raw message

engines/nvme.h +190 -1

240 lines not on this page · raw message

fio.1 +35 -0
index 6b49a747..f0dc49ab 100644
@@ -2250,6 +2250,41 @@ identifier only at indices 0, 2 and 5 specify, you would set `fdp_pli=0,2,5`.
 .BI (io_uring_cmd)md_per_io_size \fR=\fPint
 Size in bytes for separate metadata buffer per IO. Default: 0.
 .TP
+.BI (io_uring_cmd)pi_act \fR=\fPint
+Action to take when nvme namespace is formatted with protection information.
+If this is set to 1 and namespace is formatted with metadata size equal to
+protection information size, fio won't use separate metadata buffer or extended
+logical block. If this is set to 1 and namespace is formatted with metadata
+size greater than protection information size, fio will not generate or verify
+the protection information portion of metadata for write or read case
+respectively. If this is set to 0, fio generates protection information for
+write case and verifies for read case. Default: 1.
+.TP
+.BI (io_uring_cmd)pi_chk \fR=\fPstr[,str][,str]
+Controls the protection information check. This can take one or more of these
+values. Default: none.
+.RS
+.RS
+.TP
+.B GUARD
+Enables protection information checking of guard field.
+.TP
+.B REFTAG
+Enables protection information checking of logical block reference tag field.
+.TP
+.B APPTAG
+Enables protection information checking of application tag field.
+.RE
+.RE
+.TP
+.BI (io_uring_cmd)apptag \fR=\fPint
+Specifies logical block application tag value, if namespace is formatted to use
+end to end protection information. Default: 0x1234.
+.TP
+.BI (io_uring_cmd)apptag_mask \fR=\fPint
+Specifies logical block application tag mask value, if namespace is formatted
+to use end to end protection information. Default: 0xffff.
+.TP
 .BI (cpuio)cpuload \fR=\fPint
 Attempt to use the specified percentage of CPU cycles. This is a mandatory
 option when using cpuio I/O engine.
Signature · 1 line
2.25.1
#6 lore raw

PATCH v2 5/10 io_u: move engine data out of union

In reply to Ankit Kumar

io_uring_cmd ioengine requires engine data to store nvme protection information data.

---
 io_u.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
io_u.h +1 -1
index b432a540..786251d5 100644
@@ -89,8 +89,8 @@ struct io_u {
 	union {
 		unsigned int index;
 		unsigned int seen;
-		void *engine_data;
 	};
+	void *engine_data;
 
 	union {
 		struct flist_head verify_list;
Signature · 1 line
2.25.1
#7 lore raw

PATCH v2 6/10 crc: pull required crc16-t10 files from linux kernel

In reply to Ankit Kumar

Pull the required crc16 t10 files from the linux kernel. This is required to generate and verify guard tag for nvme backend of io_uring_cmd ioengine.

---
 crc/crc-t10dif.h       | 10 ++++++
 crc/crct10dif_common.c | 78 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 crc/crc-t10dif.h
 create mode 100644 crc/crct10dif_common.c
crc/crc-t10dif.h new file +10 -0
new file mode 100644
index 00000000..22121244
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __CRC_T10DIF_H
+#define __CRC_T10DIF_H
+
+#include <linux/types.h>
+
+extern __u16 fio_crc_t10dif_generic(__u16 crc, const unsigned char *buffer,
+				    unsigned int len);
+
+#endif
crc/crct10dif_common.c new file +78 -0

78 lines not on this page · raw message

Signature · 1 line
2.25.1
#8 lore raw

PATCH v2 7/10 engines:io_uring: generate and verify pi for 16b guard

In reply to Ankit Kumar

Generate and verify protection information for 16 bit guard format, for the nvme backend of io_uring_cmd ioengine. The support is there for both the cases where metadata is transferred in separate buffer, or transferred at the end of logical block creating an extended logical block.
This support also takes into consideration when protection information resides in last or first 8 bytes of metadata.

---
 engines/io_uring.c |  34 +++++++++
 engines/nvme.c     | 173 +++++++++++++++++++++++++++++++++++++++++++++
 engines/nvme.h     |  12 ++++
 3 files changed, 219 insertions(+)
engines/io_uring.c +34 -0
index 376a2a27..7ac7c755 100644
@@ -456,7 +456,9 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
 	struct ioring_options *o = td->eo;
 	struct io_uring_cqe *cqe;
 	struct io_u *io_u;
+	struct nvme_data *data;
 	unsigned index;
+	int ret;
 
 	index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
 	if (o->cmd_type == FIO_URING_CMD_NVME)
@@ -470,6 +472,15 @@ static struct io_u *fio_ioring_cmd_event(struct thread_data *td, int event)
 	else
 		io_u->error = 0;
 
+	if (o->cmd_type == FIO_URING_CMD_NVME) {
+		data = FILE_ENG_DATA(io_u->file);
+		if (data->pi_type && (io_u->ddir == DDIR_READ) && !o->pi_act) {
+			ret = fio_nvme_pi_verify(data, io_u);
+			if (ret)
+				io_u->error = ret;
+		}
+	}
+
 	return io_u;
 }
 
@@ -1190,6 +1201,7 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 {
 	struct ioring_data *ld = td->io_ops_data;
 	struct ioring_options *o = td->eo;
+	struct nvme_pi_data *pi_data;
 	char *p;
 
 	ld->io_u_index[io_u->index] = io_u;
@@ -1198,11 +1210,32 @@ static int fio_ioring_io_u_init(struct thread_data *td, struct io_u *io_u)
 		p = PTR_ALIGN(ld->md_buf, page_mask) + td->o.mem_align;
 		p += o->md_per_io_size * io_u->index;
 		io_u->mmap_data = p;
+
+		if (!o->pi_act) {
+			pi_data = calloc(1, sizeof(*pi_data));
+			pi_data->io_flags |= o->prchk;
+			pi_data->apptag_mask = o->apptag_mask;
+			pi_data->apptag = o->apptag;
+			io_u->engine_data = pi_data;
+		}
 	}
 
 	return 0;
 }
 
+static void fio_ioring_io_u_free(struct thread_data *td, struct io_u *io_u)
+{
+	struct ioring_options *o = td->eo;
+	struct nvme_pi *pi;
+
+	if (!strcmp(td->io_ops->name, "io_uring_cmd") &&
+	    (o->cmd_type == FIO_URING_CMD_NVME)) {
+		pi = io_u->engine_data;
+		free(pi);
+		io_u->engine_data = NULL;
+	}
+}
+
 static int fio_ioring_open_file(struct thread_data *td, struct fio_file *f)
 {
 	struct ioring_data *ld = td->io_ops_data;
@@ -1411,6 +1444,7 @@ static struct ioengine_ops ioengine_uring_cmd = {
 	.init			= fio_ioring_init,
 	.post_init		= fio_ioring_cmd_post_init,
 	.io_u_init		= fio_ioring_io_u_init,
+	.io_u_free		= fio_ioring_io_u_free,
 	.prep			= fio_ioring_cmd_prep,
 	.queue			= fio_ioring_queue,
 	.commit			= fio_ioring_commit,
engines/nvme.c +173 -0

197 lines not on this page · raw message

engines/nvme.h +12 -0
index f3598352..a1102dfe 100644
@@ -44,6 +44,9 @@ struct nvme_uring_cmd {
 #define NVME_IDENTIFY_CSI_SHIFT 24
 #define NVME_NQN_LENGTH	256
 
+#define NVME_PI_APP_DISABLE 0xFFFF
+#define NVME_PI_REF_DISABLE 0xFFFFFFFF
+
 #define NVME_ZNS_ZRA_REPORT_ZONES 0
 #define NVME_ZNS_ZRAS_FEAT_ERZ (1 << 16)
 #define NVME_ZNS_ZSA_RESET 0x4
@@ -131,6 +134,13 @@ enum nvme_io_control_flags {
 	NVME_IO_PRINFO_PRACT		= 1U << 29,
 };
 
+struct nvme_pi_data {
+	__u32 interval;
+	__u32 io_flags;
+	__u16 apptag;
+	__u16 apptag_mask;
+};
+
 struct nvme_lbaf {
 	__le16			ms;
 	__u8			ds;
@@ -415,6 +425,8 @@ int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 		      struct nvme_cmd_ext_io_opts *opts);
 
+int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u);
+
 int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
 			     enum zbd_zoned_model *model);
 
Signature · 1 line
2.25.1
#9 lore raw

PATCH v2 8/10 crc: pull required crc64 nvme apis from linux kernel

In reply to Ankit Kumar

Pull the required nvme crc64 apis and table from the linux kernel. This is required to generate and verify 64 bit guard tag for nvme backend of io_uring_cmd ioengine.

---
 crc/crc64.c      |  32 ++++++++++++
 crc/crc64.h      |   3 ++
 crc/crc64table.h | 130 +++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 165 insertions(+)
 create mode 100644 crc/crc64table.h
crc/crc64.c +32 -0
index bf24a97b..c910e5b8 100644
@@ -1,4 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * crc64nvme[256] table is from the generator polynomial specified by NVMe
+ * 64b CRC and is defined as,
+ *
+ * x^64 + x^63 + x^61 + x^59 + x^58 + x^56 + x^55 + x^52 + x^49 + x^48 + x^47 +
+ * x^46 + x^44 + x^41 + x^37 + x^36 + x^34 + x^32 + x^31 + x^28 + x^26 + x^23 +
+ * x^22 + x^19 + x^16 + x^13 + x^12 + x^10 + x^9 + x^6 + x^4 + x^3 + 1
+ *
+ */
+
 #include "crc64.h"
+#include "crc64table.h"
 
 /*
  * poly 0x95AC9329AC4BC9B5ULL and init 0xFFFFFFFFFFFFFFFFULL
@@ -102,3 +114,23 @@ unsigned long long fio_crc64(const unsigned char *buffer, unsigned long length)
 	return crc;
 }
 
+/**
+ * fio_crc64_nvme - Calculate bitwise NVMe CRC64
+ * @crc: seed value for computation. 0 for a new CRC calculation, or the
+ * 	 previous crc64 value if computing incrementally.
+ * @p: pointer to buffer over which CRC64 is run
+ * @len: length of buffer @p
+ */
+unsigned long long fio_crc64_nvme(unsigned long long crc, const void *p,
+				  unsigned int len)
+{
+	const unsigned char *_p = p;
+	unsigned int i;
+
+	crc = ~crc;
+
+	for (i = 0; i < len; i++)
+		crc = (crc >> 8) ^ crc64nvmetable[(crc & 0xff) ^ *_p++];
+
+	return ~crc;
+}
crc/crc64.h +3 -0
index fe9cad3e..e586edee 100644
@@ -3,4 +3,7 @@
 
 unsigned long long fio_crc64(const unsigned char *, unsigned long);
 
+unsigned long long fio_crc64_nvme(unsigned long long crc, const void *p,
+				  unsigned int len);
+
 #endif
crc/crc64table.h new file +130 -0

130 lines not on this page · raw message

Signature · 1 line
2.25.1
#10 lore raw

PATCH v2 9/10 engines:nvme: pull required 48 bit accessors from linux kernel

In reply to Ankit Kumar

Pull the 48 bit helpers, required for supporting 48 bit reference tags. Add GPL 2.0 license to nvme.c and nvme.h files.

---
 engines/nvme.c |  1 +
 engines/nvme.h | 17 +++++++++++++++++
 2 files changed, 18 insertions(+)
engines/nvme.c +1 -0
index e308331f..18740138 100644
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * nvme structure declarations and helper functions for the
  * io_uring_cmd engine.
engines/nvme.h +17 -0
index a1102dfe..fb1f7760 100644
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
 /*
  * nvme structure declarations and helper functions for the
  * io_uring_cmd engine.
@@ -440,4 +441,20 @@ int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
 int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
 				unsigned int *max_open_zones);
 
+static inline void put_unaligned_be48(__u64 val, __u8 *p)
+{
+	*p++ = val >> 40;
+	*p++ = val >> 32;
+	*p++ = val >> 24;
+	*p++ = val >> 16;
+	*p++ = val >> 8;
+	*p++ = val;
+}
+
+static inline __u64 get_unaligned_be48(__u8 *p)
+{
+	return (__u64)p[0] << 40 | (__u64)p[1] << 32 | (__u64)p[2] << 24 |
+		p[3] << 16 | p[4] << 8 | p[5];
+}
+
 #endif
Signature · 1 line
2.25.1
#11 lore raw

PATCH v2 10/10 engines:io_uring: generate and verify pi for 64b guard

In reply to Ankit Kumar

Generate and verify protection information for 64 bit guard format, for the nvme backend of io_uring_cmd ioengine. The support is there for both the cases where metadata is transferred in separate buffer, or transferred at the end of logical block creating an extended logical block.
This support also takes into consideration when protection information resides in last or first 16 bytes of metadata.

---
 engines/nvme.c | 158 +++++++++++++++++++++++++++++++++++++++++++++++++
 engines/nvme.h |   7 +++
 2 files changed, 165 insertions(+)
engines/nvme.c +158 -0
index 18740138..76f3fe2b 100644
@@ -6,6 +6,7 @@
 
 #include "nvme.h"
 #include "../crc/crc-t10dif.h"
+#include "../crc/crc64.h"
 
 static inline __u64 get_slba(struct nvme_data *data, struct io_u *io_u)
 {
@@ -175,6 +176,158 @@ next:
 	return 0;
 }
 
+static void fio_nvme_generate_pi_64b_guard(struct nvme_data *data,
+					   struct io_u *io_u,
+					   struct nvme_cmd_ext_io_opts *opts)
+{
+	struct nvme_pi_data *pi_data = io_u->engine_data;
+	struct nvme_64b_guard_pif *pi;
+	unsigned char *buf = io_u->xfer_buf;
+	unsigned char *md_buf = io_u->mmap_data;
+	uint64_t guard = 0;
+	__u64 slba = get_slba(data, io_u);
+	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 lba_num = 0;
+
+	if (data->pi_loc) {
+		if (data->lba_ext)
+			pi_data->interval = data->lba_ext - data->ms;
+		else
+			pi_data->interval = 0;
+	} else {
+		if (data->lba_ext)
+			pi_data->interval = data->lba_ext - sizeof(struct nvme_64b_guard_pif);
+		else
+			pi_data->interval = data->ms - sizeof(struct nvme_64b_guard_pif);
+	}
+
+	if (io_u->ddir != DDIR_WRITE)
+		return;
+
+	while (lba_num < nlb) {
+		if (data->lba_ext)
+			pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
+		else
+			pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+			if (data->lba_ext) {
+				guard = fio_crc64_nvme(0, buf, pi_data->interval);
+			} else {
+				guard = fio_crc64_nvme(0, buf, data->lba_size);
+				guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
+			}
+			pi->guard = cpu_to_be64(guard);
+		}
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_APP)
+			pi->apptag = cpu_to_be16(pi_data->apptag);
+
+		if (opts->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+			switch (data->pi_type) {
+			case NVME_NS_DPS_PI_TYPE1:
+			case NVME_NS_DPS_PI_TYPE2:
+				put_unaligned_be48(slba + lba_num, pi->srtag);
+				break;
+			case NVME_NS_DPS_PI_TYPE3:
+				break;
+			}
+		}
+		if (data->lba_ext) {
+			buf += data->lba_ext;
+		} else {
+			buf += data->lba_size;
+			md_buf += data->ms;
+		}
+		lba_num++;
+	}
+}
+
+static int fio_nvme_verify_pi_64b_guard(struct nvme_data *data,
+					struct io_u *io_u)
+{
+	struct nvme_pi_data *pi_data = io_u->engine_data;
+	struct nvme_64b_guard_pif *pi;
+	struct fio_file *f = io_u->file;
+	unsigned char *buf = io_u->xfer_buf;
+	unsigned char *md_buf = io_u->mmap_data;
+	__u64 slba = get_slba(data, io_u);
+	__u64 ref, ref_exp, guard = 0;
+	__u32 nlb = get_nlb(data, io_u) + 1;
+	__u32 lba_num = 0;
+	__u16 unmask_app, unmask_app_exp;
+
+	while (lba_num < nlb) {
+		if (data->lba_ext)
+			pi = (struct nvme_64b_guard_pif *)(buf + pi_data->interval);
+		else
+			pi = (struct nvme_64b_guard_pif *)(md_buf + pi_data->interval);
+
+		if (data->pi_type == NVME_NS_DPS_PI_TYPE3) {
+			if (pi->apptag == NVME_PI_APP_DISABLE &&
+			    fio_nvme_pi_ref_escape(pi->srtag))
+				goto next;
+		} else if (data->pi_type == NVME_NS_DPS_PI_TYPE1 ||
+			   data->pi_type == NVME_NS_DPS_PI_TYPE2) {
+			if (pi->apptag == NVME_PI_APP_DISABLE)
+				goto next;
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_GUARD) {
+			if (data->lba_ext) {
+				guard = fio_crc64_nvme(0, buf, pi_data->interval);
+			} else {
+				guard = fio_crc64_nvme(0, buf, data->lba_size);
+				guard = fio_crc64_nvme(guard, md_buf, pi_data->interval);
+			}
+			if (be64_to_cpu((uint64_t)pi->guard) != guard) {
+				log_err("%s: Guard compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
+					f->file_name, (unsigned long long)slba,
+					guard, be64_to_cpu((uint64_t)pi->guard));
+				return -EIO;
+			}
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_APP) {
+			unmask_app = be16_to_cpu(pi->apptag) & pi_data->apptag_mask;
+			unmask_app_exp = pi_data->apptag & pi_data->apptag_mask;
+			if (unmask_app != unmask_app_exp) {
+				log_err("%s: APPTAG compare error: LBA: %llu Expected=%x, Actual=%x\n",
+					f->file_name, (unsigned long long)slba,
+					unmask_app_exp, unmask_app);
+				return -EIO;
+			}
+		}
+
+		if (pi_data->io_flags & NVME_IO_PRINFO_PRCHK_REF) {
+			switch (data->pi_type) {
+			case NVME_NS_DPS_PI_TYPE1:
+			case NVME_NS_DPS_PI_TYPE2:
+				ref = get_unaligned_be48(pi->srtag);
+				ref_exp = (slba + lba_num) & ((1ULL << 48) - 1);
+				if (ref != ref_exp) {
+					log_err("%s: REFTAG compare error: LBA: %llu Expected=%llx, Actual=%llx\n",
+						f->file_name, (unsigned long long)slba,
+						ref_exp, ref);
+					return -EIO;
+				}
+				break;
+			case NVME_NS_DPS_PI_TYPE3:
+				break;
+			}
+		}
+next:
+		if (data->lba_ext) {
+			buf += data->lba_ext;
+		} else {
+			buf += data->lba_size;
+			md_buf += data->ms;
+		}
+		lba_num++;
+	}
+
+	return 0;
+}
 void fio_nvme_uring_cmd_trim_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 				  struct nvme_dsm_range *dsm)
 {
@@ -253,6 +406,8 @@ void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
 	if (data->pi_type && !(opts->io_flags & NVME_IO_PRINFO_PRACT)) {
 		if (data->guard_type == NVME_NVM_NS_16B_GUARD)
 			fio_nvme_generate_pi_16b_guard(data, io_u, opts);
+		else if (data->guard_type == NVME_NVM_NS_64B_GUARD)
+			fio_nvme_generate_pi_64b_guard(data, io_u, opts);
 	}
 
 	switch (data->pi_type) {
@@ -287,6 +442,9 @@ int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u)
 	case NVME_NVM_NS_16B_GUARD:
 		ret = fio_nvme_verify_pi_16b_guard(data, io_u);
 		break;
+	case NVME_NVM_NS_64B_GUARD:
+		ret = fio_nvme_verify_pi_64b_guard(data, io_u);
+		break;
 	default:
 		break;
 	}
engines/nvme.h +7 -0
index fb1f7760..792b35d8 100644
@@ -457,4 +457,11 @@ static inline __u64 get_unaligned_be48(__u8 *p)
 		p[3] << 16 | p[4] << 8 | p[5];
 }
 
+static inline bool fio_nvme_pi_ref_escape(__u8 *reftag)
+{
+	__u8 ref_esc[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
+
+	return memcmp(reftag, ref_esc, sizeof(ref_esc)) == 0;
+}
+
 #endif
Signature · 1 line
2.25.1