virtio nvme
Subsystems (3)
cover virtio nvme
Hi all,
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
-drive file=disk.img,format=raw,if=none,id=D22 \
-device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to host(vhost_nvme) to LIO NVMe-over-fabrics target.
Now there are lots of duplicated code with linux/nvme-core.c and qemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example
.-------------------------.
| NVMe device register |
Upper level | NVMe protocol process |
| |
'-------------------------'
.-----------. .-----------. .------------------.
Lower level | PCIe | | VIRTIO | |NVMe over Fabrics |
| | | | |initiator |
'-----------' '-----------' '------------------'todo:
- tune performance. Should be as good as virtio-blk/virtio-scsi
- support discard/flush/integrity
- need Redhat's help for the VIRTIO_ID_NVME pci id
- multi level NVMe stack
Code also available at:
http://www.minggr.net/cgit/cgit.cgi/linux/commit/?h=virtio-nvme/v1
http://www.minggr.net/cgit/cgit.cgi/qemu/log/?h=virtio-nvme/v1
Thanks,
Ming
RFC PATCH 1/2 virtio_nvme(kernel): virtual NVMe driver using virtio
--- drivers/block/Kconfig | 7 + drivers/block/Makefile | 1 + drivers/block/nvme-core.c | 1 + drivers/block/virtio_nvme.c | 853 +++++++++++++++++++++++++++++++++++++++ include/linux/virtio_nvme.h | 53 +++ include/uapi/linux/virtio_ids.h | 1 + include/uapi/linux/virtio_nvme.h | 30 ++ 7 files changed, 946 insertions(+) create mode 100644 drivers/block/virtio_nvme.c create mode 100644 include/linux/virtio_nvme.h create mode 100644 include/uapi/linux/virtio_nvme.h
drivers/block/Kconfig +7 -0
@@ -519,6 +519,13 @@ config VIRTIO_BLK This is the virtual block driver for virtio. It can be used with lguest or QEMU based VMMs (like KVM or Xen). Say Y or M. +config VIRTIO_NVME + tristate "Virtio NVMe driver" + depends on VIRTIO + ---help--- + This is the virtual NVMe driver for virtio. It can be used with + lguest or QEMU based VMMs (like KVM or Xen). Say Y or M. + config BLK_DEV_HD bool "Very old hard disk (MFM/RLL/IDE) driver" depends on HAVE_IDE
drivers/block/Makefile +1 -0
@@ -30,6 +30,7 @@ obj-$(CONFIG_BLK_DEV_UMEM) += umem.o obj-$(CONFIG_BLK_DEV_NBD) += nbd.o obj-$(CONFIG_BLK_DEV_CRYPTOLOOP) += cryptoloop.o obj-$(CONFIG_VIRTIO_BLK) += virtio_blk.o +obj-$(CONFIG_VIRTIO_NVME) += virtio_nvme.o obj-$(CONFIG_BLK_DEV_SX8) += sx8.o obj-$(CONFIG_BLK_DEV_HD) += hd.o
drivers/block/nvme-core.c +1 -0
@@ -1059,6 +1059,7 @@ int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, {return __nvme_submit_sync_cmd (q, cmd, buffer,NULL , bufflen,NULL ,0 ); } +EXPORT_SYMBOL_GPL (nvme_submit_sync_cmd);static int nvme_submit_async_admin_req (struct nvme_dev *dev) {
drivers/block/virtio_nvme.c new file +853 -0
853 lines not on this page · raw message
include/linux/virtio_nvme.h new file +53 -0
53 lines not on this page · raw message
include/uapi/linux/virtio_ids.h +1 -0
@@ -41,5 +41,6 @@#define VIRTIO_ID_CAIF 12/* Virtio caif */ #define VIRTIO_ID_GPU 16/* virtio GPU */ #define VIRTIO_ID_INPUT 18/* virtio input */ +#define VIRTIO_ID_NVME 19/* TBD: virtio NVMe, need Redhat's help to get this id */ #endif /* _LINUX_VIRTIO_IDS_H */
include/uapi/linux/virtio_nvme.h new file +30 -0
@@ -0,0 +1,30 @@ +#ifndef _UAPI_LINUX_VIRTIO_NVME_H +#define _UAPI_LINUX_VIRTIO_NVME_H + +#include <linux/types.h> +#include <linux/virtio_ids.h> +#include <linux/virtio_config.h> +#include <linux/virtio_types.h> + +/* Feature bits */ +#define VIRTIO_NVME_F_SEG_MAX 1/* Indicates maximum # of segments */ +#define VIRTIO_NVME_F_MQ 2/* support more than one vq */ + +struct virtio_nvme_config { +__u64 cap; +__u32 ctrl_config; +__u32 csts; + +/* The maximum number of segments (if VIRTIO_NVME_F_SEG_MAX) */ +__u32 seg_max; +/* number of vqs, only available when VIRTIO_NVME_F_MQ is set */ +__u32 num_queues; +} __attribute__((packed)); + +struct virtio_nvme_resp { +__u32 result; +__u16 cid; +__u16 status; +}; + +#endif
Signature · 1 line
1.9.1
RFC PATCH 2/2 virtio-nvme(qemu): NVMe device using virtio
In reply to Ming LinPlay it with:
-drive file=disk.img,format=raw,if=none,id=D22 \
-device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4
--- hw/block/Makefile.objs | 2 +- hw/block/virtio-nvme.c | 449 +++++++++++++++++++++++++++ hw/virtio/virtio-pci.c | 42 +++ hw/virtio/virtio-pci.h | 14 + include/hw/pci/pci.h | 1 + include/hw/virtio/virtio-nvme.h | 60 ++++ include/standard-headers/linux/virtio_ids.h | 1 + include/standard-headers/linux/virtio_nvme.h | 16 + 8 files changed, 584 insertions(+), 1 deletion(-) create mode 100644 hw/block/virtio-nvme.c create mode 100644 include/hw/virtio/virtio-nvme.h create mode 100644 include/standard-headers/linux/virtio_nvme.h
hw/block/Makefile.objs +1 -1
@@ -11,5 +11,5 @@ common-obj-$(CONFIG_NVME_PCI) += nvme.o obj-$(CONFIG_SH4) += tc58128.o -obj-$(CONFIG_VIRTIO) += virtio-blk.o +obj-$(CONFIG_VIRTIO) += virtio-blk.o virtio-nvme.o obj-$(CONFIG_VIRTIO) += dataplane/
hw/block/virtio-nvme.c new file +449 -0
449 lines not on this page · raw message
hw/virtio/virtio-pci.c +42 -0
54 lines not on this page · raw message
hw/virtio/virtio-pci.h +14 -0
@@ -32,10 +32,12 @@#ifdef CONFIG_VHOST_SCSI #include "hw/virtio/vhost-scsi.h" #endif +#include "hw/virtio/virtio-nvme.h" typedef struct VirtIOPCIProxy VirtIOPCIProxy ;typedef struct VirtIOBlkPCI VirtIOBlkPCI ;typedef struct VirtIOSCSIPCI VirtIOSCSIPCI ; +typedef struct VirtIONVMEPCI VirtIONVMEPCI ;typedef struct VirtIOBalloonPCI VirtIOBalloonPCI ;typedef struct VirtIOSerialPCI VirtIOSerialPCI ;typedef struct VirtIONetPCI VirtIONetPCI ;
@@ -179,6 +181,18 @@ struct VirtIOBlkPCI { };/* +* virtio-nvme-pci: This extends VirtioPCIProxy. +*/ +#define TYPE_VIRTIO_NVME_PCI "virtio-nvme-pci" +#define VIRTIO_NVME_PCI (obj) \ + OBJECT_CHECK(VirtIONVMEPCI, (obj), TYPE_VIRTIO_NVME_PCI) + +struct VirtIONVMEPCI { +VirtIOPCIProxy parent_obj; +VirtIONVME vdev; +}; + +/* * virtio-balloon-pci: This extends VirtioPCIProxy. */ #define TYPE_VIRTIO_BALLOON_PCI "virtio-balloon-pci"
include/hw/pci/pci.h +1 -0
@@ -81,6 +81,7 @@#define PCI_DEVICE_ID_VIRTIO_SCSI 0x1004#define PCI_DEVICE_ID_VIRTIO_RNG 0x1005#define PCI_DEVICE_ID_VIRTIO_9P 0x1009 +#define PCI_DEVICE_ID_VIRTIO_NVME 0x100a#define PCI_VENDOR_ID_REDHAT 0x1b36#define PCI_DEVICE_ID_REDHAT_BRIDGE 0x0001
include/hw/virtio/virtio-nvme.h new file +60 -0
60 lines not on this page · raw message
include/standard-headers/linux/virtio_ids.h +1 -0
@@ -41,5 +41,6 @@#define VIRTIO_ID_CAIF 12/* Virtio caif */ #define VIRTIO_ID_GPU 16/* virtio GPU */ #define VIRTIO_ID_INPUT 18/* virtio input */ +#define VIRTIO_ID_NVME 19/* TBD: virtio NVMe, need Redhat's help to get this id */ #endif /* _LINUX_VIRTIO_IDS_H */
include/standard-headers/linux/virtio_nvme.h new file +16 -0
@@ -0,0 +1,16 @@ +#ifndef _LINUX_VIRTIO_NVME_H +#define _LINUX_VIRTIO_NVME_H + +/* Feature bits */ +#define VIRTIO_NVME_F_SEG_MAX 1/* Indicates maximum # of segments */ +#define VIRTIO_NVME_F_MQ 2/* support more than one vq */ + +struct virtio_nvme_config { +uint64_t cap; +uint32_t ctrl_config; +uint32_t csts; +uint32_t seg_max; +uint32_t num_queues; +}QEMU_PACKED ; + +#endif
Signature · 1 line
1.9.1
cover virtio nvme
In reply to Ming LinQuoted 20 lines · Ming Lin
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------' .-----------. .-----------. .------------------. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'
Quoted 5 lines
todo:
- tune performance. Should be as good as virtio-blk/virtio-scsi
- support discard/flush/integrity
- need Redhat's help for theVIRTIO_ID_NVMEpci id
- multi level NVMe stack
Hi Ming,
I'll be out for travel for the next week, so I won't have much time to do a proper review till the following week.
I think it'd be better to get this hierarchy setup to make the most reuse possible than to have this much code duplication between the existing driver and emulated qemu nvme. For better or worse, I think the generic nvme layer is where things are going. Are you signed up with the fabrics contributors?
cover virtio nvme
Quoted 21 lines · Keith Busch
Quoted 20 lines · Ming Lin
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------' .-----------. .-----------. .------------------. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'
Quoted 7 lines
Quoted 5 lines
todo:
- tune performance. Should be as good as virtio-blk/virtio-scsi
- support discard/flush/integrity
- need Redhat's help for theVIRTIO_ID_NVMEpci id
- multi level NVMe stackHi Ming,
Hi Keith,
Quoted 9 lines
I'll be out for travel for the next week, so I won't have much time to do a proper review till the following week.
I think it'd be better to get this hierarchy setup to make the most reuse possible than to have this much code duplication between the existing driver and emulated qemu nvme. For better or worse, I think the generic nvme layer is where things are going. Are you signed up with the fabrics contributors?
No. How to sign up?
cover virtio nvme
Quoted 36 lines · Ming Lin
Quoted 21 lines · Keith Busch
Quoted 20 lines · Ming Lin
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------' .-----------. .-----------. .------------------. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'Quoted 7 lines
Quoted 5 lines
todo:
- tune performance. Should be as good as virtio-blk/virtio-scsi
- support discard/flush/integrity
- need Redhat's help for theVIRTIO_ID_NVMEpci id
- multi level NVMe stackHi Ming,
Hi Keith,
Quoted 4 lines
I'll be out for travel for the next week, so I won't have much time to do a proper review till the following week.
Quoted 7 lines
Quoted 5 lines
I think it'd be better to get this hierarchy setup to make the most reuse possible than to have this much code duplication between the existing driver and emulated qemu nvme. For better or worse, I think the generic nvme layer is where things are going. Are you signed up with the fabrics contributors?
No. How to sign up?
Got it. Just saw your previous mail: A message on NVMe Over Fabrics
cover virtio nvme
In reply to Ming LinQuoted 53 lines · Ming Lin
Quoted 27 lines · Keith Busch
Quoted 26 lines · Ming Lin
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c.
The ideal result is to have a multi level NVMe stack(similar as SCSI).
So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------' .-----------. .-----------. .----------------- -. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'
Quoted 7 lines
Quoted 5 lines
todo:
- tune performance. Should be as good as virtio-blk/virtio-scsi
- support discard/flush/integrity
- need Redhat's help for theVIRTIO_ID_NVMEpci id
- multi level NVMe stackHi Ming,
Hi Keith,
Quoted 5 lines
I'll be out for travel for the next week, so I won't have much time to
do a proper review till the following week.Quoted 9 lines
I think it'd be better to get this hierarchy setup to make the most reuse
possible than to have this much code duplication between the existing
driver and emulated qemu nvme. For better or worse, I think the generic
nvme layer is where things are going. Are you signed up with the fabrics
contributors?
Quoted 1 line
No. How to sign up?
Ming,
Here is the email Keith sent out on this list that says how to sign up:
http://lists.infradead.org/pipermail/linux-nvme/2015
-September/002331.html
Jay
Quoted 7 lines
_______________________________________________
Linux-nvme mailing list
Linux-nvme at lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme
cover virtio nvme
In reply to Ming LinQuoted 11 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
(quoted diff) +0
-2-drive file=disk.img,format=raw,if=none,id=D22 \ -device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.
Why is a virtio-nvme guest device needed? I guess there must either be NVMe-only features that you want to pass through, or you think the performance will be significantly better than virtio-blk/virtio-scsi?
At first glance it seems like the virtio_nvme guest driver is just another block driver like virtio_blk, so I'm not clear why a virtio-nvme device makes sense.
Quoted 6 lines
Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |
Quoted 10 lines
Upper level | NVMe protocol process |
| |
'-------------------------'
.-----------. .-----------. .------------------.
Lower level | PCIe | | VIRTIO | |NVMe over Fabrics |
| | | | |initiator |
'-----------' '-----------' '------------------'You mentioned LIO and SCSI. How will NVMe over Fabrics be integrated into LIO? If it is mapped to SCSI then using virtio_scsi in the guest and tcm_vhost should work.
Please also post virtio draft specifications documenting the virtio device.
Stefan
cover virtio nvme
Quoted 10 lines · Stefan Hajnoczi
Quoted 9 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
(quoted diff) +0
-2-drive file=disk.img,format=raw,if=none,id=D22 \ -device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4
Quoted 6 lines
Quoted 2 lines
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Why is a virtio-nvme guest device needed? I guess there must either be NVMe-only features that you want to pass through, or you think the performance will be significantly better than virtio-blk/virtio-scsi?
It simply passes through NVMe commands.
Right now performance is poor. Performance tunning is on my todo list. It should be as good as virtio-blk/virtio-scsi.
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.
I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-target
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-target
Quoted 13 lines
Quoted 13 lines
Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------'
Quoted 8 lines
Quoted 4 lines
.-----------. .-----------. .------------------. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'You mentioned LIO and SCSI. How will NVMe over Fabrics be integrated into LIO? If it is mapped to SCSI then using
virtio_scsiin the guest andtcm_vhostshould work.
I think it's not mapped to SCSI.
Nick, would you share more here?
Quoted 2 lines
Please also post virtio draft specifications documenting the virtio device.
I'll do this later.
Quoted 2 lines
Stefan
cover virtio nvme
Quoted 11 lines · Ming Lin
Quoted 10 lines · Stefan Hajnoczi
Quoted 9 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
(quoted diff) +0
-2-drive file=disk.img,format=raw,if=none,id=D22 \ -device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4
Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Why is a virtio-nvme guest device needed? I guess there must either be NVMe-only features that you want to pass through, or you think the performance will be significantly better than virtio-blk/virtio-scsi?
It simply passes through NVMe commands.
I understand that. My question is why the guest needs to send NVMe commands?
If the virtio_nvme.ko guest driver only sends read/write/flush then there's no advantage over virtio-blk.
There must be something you are trying to achieve which is not possible with virtio-blk or virtio-scsi. What is that?
Stefan
cover virtio nvme
Quoted 21 lines · Stefan Hajnoczi
Quoted 11 lines · Ming Lin
Quoted 10 lines · Stefan Hajnoczi
Quoted 9 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
(quoted diff) +0
-2-drive file=disk.img,format=raw,if=none,id=D22 \ -device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Why is a virtio-nvme guest device needed? I guess there must either be NVMe-only features that you want to pass through, or you think the performance will be significantly better than virtio-blk/virtio-scsi?
It simply passes through NVMe commands.
Quoted 7 lines
I understand that. My question is why the guest needs to send NVMe commands?
If the virtio_nvme.ko guest driver only sends read/write/flush then there's no advantage over virtio-blk.
There must be something you are trying to achieve which is not possible with virtio-blk or virtio-scsi. What is that?
I actually learned from your virtio-scsi work.
http://www.linux-kvm.org/images/f/f5/2011-forum-virtio-scsi.pdf
Then I thought a full NVMe stack from guest to host to target seems reasonable.
Trying to achieve similar things as virtio-scsi, but all NVMe protocol.
- Effective NVMe passthrough
- Multiple target choices: QEMU, LIO-NVMe(vhost_nvme)
- Almost unlimited scalability. Thousands of namespaces per PCI device
- True NVMe device
- End-to-end Protection Information
- ....
cover virtio nvme
Quoted 36 lines · Ming Lin
Quoted 21 lines · Stefan Hajnoczi
Quoted 11 lines · Ming Lin
Quoted 10 lines · Stefan Hajnoczi
Quoted 9 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
(quoted diff) +0
-2-drive file=disk.img,format=raw,if=none,id=D22 \ -device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Why is a virtio-nvme guest device needed? I guess there must either be NVMe-only features that you want to pass through, or you think the performance will be significantly better than virtio-blk/virtio-scsi?
It simply passes through NVMe commands.
Quoted 7 lines
I understand that. My question is why the guest needs to send NVMe commands?
If the virtio_nvme.ko guest driver only sends read/write/flush then there's no advantage over virtio-blk.
There must be something you are trying to achieve which is not possible with virtio-blk or virtio-scsi. What is that?
I actually learned from your virtio-scsi work.
http://www.linux-kvm.org/images/f/f5/2011-forum-virtio-scsi.pdfThen I thought a full NVMe stack from guest to host to target seems reasonable.
Quoted 8 lines
Trying to achieve similar things as virtio-scsi, but all NVMe protocol.
- Effective NVMe passthrough
- Multiple target choices: QEMU, LIO-NVMe(vhost_nvme)
- Almost unlimited scalability. Thousands of namespaces per PCI device
- True NVMe device
- End-to-end Protection Information
- ....
The advantages you mentioned are already available in virtio-scsi, except for the NVMe command set.
I don't understand what unique problem virtio-nvme solves yet. If someone asked me to explain why NVMe-over-virtio makes sense compared to the existing virtio-blk/virtio-scsi or NVMe SR-IOV options, I wouldn't know the answer. I'd like to learn that from you or anyone else on CC.
Do you have a use case in mind?
Stefan
cover virtio nvme
Quoted 49 lines · Stefan Hajnoczi
Quoted 36 lines · Ming Lin
Quoted 21 lines · Stefan Hajnoczi
Quoted 11 lines · Ming Lin
Quoted 10 lines · Stefan Hajnoczi
Quoted 9 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
Play it in Qemu with:
(quoted diff) +0
-2-drive file=disk.img,format=raw,if=none,id=D22 \ -device virtio-nvme-pci,drive=D22,serial=1234,num_queues=4Quoted 8 lines
Quoted 6 lines
Quoted 2 lines
The goal is to have a full NVMe stack from VM guest(virtio-nvme) to
host(vhost_nvme)to LIO NVMe-over-fabrics target.Why is a virtio-nvme guest device needed? I guess there must either be NVMe-only features that you want to pass through, or you think the performance will be significantly better than virtio-blk/virtio-scsi?
It simply passes through NVMe commands.
Quoted 7 lines
I understand that. My question is why the guest needs to send NVMe commands?
If the virtio_nvme.ko guest driver only sends read/write/flush then there's no advantage over virtio-blk.
There must be something you are trying to achieve which is not possible with virtio-blk or virtio-scsi. What is that?
I actually learned from your virtio-scsi work.
http://www.linux-kvm.org/images/f/f5/2011-forum-virtio-scsi.pdfThen I thought a full NVMe stack from guest to host to target seems reasonable.
Quoted 8 lines
Trying to achieve similar things as virtio-scsi, but all NVMe protocol.
- Effective NVMe passthrough
- Multiple target choices: QEMU, LIO-NVMe(vhost_nvme)
- Almost unlimited scalability. Thousands of namespaces per PCI device
- True NVMe device
- End-to-end Protection Information
- ....The advantages you mentioned are already available in virtio-scsi, except for the NVMe command set.
Quoted 7 lines
I don't understand what unique problem virtio-nvme solves yet. If someone asked me to explain why NVMe-over-virtio makes sense compared to the existing virtio-blk/virtio-scsi or NVMe SR-IOV options, I wouldn't know the answer. I'd like to learn that from you or anyone else on CC.
Do you have a use case in mind?
One use case is for All NVMe storage array. There is no SCSI device at all.
Samsung demoed one in flash memory summit 2015.
http://www.tomsitpro.com/articles/best-of-flash-memory-summit,1-2806.html
Quoted 2 lines
Stefan
cover virtio nvme
In reply to Ming LinHi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-target
Quoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-target
I think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Quoted 18 lines
Quoted 18 lines
Quoted 17 lines
Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------' .-----------. .-----------. .------------------. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'
Quoted 8 lines
Quoted 3 lines
You mentioned LIO and SCSI. How will NVMe over Fabrics be integrated into LIO? If it is mapped to SCSI then using
virtio_scsiin the guest andtcm_vhostshould work.I think it's not mapped to SCSI.
Nick, would you share more here?
(Adding Dave M. CC')
So NVMe target code needs to function in at least two different modes:
- Direct mapping of nvme backend driver provided hw queues to nvme fabric driver provided hw queues.
- Decoding of NVMe command set for basic Read/Write/Flush I/O for submission to existing backend drivers (eg: iblock, fileio, rd_mcp)
With the former case, it's safe to assumes there to be anywhere from a very small amount of code involved, to no code involved for fast-path operation.
For more involved logic like PR, ALUA, and EXTENDED_COPY, I think both modes will still mostly likely handle some aspects of this in software, and not entirely behind a backend nvme host hw interface.
--nab
cover virtio nvme
Quoted 1 line · Nicholas A. Bellinger
Hi Ming & Co,
Hi Nic,
Quoted 31 lines · Ming Lin
Quoted 6 lines · Stefan Hajnoczi
Quoted 5 lines · Ming Lin
Quoted 4 lines
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-target
Quoted 6 lines
I think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Are the patches already in qemu upstream and LIO upstream?
I found you played it in 2010. Is it?
[QEMU-KVM]: Megasas + TCM_Loop + SG_IO into Windows XP guests
https://groups.google.com/forum/#!topic/linux-iscsi-target-dev/3hdaI6H3X0A
Quoted 5 lines
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
cover virtio nvme
Quoted 30 lines · Ming Lin
Quoted 21 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetQuoted 6 lines
I think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Quoted 6 lines
Are the patches already in qemu upstream and LIO upstream?
I found you played it in 2010. Is it?
[QEMU-KVM]: Megasas +TCM_Loop+SG_IOinto Windows XP guests
https://groups.google.com/forum/#!topic/linux-iscsi-target-dev/3hdaI6H3X0A
AFAIK it was never submitted for kernel/QEMU upstream inclusion.
Dr Hannes, would you be so kind to put this code in a public repo to grok as an example for nvme host-aware emulation..?
--nab
Quoted 30 lines · Ming Lin
Quoted 21 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetQuoted 6 lines
I think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Quoted 6 lines
Are the patches already in qemu upstream and LIO upstream?
I found you played it in 2010. Is it?
[QEMU-KVM]: Megasas +TCM_Loop+SG_IOinto Windows XP guests
https://groups.google.com/forum/#!topic/linux-iscsi-target-dev/3hdaI6H3X0A
AFAIK it was never submitted for kernel/QEMU upstream inclusion.
Dr Hannes, would you be so kind to put this code in a public repo to grok as an example for nvme host-aware emulation..?
--nab
Quoted 1 line · Nicholas A. Bellinger
Hi Ming & Co,
Hi Nic,
Quoted 31 lines · Ming Lin
Quoted 6 lines · Stefan Hajnoczi
Quoted 5 lines · Ming Lin
Quoted 4 lines
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-target
Quoted 6 lines
I think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Are the patches already in qemu upstream and LIO upstream?
I found you played it in 2010. Is it?
[QEMU-KVM]: Megasas + TCM_Loop + SG_IO into Windows XP guests
https://groups.google.com/forum/#!topic/linux-iscsi-target-dev/3hdaI6H3X0A
Quoted 5 lines
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
cover virtio nvme
In reply to Nicholas A. BellingerQuoted 35 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?
.------------------------. MMIO .---------------------------------------.
| Guest |--------> | Qemu |
| Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) |
'------------------------' '---------------------------------------'
| ^
write NVMe | | notify command
command | | completion
to eventfd | | to eventfd
v |
.--------------------------------------.
| Host: |
| eventfd based LIO NVMe fabric driver |
'--------------------------------------'
|
| nvme_queue_rq()
v
.--------------------------------------.
| NVMe driver |
'--------------------------------------'
|
|
v
.-------------------------------------.
| NVMe device |
'-------------------------------------'cover virtio nvme
Quoted 65 lines · Ming Lin
Quoted 35 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.
Quoted 9 lines
| NVMe driver |
'--------------------------------------'
|
|
v
.-------------------------------------.
| NVMe device |
'-------------------------------------'Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
As with the nvme-over-fabric case, it would be possible to do a mapping between backend driver queue resources for real NVMe hardware (eg:target_core_nvme), but since it would still be doing close to the same amount of software emulation for both backend driver cases, I wouldn't expect there to be much performance advantage over just using normal submit_bio().
--nab
cover virtio nvme
Quoted 76 lines · Nicholas A. Bellinger
Quoted 65 lines · Ming Lin
Quoted 35 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.Quoted 9 lines
| NVMe driver | '--------------------------------------' | | v .-------------------------------------. | NVMe device | '-------------------------------------'
Quoted 4 lines
Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
Let me call the "eventfd based LIO NVMe fabric driver" as "tcm_eventfd_nvme"
Currently, LIO frontend driver(iscsi, fc, vhost-scsi etc) talk to LIO backend driver(fileio, iblock etc) with SCSI commands.
Did you mean the "tcm_eventfd_nvme" driver need to translate NVMe commands to SCSI commands and then submit to backend driver?
But I thought the future "LIO NVMe target" can support frontend driver talk to backend driver directly with NVMe commands without translation.
Am I wrong?
Quoted 10 lines
As with the nvme-over-fabric case, it would be possible to do a mapping between backend driver queue resources for real NVMe hardware (eg:
target_core_nvme), but since it would still be doing close to the same amount of software emulation for both backend driver cases, I wouldn't expect there to be much performance advantage over just using normalsubmit_bio().--nab
cover virtio nvme
Quoted 4 lines · Ming Lin
Quoted 3 lines · Nicholas A. Bellinger
Quoted 2 lines · Ming Lin
Quoted 1 line · Nicholas A. Bellinger
Hi Ming & Co,
<SNIP>
Quoted 62 lines
Quoted 57 lines
Quoted 47 lines
Quoted 18 lines
Quoted 9 lines
I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.Quoted 9 lines
| NVMe driver | '--------------------------------------' | | v .-------------------------------------. | NVMe device | '-------------------------------------'Quoted 4 lines
Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
Quoted 9 lines
Let me call the "eventfd based LIO NVMe fabric driver" as "
tcm_eventfd_nvme"Currently, LIO frontend
driver(iscsi, fc, vhost-scsi etc)talk to LIO backenddriver(fileio, iblock etc)with SCSI commands.Did you mean the "
tcm_eventfd_nvme" driver need to translate NVMe commands to SCSI commands and then submit to backend driver?
IBLOCK + FILEIO + RD_MCP don't speak SCSI, they simply process I/Os with LBA + length based on SGL memory or pass along a FLUSH with LBA + length.
So once the 'tcm_eventfd_nvme' driver on KVM host receives a nvme host hardware frame via eventfd, it would decode the frame and send along the Read/Write/Flush when exposing existing (non nvme native) backend drivers.
This doesn't apply to PSCSI backend driver of course, because it expects to process actual SCSI CDBs.
Quoted 3 lines
But I thought the future "LIO NVMe target" can support frontend driver talk to backend driver directly with NVMe commands without translation.
The native target_core_nvme backend driver is not processing nvme commands per-say, but simply exposing nvme hardware queue resources to a frontend fabric driver.
For the nvme-over-fabrics case, backend nvme submission/complete queues are mapped to RDMA hardware queues. So essentially the nvme physical region page (PRP) is mapped to ib_sgl->addr.
For a 'tcm_eventfd_nvme' style host-paravirt fabric case, it's less clear how exposing native nvme backend hardware resources would work, or if there is a significant performance benefit over just using submit_bio() for Read/Write/Flush.
--nab
cover virtio nvme
Quoted 100 lines · Nicholas A. Bellinger
Quoted 4 lines · Ming Lin
Quoted 3 lines · Nicholas A. Bellinger
Quoted 2 lines · Ming Lin
Quoted 1 line · Nicholas A. Bellinger
Hi Ming & Co,
<SNIP>
Quoted 62 lines
Quoted 57 lines
Quoted 47 lines
Quoted 18 lines
Quoted 9 lines
I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.Quoted 9 lines
| NVMe driver | '--------------------------------------' | | v .-------------------------------------. | NVMe device | '-------------------------------------'Quoted 4 lines
Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
Quoted 9 lines
Let me call the "eventfd based LIO NVMe fabric driver" as "
tcm_eventfd_nvme"Currently, LIO frontend
driver(iscsi, fc, vhost-scsi etc)talk to LIO backenddriver(fileio, iblock etc)with SCSI commands.Did you mean the "
tcm_eventfd_nvme" driver need to translate NVMe commands to SCSI commands and then submit to backend driver?IBLOCK + FILEIO +
RD_MCPdon't speak SCSI, they simply process I/Os with LBA + length based on SGL memory or pass along a FLUSH with LBA + length.So once the '
tcm_eventfd_nvme' driver on KVM host receives a nvme host hardware frame via eventfd, it would decode the frame and send along the Read/Write/Flush when exposing existing (non nvme native) backend drivers.This doesn't apply to PSCSI backend driver of course, because it expects to process actual SCSI CDBs.
Quoted 3 lines
But I thought the future "LIO NVMe target" can support frontend driver talk to backend driver directly with NVMe commands without translation.
The native
target_core_nvmebackend driver is not processing nvme commands per-say, but simply exposing nvme hardware queue resources to a frontend fabric driver.
Quoted 8 lines
For the nvme-over-fabrics case, backend nvme submission/complete queues are mapped to RDMA hardware queues. So essentially the nvme physical region page (PRP) is mapped to
ib_sgl->addr.For a '
tcm_eventfd_nvme' style host-paravirt fabric case, it's less clear how exposing native nvme backend hardware resources would work, or if there is a significant performance benefit over just usingsubmit_bio()for Read/Write/Flush.
Now it's much more clear. I'll do a tcm_eventfd_nvme prototype.
Thanks for all the detail explanation.
Quoted 3 lines
--nab
cover virtio nvme
In reply to Nicholas A. BellingerQuoted 80 lines · Nicholas A. Bellinger
Quoted 4 lines · Ming Lin
Quoted 3 lines · Nicholas A. Bellinger
Quoted 2 lines · Ming Lin
Quoted 1 line · Nicholas A. Bellinger
Hi Ming & Co,
<SNIP>
Quoted 62 lines
Quoted 57 lines
Quoted 47 lines
Quoted 18 lines
Quoted 9 lines
I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.Quoted 9 lines
| NVMe driver | '--------------------------------------' | | v .-------------------------------------. | NVMe device | '-------------------------------------'Quoted 4 lines
Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
Quoted 9 lines
Let me call the "eventfd based LIO NVMe fabric driver" as "
tcm_eventfd_nvme"Currently, LIO frontend
driver(iscsi, fc, vhost-scsi etc)talk to LIO backenddriver(fileio, iblock etc)with SCSI commands.Did you mean the "
tcm_eventfd_nvme" driver need to translate NVMe commands to SCSI commands and then submit to backend driver?
Quoted 8 lines
IBLOCK + FILEIO +
RD_MCPdon't speak SCSI, they simply process I/Os with LBA + length based on SGL memory or pass along a FLUSH with LBA + length.So once the '
tcm_eventfd_nvme' driver on KVM host receives a nvme host hardware frame via eventfd, it would decode the frame and send along the Read/Write/Flush when exposing existing (non nvme native) backend drivers.
Learned vhost architecture:
http://blog.vmsplice.net/2011/09/qemu-internals-vhost-architecture.html
The nice thing is it is not tied to KVM in any way.
For SCSI, there are "virtio-scsi" in guest kernel and "vhost-scsi" in host kernel.
For NVMe, there is no "virtio-nvme" in guest kernel(just unmodified NVMe driver), but I'll do similar thing in Qemu with vhost infrastructure. And there is "vhost_nvme" in host kernel.
For the "virtqueue" implementation in qemu-nvme, I'll possibly just use/copy drivers/virtio/virtio_ring.c, same as what linux/tools/virtio/virtio_test.c does.
A bit more detail graph as below. What do you think?
.-----------------------------------------. .------------------------.
| Guest(Linux, Windows, FreeBSD, Solaris) | NVMe | qemu |
| unmodified NVMe driver | command | NVMe device emulation |
| | -------> | vhost + virtqueue |
'-----------------------------------------' '------------------------'
| | ^
passthrough | kick/notify
NVMe command | via eventfd
userspace via virtqueue | | |
v v |
----------------------------------------------------------------------------------
.-----------------------------------------------------------------------.
kernel | LIO frontend driver |
| - vhost_nvme | '-----------------------------------------------------------------------'
| translate ^
| (NVMe command) |
| to |
v (LBA, length) |
.----------------------------------------------------------------------.
| LIO backend driver |
| - fileio (/mnt/xxx.file) |
| - iblock (/dev/sda1, /dev/nvme0n1, ...) |
'----------------------------------------------------------------------'
| ^
| submit_bio() |
v |
.----------------------------------------------------------------------.
| block layer |
| |
'----------------------------------------------------------------------'
| ^
| |
v |
.----------------------------------------------------------------------.
| block device driver |
| |
'----------------------------------------------------------------------'
| | | |
| | | |
v v v v
.------------. .-----------. .------------. .---------------.
| SATA | | SCSI | | NVMe | | .... |
'------------' '-----------' '------------' '---------------'
cover virtio nvme
Quoted 3 lines · Ming Lin
Quoted 2 lines · Nicholas A. Bellinger
Quoted 1 line · Ming Lin
On Thu, 2015-09-17@17:55 -0700, Nicholas A. Bellinger wrote:
<SNIP>
Quoted 9 lines
Quoted 8 lines
IBLOCK + FILEIO +
RD_MCPdon't speak SCSI, they simply process I/Os with LBA + length based on SGL memory or pass along a FLUSH with LBA + length.So once the '
tcm_eventfd_nvme' driver on KVM host receives a nvme host hardware frame via eventfd, it would decode the frame and send along the Read/Write/Flush when exposing existing (non nvme native) backend drivers.
Quoted 5 lines
Learned vhost architecture:
http://blog.vmsplice.net/2011/09/qemu-internals-vhost-architecture.htmlThe nice thing is it is not tied to KVM in any way.
Yes.
There are assumptions vhost currently makes about the guest using virtio queues however, and at least for an initial vhost_nvme prototype it's probably easier to avoid hacking up drivers/vhost/* (for now)..
(Adding MST CC')
Quoted 23 lines
For SCSI, there are "virtio-scsi" in guest kernel and "vhost-scsi" in host kernel.
For NVMe, there is no "virtio-nvme" in guest kernel(just unmodified NVMe driver), but I'll do similar thing in Qemu with vhost infrastructure. And there is "
vhost_nvme" in host kernel.For the "virtqueue" implementation in qemu-nvme, I'll possibly just use/copy
drivers/virtio/virtio_ring.c, same as whatlinux/tools/virtio/virtio_test.cdoes.A bit more detail graph as below. What do you think?
.-----------------------------------------. .------------------------. | Guest(Linux, Windows, FreeBSD, Solaris) | NVMe | qemu | | unmodified NVMe driver | command | NVMe device emulation | | | -------> | vhost + virtqueue | '-----------------------------------------' '------------------------' | | ^ passthrough | kick/notify NVMe command | via eventfd userspace via virtqueue | | | v v |
Quoted 1 line
----------------------------------------------------------------------------------
This should read something like:
Passthrough of nvme hardware frames via QEMU PCI-e struct vhost_mem into a custom vhost_nvme kernel driver ioctl using struct file + struct eventfd_ctx primitives.
Eg: QEMU user-space is not performing the nvme command decode before passing emulated nvme hardware frame up to host kernel driver.
Quoted 8 lines
.-----------------------------------------------------------------------. kernel | LIO frontend driver | | - vhost_nvme |'-----------------------------------------------------------------------'
| translate ^
| (NVMe command) |
| to |
v (LBA, length) |
vhost_nvme is performing host kernel level decode of user-space provided nvme hardware frames into nvme command + LBA +length + SGL buffer for target backend driver submission
Quoted 4 lines
.----------------------------------------------------------------------.
| LIO backend driver |
| - fileio (/mnt/xxx.file) |
| - iblock (/dev/sda1, /dev/nvme0n1, ...) |Quoted 8 lines
'----------------------------------------------------------------------'
| ^
| submit_bio() |
v |
.----------------------------------------------------------------------.
| block layer |
| |
'----------------------------------------------------------------------'For this part, HCH mentioned he is currently working on some code to pass native NVMe commands + SGL memory via blk-mq struct request into struct nvme_dev and/or struct nvme_queue.
Quoted 15 lines
| ^ | | v |.----------------------------------------------------------------------.
| block device driver |
| |
'----------------------------------------------------------------------'
| | | |
| | | |
v v v v
.------------. .-----------. .------------. .---------------.
| SATA | | SCSI | | NVMe | | .... |
'------------' '-----------' '------------' '---------------'
Looks fine.
Btw, after chatting with Dr. Hannes this week at SDC here are his original rts-megasas -v6 patches from Feb 2013.
Note they are standalone patches that require a sufficiently old enough LIO + QEMU to actually build + function.
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-qemu-v6.patch
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-fabric-v6.patch
For groking purposes, they demonstrate the principle design for a host kernel level driver, along with the megasas firmware interface (MFI) specific emulation magic that makes up the bulk of the code.
Take a look.
--nab
cover virtio nvme
Quoted 7 lines · Nicholas A. Bellinger
Btw, after chatting with Dr. Hannes this week at SDC here are his original rts-megasas -v6 patches from Feb 2013.
Note they are standalone patches that require a sufficiently old enough LIO + QEMU to actually build + function.
Quoted 8 lines
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-qemu-v6.patch
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-fabric-v6.patchFor groking purposes, they demonstrate the principle design for a host kernel level driver, along with the megasas firmware interface (MFI) specific emulation magic that makes up the bulk of the code.
Take a look.
Big thanks. Reading the patches now.
Quoted 3 lines
--nab
Quoted 7 lines · Nicholas A. Bellinger
Btw, after chatting with Dr. Hannes this week at SDC here are his original rts-megasas -v6 patches from Feb 2013.
Note they are standalone patches that require a sufficiently old enough LIO + QEMU to actually build + function.
Quoted 8 lines
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-qemu-v6.patch
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-fabric-v6.patchFor groking purposes, they demonstrate the principle design for a host kernel level driver, along with the megasas firmware interface (MFI) specific emulation magic that makes up the bulk of the code.
Take a look.
Big thanks. Reading the patches now.
Quoted 3 lines
--nab
cover virtio nvme
In reply to Nicholas A. BellingerQuoted 7 lines · Nicholas A. Bellinger
Quoted 3 lines · Ming Lin
Quoted 2 lines · Nicholas A. Bellinger
Quoted 1 line · Ming Lin
On Thu, 2015-09-17@17:55 -0700, Nicholas A. Bellinger wrote:
<SNIP>
<Even more SNIP>
Quoted 7 lines
Btw, after chatting with Dr. Hannes this week at SDC here are his original rts-megasas -v6 patches from Feb 2013.
Note they are standalone patches that require a sufficiently old enough LIO + QEMU to actually build + function.
Quoted 7 lines
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-qemu-v6.patch
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-fabric-v6.patchFor groking purposes, they demonstrate the principle design for a host kernel level driver, along with the megasas firmware interface (MFI) specific emulation magic that makes up the bulk of the code.
And indeed, Nic persuaded me to have them updated to qemu latest. Which I'll be doing shortly. Stay tuned.
Cheers,
Hannes
Signature · 5 lines
Dr. Hannes Reinecke zSeries & Storage hare at suse.de +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton HRB 21284 (AG N?rnberg)
Quoted 7 lines · Nicholas A. Bellinger
Quoted 3 lines · Ming Lin
Quoted 2 lines · Nicholas A. Bellinger
Quoted 1 line · Ming Lin
On Thu, 2015-09-17 at 17:55 -0700, Nicholas A. Bellinger wrote:
<SNIP>
<Even more SNIP>
Quoted 7 lines
Btw, after chatting with Dr. Hannes this week at SDC here are his original rts-megasas -v6 patches from Feb 2013.
Note they are standalone patches that require a sufficiently old enough LIO + QEMU to actually build + function.
Quoted 7 lines
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-qemu-v6.patch
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-fabric-v6.patchFor groking purposes, they demonstrate the principle design for a host kernel level driver, along with the megasas firmware interface (MFI) specific emulation magic that makes up the bulk of the code.
And indeed, Nic persuaded me to have them updated to qemu latest. Which I'll be doing shortly. Stay tuned.
Cheers,
Hannes
Signature · 9 lines
Dr. Hannes Reinecke zSeries & Storage [email protected] +49 911 74053 688 SUSE LINUX GmbH, Maxfeldstr. 5, 90409 Nürnberg GF: F. Imendörffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton HRB 21284 (AG Nürnberg) _______________________________________________ Virtualization mailing list [email protected] https://lists.linuxfoundation.org/mailman/listinfo/virtualization
Quoted 3 lines · Ming Lin
Quoted 2 lines · Nicholas A. Bellinger
Quoted 1 line · Ming Lin
On Thu, 2015-09-17 at 17:55 -0700, Nicholas A. Bellinger wrote:
<SNIP>
Quoted 9 lines
Quoted 8 lines
IBLOCK + FILEIO +
RD_MCPdon't speak SCSI, they simply process I/Os with LBA + length based on SGL memory or pass along a FLUSH with LBA + length.So once the '
tcm_eventfd_nvme' driver on KVM host receives a nvme host hardware frame via eventfd, it would decode the frame and send along the Read/Write/Flush when exposing existing (non nvme native) backend drivers.
Quoted 5 lines
Learned vhost architecture:
http://blog.vmsplice.net/2011/09/qemu-internals-vhost-architecture.htmlThe nice thing is it is not tied to KVM in any way.
Yes.
There are assumptions vhost currently makes about the guest using virtio queues however, and at least for an initial vhost_nvme prototype it's probably easier to avoid hacking up drivers/vhost/* (for now)..
(Adding MST CC')
Quoted 23 lines
For SCSI, there are "virtio-scsi" in guest kernel and "vhost-scsi" in host kernel.
For NVMe, there is no "virtio-nvme" in guest kernel(just unmodified NVMe driver), but I'll do similar thing in Qemu with vhost infrastructure. And there is "
vhost_nvme" in host kernel.For the "virtqueue" implementation in qemu-nvme, I'll possibly just use/copy
drivers/virtio/virtio_ring.c, same as whatlinux/tools/virtio/virtio_test.cdoes.A bit more detail graph as below. What do you think?
.-----------------------------------------. .------------------------. | Guest(Linux, Windows, FreeBSD, Solaris) | NVMe | qemu | | unmodified NVMe driver | command | NVMe device emulation | | | -------> | vhost + virtqueue | '-----------------------------------------' '------------------------' | | ^ passthrough | kick/notify NVMe command | via eventfd userspace via virtqueue | | | v v |
Quoted 1 line
----------------------------------------------------------------------------------
This should read something like:
Passthrough of nvme hardware frames via QEMU PCI-e struct vhost_mem into a custom vhost_nvme kernel driver ioctl using struct file + struct eventfd_ctx primitives.
Eg: QEMU user-space is not performing the nvme command decode before passing emulated nvme hardware frame up to host kernel driver.
Quoted 8 lines
.-----------------------------------------------------------------------. kernel | LIO frontend driver | | - vhost_nvme |'-----------------------------------------------------------------------'
| translate ^
| (NVMe command) |
| to |
v (LBA, length) |
vhost_nvme is performing host kernel level decode of user-space provided nvme hardware frames into nvme command + LBA +length + SGL buffer for target backend driver submission
Quoted 4 lines
.----------------------------------------------------------------------.
| LIO backend driver |
| - fileio (/mnt/xxx.file) |
| - iblock (/dev/sda1, /dev/nvme0n1, ...) |Quoted 8 lines
'----------------------------------------------------------------------'
| ^
| submit_bio() |
v |
.----------------------------------------------------------------------.
| block layer |
| |
'----------------------------------------------------------------------'For this part, HCH mentioned he is currently working on some code to pass native NVMe commands + SGL memory via blk-mq struct request into struct nvme_dev and/or struct nvme_queue.
Quoted 15 lines
| ^ | | v |.----------------------------------------------------------------------.
| block device driver |
| |
'----------------------------------------------------------------------'
| | | |
| | | |
v v v v
.------------. .-----------. .------------. .---------------.
| SATA | | SCSI | | NVMe | | .... |
'------------' '-----------' '------------' '---------------'
Looks fine.
Btw, after chatting with Dr. Hannes this week at SDC here are his original rts-megasas -v6 patches from Feb 2013.
Note they are standalone patches that require a sufficiently old enough LIO + QEMU to actually build + function.
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-qemu-v6.patch
https://github.com/Datera/rts-megasas/blob/master/rts_megasas-fabric-v6.patch
For groking purposes, they demonstrate the principle design for a host kernel level driver, along with the megasas firmware interface (MFI) specific emulation magic that makes up the bulk of the code.
Take a look.
--nab
Quoted 76 lines · Nicholas A. Bellinger
Quoted 65 lines · Ming Lin
Quoted 35 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.Quoted 9 lines
| NVMe driver | '--------------------------------------' | | v .-------------------------------------. | NVMe device | '-------------------------------------'
Quoted 4 lines
Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
Let me call the "eventfd based LIO NVMe fabric driver" as "tcm_eventfd_nvme"
Currently, LIO frontend driver(iscsi, fc, vhost-scsi etc) talk to LIO backend driver(fileio, iblock etc) with SCSI commands.
Did you mean the "tcm_eventfd_nvme" driver need to translate NVMe commands to SCSI commands and then submit to backend driver?
But I thought the future "LIO NVMe target" can support frontend driver talk to backend driver directly with NVMe commands without translation.
Am I wrong?
Quoted 10 lines
As with the nvme-over-fabric case, it would be possible to do a mapping between backend driver queue resources for real NVMe hardware (eg:
target_core_nvme), but since it would still be doing close to the same amount of software emulation for both backend driver cases, I wouldn't expect there to be much performance advantage over just using normalsubmit_bio().--nab
Quoted 65 lines · Ming Lin
Quoted 35 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?.------------------------. MMIO .---------------------------------------. | Guest |--------> | Qemu | | Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) | '------------------------' '---------------------------------------' | ^ write NVMe | | notify command command | | completion to eventfd | | to eventfd v | .--------------------------------------. | Host: | | eventfd based LIO NVMe fabric driver | '--------------------------------------' | | nvme_queue_rq() v .--------------------------------------.
Quoted 9 lines
| NVMe driver |
'--------------------------------------'
|
|
v
.-------------------------------------.
| NVMe device |
'-------------------------------------'Correct. The LIO driver on KVM host would be handling some amount of NVMe host interface emulation in kernel code, and would be able to decode nvme Read/Write/Flush operations and translate -> submit to existing backend drivers.
As with the nvme-over-fabric case, it would be possible to do a mapping between backend driver queue resources for real NVMe hardware (eg:target_core_nvme), but since it would still be doing close to the same amount of software emulation for both backend driver cases, I wouldn't expect there to be much performance advantage over just using normal submit_bio().
--nab
Quoted 35 lines · Nicholas A. Bellinger
Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-targetQuoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-targetI think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Quoted 8 lines
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Trying to understand it.
Is it like below?
.------------------------. MMIO .---------------------------------------.
| Guest |--------> | Qemu |
| Unmodified NVMe driver |<-------- | NVMe device simulation(eventfd based) |
'------------------------' '---------------------------------------'
| ^
write NVMe | | notify command
command | | completion
to eventfd | | to eventfd
v |
.--------------------------------------.
| Host: |
| eventfd based LIO NVMe fabric driver |
'--------------------------------------'
|
| nvme_queue_rq()
v
.--------------------------------------.
| NVMe driver |
'--------------------------------------'
|
|
v
.-------------------------------------.
| NVMe device |
'-------------------------------------'Hi Ming & Co,
Quoted 6 lines · Ming Lin
Quoted 5 lines · Stefan Hajnoczi
Quoted 4 lines · Ming Lin
These 2 patches added virtio-nvme to kernel and qemu, basically modified from virtio-blk and nvme code.
As title said, request for your comments.
<SNIP>
Quoted 14 lines
Quoted 4 lines
At first glance it seems like the
virtio_nvmeguest driver is just another block driver likevirtio_blk, so I'm not clear why a virtio-nvme device makes sense.I think the future "LIO NVMe target" only speaks NVMe protocol.
Nick(CCed), could you correct me if I'm wrong?
For SCSI stack, we have:
virtio-scsi(guest)tcm_vhost(or vhost_scsi, host)
LIO-scsi-target
Quoted 5 lines
For NVMe stack, we'll have similar components:
virtio-nvme(guest)vhost_nvme(host)
LIO-NVMe-target
I think it's more interesting to consider a 'vhost style' driver that can be used with unmodified nvme host OS drivers.
Dr. Hannes (CC'ed) had done something like this for megasas a few years back using specialized QEMU emulation + eventfd based LIO fabric driver, and got it working with Linux + MSFT guests.
Doing something similar for nvme would (potentially) be on par with current virtio-scsi+vhost-scsi small-block performance for scsi-mq guests, without the extra burden of a new command set specific virtio driver.
Quoted 18 lines
Quoted 18 lines
Quoted 17 lines
Now there are lots of duplicated code with
linux/nvme-core.candqemu/nvme.c. The ideal result is to have a multi level NVMe stack(similar as SCSI). So we can re-use the nvme code, for example.-------------------------.
| NVMe device register |Upper level | NVMe protocol process | | | '-------------------------' .-----------. .-----------. .------------------. Lower level | PCIe | | VIRTIO | |NVMe over Fabrics | | | | | |initiator | '-----------' '-----------' '------------------'
Quoted 8 lines
Quoted 3 lines
You mentioned LIO and SCSI. How will NVMe over Fabrics be integrated into LIO? If it is mapped to SCSI then using
virtio_scsiin the guest andtcm_vhostshould work.I think it's not mapped to SCSI.
Nick, would you share more here?
(Adding Dave M. CC')
So NVMe target code needs to function in at least two different modes:
- Direct mapping of nvme backend driver provided hw queues to nvme fabric driver provided hw queues.
- Decoding of NVMe command set for basic Read/Write/Flush I/O for submission to existing backend drivers (eg: iblock, fileio, rd_mcp)
With the former case, it's safe to assumes there to be anywhere from a very small amount of code involved, to no code involved for fast-path operation.
For more involved logic like PR, ALUA, and EXTENDED_COPY, I think both modes will still mostly likely handle some aspects of this in software, and not entirely behind a backend nvme host hw interface.
--nab