Skip to contentSkip to conversation

SCSI driver for VMware's virtual HBA - V3.

PATCH In review Alok Kataria 5 participants 9 messages Sep 3, 2009 – Sep 9, 2009 SCSI

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 SCSI driver for VMware's virtual HBA - V3.

VMware PVSCSI driver - v3.

Hi James,

Below is the version-3 of the patch, I have incorporated your and Eike's review comments. Please take a look.

Changelog (v3-v2)
- use kmalloc for allocating sg_lists. And pci_map it for device access.
- re-use pdev->revision for setting up adapter->rev.

Changelog (v2-v1)
- use PCI_VDEVICE instead of PCI_DEVICE.
- use list_first_entry
- use parenthesis for every sizeof usage
- get rid of all #ifdef for CONFIG_PCI_MSI
- use PVSCSI_MEM_SPACE_SIZE while checking for MMIO resource len.
- use kcalloc instead of kmalloc.
- replaced a couple of scmd_printk usage with dev_dbg
- use dev_info and pr_info at couple of places.
- add a comment to pvscsi_map_context
- Add entry in MAINTAINERS.

Patch applies on top of 2.6.31-rc8.

Thanks,
Alok

--
SCSI driver for VMware's virtual HBA.

From: Alok N Kataria <[email protected]>

This is a driver for VMware's paravirtualized SCSI device, which should improve disk performance for guests running under control of VMware hypervisors that support such devices.

---

 MAINTAINERS           |    7 
 drivers/scsi/Kconfig  |    8 
 drivers/scsi/Makefile |    1 
 drivers/scsi/pvscsi.c | 1396 +++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/scsi/pvscsi.h |  395 ++++++++++++++
 5 files changed, 1807 insertions(+), 0 deletions(-)
 create mode 100644 drivers/scsi/pvscsi.c
 create mode 100644 drivers/scsi/pvscsi.h
MAINTAINERS +7 -0
index 60299a9..952fe93 100644
@@ -5491,6 +5491,13 @@ S:	Maintained
 F:	drivers/vlynq/vlynq.c
 F:	include/linux/vlynq.h
 
+VMware PVSCSI driver
+M:	Alok Kataria <[email protected]>
+L:	[email protected]
+S:	Maintained
+F:	drivers/scsi/pvscsi.c
+F:	drivers/scsi/pvscsi.h
+
 VOLTAGE AND CURRENT REGULATOR FRAMEWORK
 M:	Liam Girdwood <[email protected]>
 M:	Mark Brown <[email protected]>
drivers/scsi/Kconfig +8 -0
index 9c23122..8a47981 100644
@@ -620,6 +620,14 @@ config SCSI_FLASHPOINT
 	  substantial, so users of MultiMaster Host Adapters may not
 	  wish to include it.
 
+config VMWARE_PVSCSI
+	tristate "VMware PVSCSI driver support"
+	depends on PCI && SCSI && X86
+	help
+	  This driver supports VMware's para virtualized SCSI HBA.
+	  To compile this driver as a module, choose M here: the
+	  module will be called pvscsi.
+
 config LIBFC
 	tristate "LibFC module"
 	select SCSI_FC_ATTRS
drivers/scsi/Makefile +1 -0
index 25429ea..27fd013 100644
@@ -130,6 +130,7 @@ obj-$(CONFIG_SCSI_MVSAS)	+= mvsas/
 obj-$(CONFIG_PS3_ROM)		+= ps3rom.o
 obj-$(CONFIG_SCSI_CXGB3_ISCSI)	+= libiscsi.o libiscsi_tcp.o cxgb3i/
 obj-$(CONFIG_SCSI_BNX2_ISCSI)	+= libiscsi.o bnx2i/
+obj-$(CONFIG_VMWARE_PVSCSI)	+= pvscsi.o
 
 obj-$(CONFIG_ARM)		+= arm/
 
drivers/scsi/pvscsi.c new file +1396 -0

1,396 lines not on this page · raw message

drivers/scsi/pvscsi.h new file +395 -0

395 lines not on this page · raw message

#1 lore raw

drivers/scsi/pvscsi.c line 243

Quoted 11 lines · Alok Kataria

VMware PVSCSI driver - v3.

drivers/scsi/pvscsi.c new file +9 -0
+static void pvscsi_write_cmd_desc(const struct pvscsi_adapter *adapter,
+				  u32 cmd, const void *desc, size_t len)
+{
+	const u32 *ptr = desc;
+	unsigned i;
+
+	len /= sizeof(*ptr);
+	pvscsi_reg_write(adapter, PVSCSI_REG_OFFSET_COMMAND, cmd);
+	for (i = 0; i < len; i++)

This would limit len to 16 GiB as "unsigned" (looks like "unsigned int" is preferred in kernel) is only 32 bit long. Instead i should probably be size_t, too.

Quoted 2 lines
(quoted diff) +2 -0
+static int __devinit pvscsi_allocate_rings(struct pvscsi_adapter *adapter)
+{

...

drivers/scsi/pvscsi.c line 426

Quoted 3 lines
drivers/scsi/pvscsi.c new file +3 -0
+	BUG_ON(adapter->ringStatePA & ~PAGE_MASK);
+	BUG_ON(adapter->reqRingPA   & ~PAGE_MASK);
+	BUG_ON(adapter->cmpRingPA   & ~PAGE_MASK);

Ok, that took me a moment. How about BUG_ON(!IS_ALIGNED(foo, PAGE_SIZE))?

drivers/scsi/pvscsi.c line 895

Quoted 9 lines
drivers/scsi/pvscsi.c new file +9 -0
+static const char *pvscsi_info(struct Scsi_Host *host)
+{
+	struct pvscsi_adapter *adapter = shost_priv(host);
+	static char buf[512];
+
+	sprintf(buf, "VMware PVSCSI storage adapter rev %d, req/cmp/msg rings: "
+		"%u/%u/%u pages, cmd_per_lun=%u", adapter->rev,
+		adapter->req_pages, adapter->cmp_pages, adapter->msg_pages,
+		pvscsi_template.cmd_per_lun);

The buffer looks rather huge here. I'm too lazy to calculate an exact size but 256 should still be more than enough, no?

Quoted 6 lines
drivers/scsi/pvscsi.c new file +6 -0
+static void pvscsi_shutdown_intr(struct pvscsi_adapter *adapter)
+{
+	if (adapter->irq) {
+		free_irq(adapter->irq, adapter);
+		adapter->irq = 0;
+	}

drivers/scsi/pvscsi.c line 1097

Quoted 10 lines
drivers/scsi/pvscsi.c new file +10 -0
+	if (adapter->use_msi) {
+		pci_disable_msi(adapter->dev);
+		adapter->use_msi = 0;
+	}
+
+	if (adapter->use_msix) {
+		pci_disable_msix(adapter->dev);
+		adapter->use_msix = 0;
+	}
+}

If MSI and MSI-X are exclusive the last one could be "else if".

Greetings,

Eike

#2 lore raw
In reply to Alok Kataria

drivers/scsi/pvscsi.c line 78

Quoted 10 lines · Alok Kataria
drivers/scsi/pvscsi.c new file +10 -0
+
+struct pvscsi_adapter {
+	char			*mmioBase;
+	unsigned int		irq;
+	u8			rev;
+	bool			use_msi;
+	bool			use_msix;
+	bool			use_msg;
+
+	spinlock_t		hw_lock;

Why not just use host_lock in the scsi_host structure?

Quoted 13 lines
drivers/scsi/pvscsi.c new file +13 -0
+
+/*
+ * Map all data buffers for a command into PCI space and
+ * setup the scatter/gather list if needed.
+ */
+static void pvscsi_map_buffers(struct pvscsi_adapter *adapter,
+			       struct pvscsi_ctx *ctx,
+			       struct scsi_cmnd *cmd, PVSCSIRingReqDesc *e)
+{
+	unsigned count;
+	unsigned bufflen = scsi_bufflen(cmd);
+	struct scatterlist *sg;
+

drivers/scsi/pvscsi.c line 349

Quoted 10 lines
drivers/scsi/pvscsi.c new file +10 -0
+	e->dataLen = bufflen;
+	e->dataAddr = 0;
+	if (bufflen == 0)
+		return;
+
+	sg = scsi_sglist(cmd);
+	count = scsi_sg_count(cmd);
+	if (count != 0) {
+		int segs = pci_map_sg(adapter->dev, sg, count,
+				      cmd->sc_data_direction);

Should use scsi_dma_map here instead.

Quoted 25 lines
drivers/scsi/pvscsi.c new file +25 -0
+		if (segs > 1) {
+			pvscsi_create_sg(ctx, sg, segs);
+
+			e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
+			ctx->sglPA = pci_map_single(adapter->dev, ctx->sgl,
+						    SGL_SIZE, PCI_DMA_TODEVICE);
+			e->dataAddr = ctx->sglPA;
+		} else
+			e->dataAddr = sg_dma_address(sg);
+	} else {
+		/*
+		 * In case there is no S/G list, scsi_sglist points
+		 * directly to the buffer.
+		 */
+		ctx->dataPA = pci_map_single(adapter->dev, sg, bufflen,
+					     cmd->sc_data_direction);
+		e->dataAddr = ctx->dataPA;
+	}
+}
+
+static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,
+				 struct pvscsi_ctx *ctx)
+{
+	struct scsi_cmnd *cmd;
+	unsigned bufflen;

drivers/scsi/pvscsi.c line 384

Quoted 10 lines
drivers/scsi/pvscsi.c new file +10 -0
+
+	cmd = ctx->cmd;
+	bufflen = scsi_bufflen(cmd);
+
+	if (bufflen != 0) {
+		unsigned count = scsi_sg_count(cmd);
+
+		if (count != 0) {
+			pci_unmap_sg(adapter->dev, scsi_sglist(cmd), count,
+				     cmd->sc_data_direction);

Use scsi_dma_unmap here instead.

Quoted 14 lines
drivers/scsi/pvscsi.c new file +14 -0
+			if (ctx->sglPA) {
+				pci_unmap_single(adapter->dev, ctx->sglPA,
+						 SGL_SIZE, PCI_DMA_TODEVICE);
+				ctx->sglPA = 0;
+			}
+		} else
+			pci_unmap_single(adapter->dev, ctx->dataPA, bufflen,
+					 cmd->sc_data_direction);
+	}
+	if (cmd->sense_buffer)
+		pci_unmap_single(adapter->dev, ctx->sensePA,
+				 SCSI_SENSE_BUFFERSIZE, PCI_DMA_FROMDEVICE);
+}
+
Signature · 3 lines
Brian King
Linux on Power Virtualization
IBM Linux Technology Center
#3 lore raw
Quoted 11 lines

-----Original Message-----
From: [email protected] [mailto:linux-kernel- [email protected]] On Behalf Of Brian King
Sent: Thursday, September 03, 2009 5:53 PM
To: [email protected]
Cc: James Bottomley; Rolf Eike Beer; Matthew Wilcox; Roland Dreier; Bart Van Assche; Robert Love; Randy Dunlap; Mike Christie; linux- [email protected]; LKML; Andrew Morton; [email protected]
Subject: Re: [PATCH] SCSI driver for VMware's virtual HBA - V3.

Alok Kataria wrote:

Quoted 13 lines

drivers/scsi/pvscsi.c line 78

Quoted 10 lines
drivers/scsi/pvscsi.c new file +10 -0
+
+struct pvscsi_adapter {
+	char			*mmioBase;
+	unsigned int		irq;
+	u8			rev;
+	bool			use_msi;
+	bool			use_msix;
+	bool			use_msg;
+
+	spinlock_t		hw_lock;

Why not just use host_lock in the scsi_host structure?

Why not an adapter_resource_lock? Why block everyone @ the scsi-mid layer.

Alok,

(This is just a optimization.You don't need it in order to get your code merged upstream.You can always patch it later on)In fact hw_lock should be split into two locks -

req_path_lock(aka tx_lock) and
cmpl_path_lock(aka rx_lock).

You can then have a full-duplex channel.

Revised-code -

	::pvscsi_acquire_context->spin_lock_irqsave(tx_lock)
	::pvscsi_isr->spin_lock_irqsave(rx_lock)
	::some_future_napi_style_pvrx_poll_func->spin_lock_irqsave(rx_lock)
	
	::for aborts synchronize on the rx_lock so that you know what's completed and what's not.

This is the kind of common code that should be provided by the bottom edge of the scsi-layer so that all the scsi-LLDD's can split their tx/rx paths!

Quoted 4 lines

Brian King
Linux on Power Virtualization
IBM Linux Technology Center

Chetan

#4 lore raw

Hi Chetan,

Quoted 11 lines

Alok,

(This is just a optimization.You don't need it in order to get your code merged upstream.You can always patch it later on)In fact hw_lock should be split into two locks -

req_path_lock(aka tx_lock) and
cmpl_path_lock(aka rx_lock).

You can then have a full-duplex channel.

Quoted 8 lines

Revised-code -

	::pvscsi_acquire_context->spin_lock_irqsave(tx_lock)
	::pvscsi_isr->spin_lock_irqsave(rx_lock)
	::some_future_napi_style_pvrx_poll_func->spin_lock_irqsave(rx_lock)
	
	::for aborts synchronize on the rx_lock so that you know what's

completed and what's not.

I agree about the optimization. Though IMO this will be more attractive once we have a device which supports multiple queue pairs. Anyways, let me keep this on my plate, I will check how much of an benefit this gets us and will decide if this should be done, but as you suggested this is for later.

Thanks for taking a look.

--Alok

Quoted 4 lines

This is the kind of common code that should be provided by the bottom edge of the scsi-layer so that all the scsi-LLDD's can split their tx/rx paths!

#5 lore raw
Quoted 36 lines

-----Original Message-----
From: Alok Kataria [mailto:[email protected]]
Sent: Tuesday, September 08, 2009 9:08 PM
To: Loke,Chetan
Cc: [email protected]; [email protected]; eike-kernel@sf- tec.de; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; akpm@linux- foundation.org; [email protected]
Subject: RE: [PATCH] SCSI driver for VMware's virtual HBA - V3.

Hi Chetan,

Quoted 11 lines

Alok,

(This is just a optimization.You don't need it in order to get your code merged upstream.You can always patch it later on)In fact hw_lock should be split into two locks -

req_path_lock(aka tx_lock) and
cmpl_path_lock(aka rx_lock).

You can then have a full-duplex channel.

Quoted 8 lines

Revised-code -

	::pvscsi_acquire_context->spin_lock_irqsave(tx_lock)
	::pvscsi_isr->spin_lock_irqsave(rx_lock)
	::some_future_napi_style_pvrx_poll_func->spin_lock_irqsave(rx_lock)

	::for aborts synchronize on the rx_lock so that you know what's

completed and what's not.

Quoted 2 lines

I agree about the optimization. Though IMO this will be more attractive once we have a device which supports multiple queue pairs.

lpfc driver has native multi-ring support. It won't create hundred's of ring-pairs but the stuff that's present is good enough to start with(at least for this generation).

Quoted 4 lines

Anyways, let me keep this on my plate, I will check how much of an benefit this gets us and will decide if this should be done, but as you suggested this is for later.

I keep forgetting that this is a pv-driver.So, to reap the benefits of the multi-ring arch, the plumbing should be present in your hypervisor/kernel and NOT in the guest-world.

Quoted 1 line

Thanks for taking a look.

No worries.

Quoted 2 lines

--Alok

Chetan

#6 lore raw
In reply to Brian King

Hi all,

Thanks for taking another look. I have addressed all of Eike and Bryan's comments except the host_lock one, see below for details. Also I have rolled out a v4 patch with all the changes, which I will send out separately.

Quoted 14 lines · Brian King

drivers/scsi/pvscsi.c line 78

Quoted 10 lines · Alok Kataria
drivers/scsi/pvscsi.c new file +10 -0
+
+struct pvscsi_adapter {
+	char			*mmioBase;
+	unsigned int		irq;
+	u8			rev;
+	bool			use_msi;
+	bool			use_msix;
+	bool			use_msg;
+
+	spinlock_t		hw_lock;

Why not just use host_lock in the scsi_host structure?

I agree with Chetan here, lets keep these locks separate.

Quoted 17 lines
Quoted 17 lines
drivers/scsi/pvscsi.c new file +17 -0
+
+/*
+ * Map all data buffers for a command into PCI space and
+ * setup the scatter/gather list if needed.
+ */
+static void pvscsi_map_buffers(struct pvscsi_adapter *adapter,
+			       struct pvscsi_ctx *ctx,
+			       struct scsi_cmnd *cmd, PVSCSIRingReqDesc *e)
+{
+	unsigned count;
+	unsigned bufflen = scsi_bufflen(cmd);
+	struct scatterlist *sg;
+
+	e->dataLen = bufflen;
+	e->dataAddr = 0;
+	if (bufflen == 0)
+		return;
Quoted 8 lines

drivers/scsi/pvscsi.c line 349

Quoted 6 lines
drivers/scsi/pvscsi.c new file +6 -0
+
+	sg = scsi_sglist(cmd);
+	count = scsi_sg_count(cmd);
+	if (count != 0) {
+		int segs = pci_map_sg(adapter->dev, sg, count,
+				      cmd->sc_data_direction);

Should use scsi_dma_map here instead.

Done.

Quoted 29 lines
Quoted 29 lines
drivers/scsi/pvscsi.c new file +29 -0
+		if (segs > 1) {
+			pvscsi_create_sg(ctx, sg, segs);
+
+			e->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
+			ctx->sglPA = pci_map_single(adapter->dev, ctx->sgl,
+						    SGL_SIZE, PCI_DMA_TODEVICE);
+			e->dataAddr = ctx->sglPA;
+		} else
+			e->dataAddr = sg_dma_address(sg);
+	} else {
+		/*
+		 * In case there is no S/G list, scsi_sglist points
+		 * directly to the buffer.
+		 */
+		ctx->dataPA = pci_map_single(adapter->dev, sg, bufflen,
+					     cmd->sc_data_direction);
+		e->dataAddr = ctx->dataPA;
+	}
+}
+
+static void pvscsi_unmap_buffers(const struct pvscsi_adapter *adapter,
+				 struct pvscsi_ctx *ctx)
+{
+	struct scsi_cmnd *cmd;
+	unsigned bufflen;
+
+	cmd = ctx->cmd;
+	bufflen = scsi_bufflen(cmd);
+
Quoted 8 lines

drivers/scsi/pvscsi.c line 384

Quoted 6 lines
drivers/scsi/pvscsi.c new file +6 -0
+	if (bufflen != 0) {
+		unsigned count = scsi_sg_count(cmd);
+
+		if (count != 0) {
+			pci_unmap_sg(adapter->dev, scsi_sglist(cmd), count,
+				     cmd->sc_data_direction);

Use scsi_dma_unmap here instead.

Done.

Quoted 17 lines
Quoted 5 lines
drivers/scsi/pvscsi.c new file +4 -0
+			if (ctx->sglPA) {
+				pci_unmap_single(adapter->dev, ctx->sglPA,
+						 SGL_SIZE, PCI_DMA_TODEVICE);
+				ctx->sglPA = 0;

drivers/scsi/pvscsi.c line 398

Quoted 10 lines
drivers/scsi/pvscsi.c new file +10 -0
+			}
+		} else
+			pci_unmap_single(adapter->dev, ctx->dataPA, bufflen,
+					 cmd->sc_data_direction);
+	}
+	if (cmd->sense_buffer)
+		pci_unmap_single(adapter->dev, ctx->sensePA,
+				 SCSI_SENSE_BUFFERSIZE, PCI_DMA_FROMDEVICE);
+}
+

Thanks,
Alok

#8 lore raw

Hi Gerd,

Quoted 5 lines · Gerd Hoffmann

Hi,

Quoted 1 line

SCSI driver for VMware's virtual HBA.

Is the source code for the backend side available somewhere too?

Nope, the device emulation code is not open sourced.

Alok

Quoted 4 lines

cheers,
Gerd