Skip to contentSkip to conversation

Virtio draft II

RFC PATCH In review Rusty Russell 10 participants 51 messages Jun 7, 2007 – Jun 28, 2007 19 messages not in the archive

Subsystems (6)
lore
Share & tools
lore
#0 lore raw

cover Virtio draft II

Hi again all,

It turns out that networking really wants ordered requests, which the
previous patches didn't allow. This patch changes it to a callback mechanism; kudos to Avi.

The downside is that locking is more complicated, and after a few dead
ends I implemented the simplest solution: the struct virtio_device contains the spinlock to use, and it's held when your callbacks get called.

Core changes:
1) struct virtio_device has a "lock" and "priv" fields (the latter for the driver to use, esp from callbacks).
2) add_outbuf and add_inbuf take a cb function ptr and void *, rather than a used pointer.
3) lengths of buffers must now fit in an unsigned int, not long.
4) the virtio_sync wrapper is gone.

Block driver changes (+30 lines)
1) Convert to callbacks not interrupt.
2) Ensure both outbuf & inbuf have been used up before finishing request. This avoids potential remote access to freed memory.

Net driver changes (+6 lines)
1) Convert to callbacks.
2) Store id in skb->cb, not array.
3) Remove NET_BUFS limit: we queue packets until virtio chokes.
4) Locking should now be correct, thanks mainly to virtio changes.

BTW, the lguest implementations (very rough) are in the lguest patch repo for your viewing pleasure:

Implementation over dumb read/write interface:
http://lguest.ozlabs.org/patches/?file/tip/new-io-lguest-readwrite.patch
Implementation using descriptors (assuming host can access guest mem):
http://lguest.ozlabs.org/patches/?file/tip/new-io-lguest-desc.patch

Cheers,
Rusty.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

#1 lore raw

RFC PATCH 1/3 Virtio draft II: virtio.h

This attempts to implement a "virtual I/O" layer which should allow common drivers to be efficiently used across most virtual I/O mechanisms. It will no-doubt need further enhancement.

The details of probing the device are left to hypervisor-specific
code: it simple constructs the "struct virtio_device" and hands it to the probe function (eg. virtnet_probe() or virtblk_probe()).

The virtio drivers add and detach input and output buffers; as the buffers are used up their associated callbacks are filled in.

I have written two virtio device drivers (net and block) and two virtio implementations (for lguest): a read-write socket-style implementation, and a more efficient descriptor-based implementation.

---
 include/linux/virtio.h |   75 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

===================================================================

include/linux/virtio.h new file +75 -0
@@ -0,0 +1,75 @@
+#ifndef _LINUX_VIRTIO_H
+#define _LINUX_VIRTIO_H
+#include <linux/types.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+
+/**
+ * virtio_device - description and routines to drive a virtual device.
+ * @lock: the lock to hold before calling any functions.
+ * @dev: the underlying struct device.
+ * @ops: the operations for this virtual device.
+ * @priv: private pointer for the driver to use.
+ */
+struct virtio_device {
+	spinlock_t lock;
+	struct device *dev;
+	struct virtio_ops *ops;
+	void *priv;
+};
+
+/**
+ * virtio_ops - virtio abstraction layer
+ * @add_outbuf: prepare to send data to the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	cb: the function to call once the outbuf is finished & detached.
+ *	data: the token to hand to the cb function.
+ *      Returns a unique id or an error.  Note that the callback will be
+ *	called with the lock held, and possibly in an interrupt handler.
+ * @add_inbuf: prepare to receive data from the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	cb: the function to call once the inbuf is finished & detached.
+ *	data: the token to hand to the cb function.
+ *      Returns a unique id or an error (eg. -ENOSPC).  Note that the
+ *	callback will be called with the lock held, and possibly in an
+ *	interrupt handler.
+ * @sync: update after add_inbuf/add_outbuf
+ *	vdev: the virtio_device we're talking about.
+ *	After one or more add_inbuf/add_outbuf calls, invoke this to kick
+ *	the virtio layer.
+ * @detach_outbuf: make sure sent sg can no longer be read.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_outbuf.
+ *	This is not necessary (or valid!) if the outbuf callback has
+ *	already fired.
+ * @detach_inbuf: make sure sent sg can no longer be written to.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_inbuf.
+ *	This is not necessary (or valid!) if the outbuf callback has
+ *	already fired.
+ */
+struct virtio_ops {
+	unsigned long (*add_outbuf)(struct virtio_device *vdev,
+				    const struct scatterlist sg[],
+				    unsigned int num,
+				    void (*cb)(struct virtio_device *vdev,
+					       void *data, unsigned len),
+				    void *data);
+
+	unsigned long (*add_inbuf)(struct virtio_device *vdev,
+				   struct scatterlist sg[],
+				   unsigned int num,
+				   void (*cb)(struct virtio_device *vdev,
+					      void *data, unsigned len),
+				   void *data);
+
+	void (*sync)(struct virtio_device *vdev);
+
+	void (*detach_outbuf)(struct virtio_device *vdev, unsigned long id);
+	void (*detach_inbuf)(struct virtio_device *vdev, unsigned long id);
+};
+#endif /* _LINUX_VIRTIO_H */

A message that is not in the archive (sent off-list or not yet received): 1181217867.14054.195.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#3 lore raw

RFC PATCH 2/3 Virtio draft II: example block driver

The block driver uses outbufs with sg[0] being the request information (struct virtio_blk_outhdr) with the type, sector and inbuf id. For a write, the rest of the sg will contain the data to be written.

The first segment of the inbuf is a result code (struct virtio_blk_inhdr). For a read, the rest of the sg points to the input buffer.

TODO:
1) Ordered tag support.

---
 drivers/block/Makefile     |    1 
 drivers/block/virtio_blk.c |  299 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_blk.h |   28 ++++
 3 files changed, 328 insertions(+)

===================================================================

drivers/block/Makefile +1 -0
@@ -20,6 +20,7 @@ obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.
 obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.o
 obj-$(CONFIG_BLK_DEV_DAC960)	+= DAC960.o
 obj-$(CONFIG_CDROM_PKTCDVD)	+= pktcdvd.o
+obj-y				+= virtio_blk.o
 
 obj-$(CONFIG_BLK_DEV_UMEM)	+= umem.o
 obj-$(CONFIG_BLK_DEV_NBD)	+= nbd.o

===================================================================

drivers/block/virtio_blk.c new file +299 -0

299 lines not on this page · raw message

===================================================================

include/linux/virtio_blk.h new file +28 -0
@@ -0,0 +1,28 @@
+#ifndef _LINUX_VIRTIO_BLK_H
+#define _LINUX_VIRTIO_BLK_H
+#include <linux/types.h>
+struct gendisk;
+struct virtio_device;
+struct hd_geometry;
+
+/* This is the first element of the scatter-gather list. */
+struct virtio_blk_outhdr
+{
+	/* 0 == read, 1 == write */
+	u32 type;
+	/* Sector (ie. 512 byte offset) */
+	unsigned long sector;
+	/* Where to put reply. */
+	unsigned long id;
+};
+
+struct virtio_blk_inhdr
+{
+	/* 1 = OK, 0 = not ok. */
+	unsigned long status;
+};
+
+struct gendisk *virtblk_probe(struct virtio_device *vdev);
+void virtblk_remove(struct gendisk *disk);
+
+#endif /* _LINUX_VIRTIO_BLK_H */

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1181217920.14054.196.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#5 lore raw

RFC PATCH 3/3 Virtio draft II: example net driver

TODO:
1) NAPI (will require way of suppressing virtio callbacks)
2) GSO.
3) Checksum options.

---
 drivers/net/Makefile       |    2 
 drivers/net/virtio_net.c   |  243 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_net.h |   12 ++
 3 files changed, 256 insertions(+), 1 deletion(-)

===================================================================

drivers/net/Makefile +1 -1
@@ -37,7 +37,7 @@ obj-$(CONFIG_CASSINI) += cassini.o
 
 obj-$(CONFIG_MACE) += mace.o
 obj-$(CONFIG_BMAC) += bmac.o
-
+obj-y += virtio_net.o
 obj-$(CONFIG_DGRS) += dgrs.o
 obj-$(CONFIG_VORTEX) += 3c59x.o
 obj-$(CONFIG_TYPHOON) += typhoon.o

===================================================================

drivers/net/virtio_net.c new file +243 -0

243 lines not on this page · raw message

===================================================================

include/linux/virtio_net.h new file +12 -0
@@ -0,0 +1,12 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+struct net_device;
+struct virtio_device;
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN]);
+void virtnet_remove(struct net_device *dev);
+
+#endif /* _LINUX_VIRTIO_NET_H */

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

#6 lore raw

Re: [PATCH RFC 2/3] Virtio draft II: example block driver

In reply to a message not in the archive

drivers/block/virtio_blk.c line 264

Quoted 2 lines · Rusty Russell
drivers/block/virtio_blk.c new file +1 -0
+	sprintf(vblk->disk->disk_name, "vb%c", virtblk_index++);

vd%c, in keeping with hd%c and sd%c?

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 4667F7C0.3070604-atKUWr5tajBWk0Htik3J/[email protected]

#8 lore raw

Re: [PATCH RFC 2/3] Virtio draft II: example block driver

Quoted 5 lines · Avi Kivity

drivers/block/virtio_blk.c line 264

Quoted 2 lines · Rusty Russell
drivers/block/virtio_blk.c new file +1 -0
+	sprintf(vblk->disk->disk_name, "vb%c", virtblk_index++);

vd%c, in keeping with hd%c and sd%c?

Should work, since iSeries' devices are /dev/iseries/vd%c. Is there already a magic routine to take the base and assign the a... aaa etc?

Rusty.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

#9 lore raw

Re: [PATCH RFC 2/3] Virtio draft II: example block driver

In reply to Avi Kivity
Quoted 5 lines · Avi Kivity

drivers/block/virtio_blk.c line 264

Quoted 2 lines · Rusty Russell
drivers/block/virtio_blk.c new file +1 -0
+	sprintf(vblk->disk->disk_name, "vb%c", virtblk_index++);

vd%c, in keeping with hd%c and sd%c?

Should work, since iSeries' devices are /dev/iseries/vd%c. Is there already a magic routine to take the base and assign the a... aaa etc?

Rusty.

#10 lore raw

RFC PATCH 3/3 Virtio draft II: example net driver

In reply to Rusty Russell

TODO:
1) NAPI (will require way of suppressing virtio callbacks)
2) GSO.
3) Checksum options.

---
 drivers/net/Makefile       |    2 
 drivers/net/virtio_net.c   |  243 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_net.h |   12 ++
 3 files changed, 256 insertions(+), 1 deletion(-)

===================================================================

drivers/net/Makefile +1 -1
@@ -37,7 +37,7 @@ obj-$(CONFIG_CASSINI) += cassini.o
 
 obj-$(CONFIG_MACE) += mace.o
 obj-$(CONFIG_BMAC) += bmac.o
-
+obj-y += virtio_net.o
 obj-$(CONFIG_DGRS) += dgrs.o
 obj-$(CONFIG_VORTEX) += 3c59x.o
 obj-$(CONFIG_TYPHOON) += typhoon.o

===================================================================

drivers/net/virtio_net.c new file +243 -0

243 lines not on this page · raw message

===================================================================

include/linux/virtio_net.h new file +12 -0
@@ -0,0 +1,12 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+struct net_device;
+struct virtio_device;
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN]);
+void virtnet_remove(struct net_device *dev);
+
+#endif /* _LINUX_VIRTIO_NET_H */
#11 lore raw

Re: [PATCH RFC 2/3] Virtio draft II: example block driver

In reply to Rusty Russell

drivers/block/virtio_blk.c line 264

Quoted 2 lines · Rusty Russell
drivers/block/virtio_blk.c new file +1 -0
+	sprintf(vblk->disk->disk_name, "vb%c", virtblk_index++);

vd%c, in keeping with hd%c and sd%c?

Signature · 1 line
error compiling committee.c: too many arguments to function
#12 lore raw

RFC PATCH 2/3 Virtio draft II: example block driver

In reply to Rusty Russell

The block driver uses outbufs with sg[0] being the request information (struct virtio_blk_outhdr) with the type, sector and inbuf id. For a write, the rest of the sg will contain the data to be written.

The first segment of the inbuf is a result code (struct virtio_blk_inhdr). For a read, the rest of the sg points to the input buffer.

TODO:
1) Ordered tag support.

---
 drivers/block/Makefile     |    1 
 drivers/block/virtio_blk.c |  299 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_blk.h |   28 ++++
 3 files changed, 328 insertions(+)

===================================================================

drivers/block/Makefile +1 -0
@@ -20,6 +20,7 @@ obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.
 obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.o
 obj-$(CONFIG_BLK_DEV_DAC960)	+= DAC960.o
 obj-$(CONFIG_CDROM_PKTCDVD)	+= pktcdvd.o
+obj-y				+= virtio_blk.o
 
 obj-$(CONFIG_BLK_DEV_UMEM)	+= umem.o
 obj-$(CONFIG_BLK_DEV_NBD)	+= nbd.o

===================================================================

drivers/block/virtio_blk.c new file +299 -0

299 lines not on this page · raw message

===================================================================

include/linux/virtio_blk.h new file +28 -0
@@ -0,0 +1,28 @@
+#ifndef _LINUX_VIRTIO_BLK_H
+#define _LINUX_VIRTIO_BLK_H
+#include <linux/types.h>
+struct gendisk;
+struct virtio_device;
+struct hd_geometry;
+
+/* This is the first element of the scatter-gather list. */
+struct virtio_blk_outhdr
+{
+	/* 0 == read, 1 == write */
+	u32 type;
+	/* Sector (ie. 512 byte offset) */
+	unsigned long sector;
+	/* Where to put reply. */
+	unsigned long id;
+};
+
+struct virtio_blk_inhdr
+{
+	/* 1 = OK, 0 = not ok. */
+	unsigned long status;
+};
+
+struct gendisk *virtblk_probe(struct virtio_device *vdev);
+void virtblk_remove(struct gendisk *disk);
+
+#endif /* _LINUX_VIRTIO_BLK_H */
#13 lore raw

RFC PATCH 1/3 Virtio draft II: virtio.h

In reply to Rusty Russell

This attempts to implement a "virtual I/O" layer which should allow common drivers to be efficiently used across most virtual I/O mechanisms. It will no-doubt need further enhancement.

The details of probing the device are left to hypervisor-specific
code: it simple constructs the "struct virtio_device" and hands it to the probe function (eg. virtnet_probe() or virtblk_probe()).

The virtio drivers add and detach input and output buffers; as the buffers are used up their associated callbacks are filled in.

I have written two virtio device drivers (net and block) and two virtio implementations (for lguest): a read-write socket-style implementation, and a more efficient descriptor-based implementation.

---
 include/linux/virtio.h |   75 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 75 insertions(+)

===================================================================

include/linux/virtio.h new file +75 -0
@@ -0,0 +1,75 @@
+#ifndef _LINUX_VIRTIO_H
+#define _LINUX_VIRTIO_H
+#include <linux/types.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+
+/**
+ * virtio_device - description and routines to drive a virtual device.
+ * @lock: the lock to hold before calling any functions.
+ * @dev: the underlying struct device.
+ * @ops: the operations for this virtual device.
+ * @priv: private pointer for the driver to use.
+ */
+struct virtio_device {
+	spinlock_t lock;
+	struct device *dev;
+	struct virtio_ops *ops;
+	void *priv;
+};
+
+/**
+ * virtio_ops - virtio abstraction layer
+ * @add_outbuf: prepare to send data to the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	cb: the function to call once the outbuf is finished & detached.
+ *	data: the token to hand to the cb function.
+ *      Returns a unique id or an error.  Note that the callback will be
+ *	called with the lock held, and possibly in an interrupt handler.
+ * @add_inbuf: prepare to receive data from the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	cb: the function to call once the inbuf is finished & detached.
+ *	data: the token to hand to the cb function.
+ *      Returns a unique id or an error (eg. -ENOSPC).  Note that the
+ *	callback will be called with the lock held, and possibly in an
+ *	interrupt handler.
+ * @sync: update after add_inbuf/add_outbuf
+ *	vdev: the virtio_device we're talking about.
+ *	After one or more add_inbuf/add_outbuf calls, invoke this to kick
+ *	the virtio layer.
+ * @detach_outbuf: make sure sent sg can no longer be read.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_outbuf.
+ *	This is not necessary (or valid!) if the outbuf callback has
+ *	already fired.
+ * @detach_inbuf: make sure sent sg can no longer be written to.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_inbuf.
+ *	This is not necessary (or valid!) if the outbuf callback has
+ *	already fired.
+ */
+struct virtio_ops {
+	unsigned long (*add_outbuf)(struct virtio_device *vdev,
+				    const struct scatterlist sg[],
+				    unsigned int num,
+				    void (*cb)(struct virtio_device *vdev,
+					       void *data, unsigned len),
+				    void *data);
+
+	unsigned long (*add_inbuf)(struct virtio_device *vdev,
+				   struct scatterlist sg[],
+				   unsigned int num,
+				   void (*cb)(struct virtio_device *vdev,
+					      void *data, unsigned len),
+				   void *data);
+
+	void (*sync)(struct virtio_device *vdev);
+
+	void (*detach_outbuf)(struct virtio_device *vdev, unsigned long id);
+	void (*detach_inbuf)(struct virtio_device *vdev, unsigned long id);
+};
+#endif /* _LINUX_VIRTIO_H */

A message that is not in the archive (sent off-list or not yet received): 1181217762.14054.192.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#15 lore raw

cover Virtio draft III

In this episode, Rusty tries to NAPI-ize the driver and discovers that virtio callbacks are a bad idea: NAPI needs to turn interrupts off and still be able to query for new incoming packets.

Changes to core:
1) Back to "interrupt" model with get_inbuf()/get_outbuf() calls.
2) Clearer rules for locking: in calls cannot overlap, out calls cannot overlap, but in can overlap out.
3) Methods for suppressing/enabling "interrupt" calls.

Changes to example net driver:
1) NAPI, locking is now correct (and there is none)

Changes to example block driver:
1) Relay SCSI ioctls (particularly CDROMEJECT) for optional server support (VIRTIO_BLK_T_SCSI_CMD).
2) /dev/vb -> /dev/vd.
3) Barrier support.
4) Header w/ definitions can be included from userspace.

Here's the inter-diff for the three:

include/linux/virtio.h +61 -21
diff -u b/include/linux/virtio.h b/include/linux/virtio.h
@@ -7,69 +7,109 @@
 /**
  * virtio_device - description and routines to drive a virtual device.
- * @lock: the lock to hold before calling any functions.
  * @dev: the underlying struct device.
  * @ops: the operations for this virtual device.
+ * @driver_ops: set by the driver for callbacks.
  * @priv: private pointer for the driver to use.
  */
 struct virtio_device {
-	spinlock_t lock;
 	struct device *dev;
 	struct virtio_ops *ops;
+	struct virtio_driver_ops *driver_ops;
 	void *priv;
 };
 
 /**
+ * virtio_driver_ops - driver callbacks for a virtual device.
+ * @in: inbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further inbuf callbacks.
+ * @out: outbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further outbuf callbacks.
+ */
+struct virtio_driver_ops {
+	bool (*in)(struct virtio_device *dev);
+	bool (*out)(struct virtio_device *dev);
+};
+
+enum virtio_dir {
+	VIRTIO_IN = 0x1,
+	VIRTIO_OUT = 0x2,
+};
+
+/**
  * virtio_ops - virtio abstraction layer
  * @add_outbuf: prepare to send data to the other end:
  *	vdev: the virtio_device
  *	sg: the description of the buffer(s).
  *	num: the size of the sg array.
- *	cb: the function to call once the outbuf is finished & detached.
- *	data: the token to hand to the cb function.
- *      Returns a unique id or an error.  Note that the callback will be
- *	called with the lock held, and possibly in an interrupt handler.
+ *	data: the token returned by the get_outbuf function.
+ *      Returns a unique id or an error.
  * @add_inbuf: prepare to receive data from the other end:
  *	vdev: the virtio_device
  *	sg: the description of the buffer(s).
  *	num: the size of the sg array.
- *	cb: the function to call once the inbuf is finished & detached.
- *	data: the token to hand to the cb function.
- *      Returns a unique id or an error (eg. -ENOSPC).  Note that the
- *	callback will be called with the lock held, and possibly in an
- *	interrupt handler.
- * @sync: update after add_inbuf/add_outbuf
+ *	data: the token returned by the get_inbuf function.
+ *      Returns a unique id or an error (eg. -ENOSPC).
+ * @sync: update after add_inbuf and/or add_outbuf
  *	vdev: the virtio_device we're talking about.
+ *	inout: VIRTIO_IN and/or VIRTIO_OUT
  *	After one or more add_inbuf/add_outbuf calls, invoke this to kick
  *	the virtio layer.
+ * @get_outbuf: get the next used outbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length written into the outbuf
+ *	Returns NULL or the "data" token handed to add_outbuf (which has been
+ *	detached).
+ * @get_inbuf: get the next used inbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length read from the inbuf
+ *	Returns NULL or the "data" token handed to add_inbuf (which has been
+ *	detached).
  * @detach_outbuf: make sure sent sg can no longer be read.
  *	vdev: the virtio_device we're talking about.
  *	id: the id returned from add_outbuf.
- *	This is not necessary (or valid!) if the outbuf callback has
- *	already fired.
+ *	This is usually used for shutdown.  Don't try to detach twice.
  * @detach_inbuf: make sure sent sg can no longer be written to.
  *	vdev: the virtio_device we're talking about.
  *	id: the id returned from add_inbuf.
- *	This is not necessary (or valid!) if the outbuf callback has
- *	already fired.
+ *	This is usually used for shutdown.  Don't try to detach twice.
+ * @restart_in: restart calls to driver_ops->in after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	inbufs, to avoid a race.
+ * @restart_out: restart calls to driver_ops->out after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	outbufs, to avoid a race.
+ *
+ * Locking rules are straightforward: the driver is responsible for
+ * locking.  Outbuf operations can be called in parallel to inbuf
+ * operations, but no two outbuf operations nor two inbuf operations
+ * may be invoked simultaneously.
+ *
+ * All operations can be called in any context.
  */
 struct virtio_ops {
 	unsigned long (*add_outbuf)(struct virtio_device *vdev,
 				    const struct scatterlist sg[],
 				    unsigned int num,
-				    void (*cb)(struct virtio_device *vdev,
-					       void *data, unsigned len),
 				    void *data);
 
 	unsigned long (*add_inbuf)(struct virtio_device *vdev,
 				   struct scatterlist sg[],
 				   unsigned int num,
-				   void (*cb)(struct virtio_device *vdev,
-					      void *data, unsigned len),
 				   void *data);
 
-	void (*sync)(struct virtio_device *vdev);
+	void (*sync)(struct virtio_device *vdev, enum virtio_dir inout);
+
+	void *(*get_outbuf)(struct virtio_device *vdev, unsigned int *len);
+	void *(*get_inbuf)(struct virtio_device *vdev, unsigned int *len);
 
 	void (*detach_outbuf)(struct virtio_device *vdev, unsigned long id);
 	void (*detach_inbuf)(struct virtio_device *vdev, unsigned long id);
+
+	bool (*restart_in)(struct virtio_device *vdev);
+	bool (*restart_out)(struct virtio_device *vdev);
 };
 #endif /* _LINUX_VIRTIO_H */
drivers/net/virtio_net.c +92 -58

248 lines not on this page · raw message

drivers/block/virtio_blk.c +152 -66

363 lines not on this page · raw message

include/linux/virtio_blk.h +21 -10
diff -u b/include/linux/virtio_blk.h b/include/linux/virtio_blk.h
@@ -3,26 +3,37 @@
 #include <linux/types.h>
-struct gendisk;
-struct virtio_device;
-struct hd_geometry;
+
+#define VIRTIO_BLK_T_READ	0
+#define VIRTIO_BLK_T_WRITE 1
+#define VIRTIO_BLK_T_SCSI_CMD 3
+#define VIRTIO_BLK_T_BARRIER 0x80000000 /* Barrier before this op. */
 
 /* This is the first element of the scatter-gather list. */
 struct virtio_blk_outhdr
 {
-	/* 0 == read, 1 == write */
-	u32 type;
+	/* VIRTIO_BLK_T* */
+	__u32 type;
+	/* io priority. */
+	__u32 ioprio;
 	/* Sector (ie. 512 byte offset) */
-	unsigned long sector;
+	__u64 sector;
 	/* Where to put reply. */
-	unsigned long id;
+	__u64 id;
 };
 
+#define VIRTIO_BLK_S_OK		0
+#define VIRTIO_BLK_S_IOERR	1
+#define VIRTIO_BLK_S_UNSUPP	2
+
 struct virtio_blk_inhdr
 {
-	/* 1 = OK, 0 = not ok. */
-	unsigned long status;
+	unsigned char status;
 };
 
+#ifdef __KERNEL__
+struct gendisk;
+struct virtio_device;
+
 struct gendisk *virtblk_probe(struct virtio_device *vdev);
 void virtblk_remove(struct gendisk *disk);
-
+#endif /* __KERNEL__ */
 #endif /* _LINUX_VIRTIO_BLK_H */

only in patch2:
unchanged:

include/linux/Kbuild +1 -0
@@ -341,6 +341,7 @@ unifdef-y += utsname.h
 unifdef-y += utsname.h
 unifdef-y += videodev2.h
 unifdef-y += videodev.h
+unifdef-y += virtio_blk.h
 unifdef-y += wait.h
 unifdef-y += wanrouter.h
 unifdef-y += watchdog.h

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1181999552.6237.255.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#17 lore raw

RFC PATCH 1/3 Virtio draft III: virtio.h

Draft III of virtio interface

This attempts to implement a "virtual I/O" layer which should allow common drivers to be efficiently used across most virtual I/O mechanisms. It will no-doubt need further enhancement.

The details of probing the device are left to hypervisor-specific
code: it simple constructs the "struct virtio_device" and hands it to the probe function (eg. virtnet_probe() or virtblk_probe()).

The virtio drivers add and get input and output buffers; as the buffers are used up the driver "interrupt" callbacks are invoked.

I have written two virtio device drivers (net and block) and two virtio implementations (for lguest): a read-write socket-style implementation, and a more efficient descriptor-based implementation.

---
 include/linux/virtio.h |  115 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)

===================================================================

include/linux/virtio.h new file +115 -0
@@ -0,0 +1,115 @@
+#ifndef _LINUX_VIRTIO_H
+#define _LINUX_VIRTIO_H
+#include <linux/types.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+
+/**
+ * virtio_device - description and routines to drive a virtual device.
+ * @dev: the underlying struct device.
+ * @ops: the operations for this virtual device.
+ * @driver_ops: set by the driver for callbacks.
+ * @priv: private pointer for the driver to use.
+ */
+struct virtio_device {
+	struct device *dev;
+	struct virtio_ops *ops;
+	struct virtio_driver_ops *driver_ops;
+	void *priv;
+};
+
+/**
+ * virtio_driver_ops - driver callbacks for a virtual device.
+ * @in: inbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further inbuf callbacks.
+ * @out: outbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further outbuf callbacks.
+ */
+struct virtio_driver_ops {
+	bool (*in)(struct virtio_device *dev);
+	bool (*out)(struct virtio_device *dev);
+};
+
+enum virtio_dir {
+	VIRTIO_IN = 0x1,
+	VIRTIO_OUT = 0x2,
+};
+
+/**
+ * virtio_ops - virtio abstraction layer
+ * @add_outbuf: prepare to send data to the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	data: the token returned by the get_outbuf function.
+ *      Returns a unique id or an error.
+ * @add_inbuf: prepare to receive data from the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	data: the token returned by the get_inbuf function.
+ *      Returns a unique id or an error (eg. -ENOSPC).
+ * @sync: update after add_inbuf and/or add_outbuf
+ *	vdev: the virtio_device we're talking about.
+ *	inout: VIRTIO_IN and/or VIRTIO_OUT
+ *	After one or more add_inbuf/add_outbuf calls, invoke this to kick
+ *	the virtio layer.
+ * @get_outbuf: get the next used outbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length written into the outbuf
+ *	Returns NULL or the "data" token handed to add_outbuf (which has been
+ *	detached).
+ * @get_inbuf: get the next used inbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length read from the inbuf
+ *	Returns NULL or the "data" token handed to add_inbuf (which has been
+ *	detached).
+ * @detach_outbuf: make sure sent sg can no longer be read.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_outbuf.
+ *	This is usually used for shutdown.  Don't try to detach twice.
+ * @detach_inbuf: make sure sent sg can no longer be written to.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_inbuf.
+ *	This is usually used for shutdown.  Don't try to detach twice.
+ * @restart_in: restart calls to driver_ops->in after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	inbufs, to avoid a race.
+ * @restart_out: restart calls to driver_ops->out after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	outbufs, to avoid a race.
+ *
+ * Locking rules are straightforward: the driver is responsible for
+ * locking.  Outbuf operations can be called in parallel to inbuf
+ * operations, but no two outbuf operations nor two inbuf operations
+ * may be invoked simultaneously.
+ *
+ * All operations can be called in any context.
+ */
+struct virtio_ops {
+	unsigned long (*add_outbuf)(struct virtio_device *vdev,
+				    const struct scatterlist sg[],
+				    unsigned int num,
+				    void *data);
+
+	unsigned long (*add_inbuf)(struct virtio_device *vdev,
+				   struct scatterlist sg[],
+				   unsigned int num,
+				   void *data);
+
+	void (*sync)(struct virtio_device *vdev, enum virtio_dir inout);
+
+	void *(*get_outbuf)(struct virtio_device *vdev, unsigned int *len);
+	void *(*get_inbuf)(struct virtio_device *vdev, unsigned int *len);
+
+	void (*detach_outbuf)(struct virtio_device *vdev, unsigned long id);
+	void (*detach_inbuf)(struct virtio_device *vdev, unsigned long id);
+
+	bool (*restart_in)(struct virtio_device *vdev);
+	bool (*restart_out)(struct virtio_device *vdev);
+};
+#endif /* _LINUX_VIRTIO_H */

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1181999669.6237.257.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#19 lore raw

RFC PATCH 2/3 Virtio draft III: example net driver

Net driver using virtio

TODO:
1) GSO.
2) Checksum options.
3) Big packets.

---
 drivers/net/Makefile       |    2 
 drivers/net/virtio_net.c   |  277 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_net.h |   12 +
 3 files changed, 290 insertions(+), 1 deletion(-)

===================================================================

drivers/net/Makefile +1 -1
@@ -37,7 +37,7 @@ obj-$(CONFIG_CASSINI) += cassini.o
 
 obj-$(CONFIG_MACE) += mace.o
 obj-$(CONFIG_BMAC) += bmac.o
-
+obj-y += virtio_net.o
 obj-$(CONFIG_DGRS) += dgrs.o
 obj-$(CONFIG_VORTEX) += 3c59x.o
 obj-$(CONFIG_TYPHOON) += typhoon.o

===================================================================

drivers/net/virtio_net.c new file +277 -0

277 lines not on this page · raw message

===================================================================

include/linux/virtio_net.h new file +12 -0
@@ -0,0 +1,12 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+struct net_device;
+struct virtio_device;
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN]);
+void virtnet_remove(struct net_device *dev);
+
+#endif /* _LINUX_VIRTIO_NET_H */

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1181999825.6237.260.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#21 lore raw

RFC PATCH 3/3 Virtio draft III: example block driver

The block driver uses outbufs with sg[0] being the request information (struct virtio_blk_outhdr) with the type, sector and inbuf id. For a write, the rest of the sg will contain the data to be written.

The first segment of the inbuf is a result code (struct virtio_blk_inhdr). For a read, the rest of the sg points to the input buffer.

We accept the normal SCSI ioctls: they get handed through to the other side which can then handle it or reply that it's unsupported.

Although we try to reply -ENOTTY on unsupported commands, the block layer in its infinite wisdom suppressed the error so ioctl(fd, CDROMEJECT) returns success to userspace.

---
 drivers/block/Makefile     |    1 
 drivers/block/virtio_blk.c |  385 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/Kbuild       |    1 
 include/linux/virtio_blk.h |   39 ++++
 4 files changed, 426 insertions(+)

===================================================================

drivers/block/Makefile +1 -0
@@ -20,6 +20,7 @@ obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.
 obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.o
 obj-$(CONFIG_BLK_DEV_DAC960)	+= DAC960.o
 obj-$(CONFIG_CDROM_PKTCDVD)	+= pktcdvd.o
+obj-y				+= virtio_blk.o
 
 obj-$(CONFIG_BLK_DEV_UMEM)	+= umem.o
 obj-$(CONFIG_BLK_DEV_NBD)	+= nbd.o

===================================================================

drivers/block/virtio_blk.c new file +383 -0

383 lines not on this page · raw message

===================================================================

include/linux/Kbuild +1 -0
@@ -341,6 +341,7 @@ unifdef-y += utsname.h
 unifdef-y += utsname.h
 unifdef-y += videodev2.h
 unifdef-y += videodev.h
+unifdef-y += virtio_blk.h
 unifdef-y += wait.h
 unifdef-y += wanrouter.h
 unifdef-y += watchdog.h

===================================================================

include/linux/virtio_blk.h new file +39 -0
@@ -0,0 +1,39 @@
+#ifndef _LINUX_VIRTIO_BLK_H
+#define _LINUX_VIRTIO_BLK_H
+#include <linux/types.h>
+
+#define VIRTIO_BLK_T_READ	0
+#define VIRTIO_BLK_T_WRITE 1
+#define VIRTIO_BLK_T_SCSI_CMD 3
+#define VIRTIO_BLK_T_BARRIER 0x80000000 /* Barrier before this op. */
+
+/* This is the first element of the scatter-gather list. */
+struct virtio_blk_outhdr
+{
+	/* VIRTIO_BLK_T* */
+	__u32 type;
+	/* io priority. */
+	__u32 ioprio;
+	/* Sector (ie. 512 byte offset) */
+	__u64 sector;
+	/* Where to put reply. */
+	__u64 id;
+};
+
+#define VIRTIO_BLK_S_OK		0
+#define VIRTIO_BLK_S_IOERR	1
+#define VIRTIO_BLK_S_UNSUPP	2
+
+struct virtio_blk_inhdr
+{
+	unsigned char status;
+};
+
+#ifdef __KERNEL__
+struct gendisk;
+struct virtio_device;
+
+struct gendisk *virtblk_probe(struct virtio_device *vdev);
+void virtblk_remove(struct gendisk *disk);
+#endif /* __KERNEL__ */
+#endif /* _LINUX_VIRTIO_BLK_H */

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1181999920.6237.263.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#23 lore raw

PATCH 1/1 Lguest implemention of virtio draft III

This is a bonus patch for those wondering how a virtio implementation can look. I have two, this is the more efficient one (needs some modification for inter-guest though: it assumes the other end does all the accessing of our memory. It's currently tacked on to the existing lguest I/O mechanism as a demonstration, rather than replacing it.

It shows that it's possible to implement virtio without internal locking.

Userspace server-side code isn't included.
===
This allows zero-copy from guest <-> host. It uses a page of descriptors, a page to say what descriptors to use, and a page to say what's been used: one each set for inbufs and one for outbufs.

TODO:
1) More polishing
2) Get rid of old I/O
3) Inter-guest I/O implementation

---
 drivers/lguest/Makefile         |    2 
 drivers/lguest/hypercalls.c     |    4 
 drivers/lguest/lguest_virtio.c  |  511 +++++++++++++++++++++++++++++++++++++++
 include/linux/lguest.h          |    3 
 include/linux/lguest_launcher.h |   24 +
 6 files changed, 948 insertions(+), 5 deletions(-)
drivers/lguest/Makefile +1 -1
@@ -1,5 +1,5 @@
 # Guest requires the paravirt_ops replacement and the bus driver.
-obj-$(CONFIG_LGUEST_GUEST) += lguest.o lguest_asm.o lguest_bus.o
+obj-$(CONFIG_LGUEST_GUEST) += lguest.o lguest_asm.o lguest_bus.o lguest_virtio.o
 
 # Host requires the other files, which can be a module.
 obj-$(CONFIG_LGUEST)	+= lg.o

===================================================================

drivers/lguest/hypercalls.c +4 -0
@@ -86,6 +86,10 @@ static void do_hcall(struct lguest *lg, 
 		break;
 	case LHCALL_HALT:
 		lg->halted = 1;
+		break;
+	case LHCALL_NOTIFY:
+		lg->pending_key = regs->edx << PAGE_SHIFT;
+		lg->dma_is_pending = 1;
 		break;
 	default:
 		kill_guest(lg, "Bad hypercall %li\n", regs->eax);

===================================================================

drivers/lguest/lguest_virtio.c new file +511 -0

511 lines not on this page · raw message

===================================================================

include/linux/lguest.h +3 -0
@@ -23,6 +23,9 @@
 #define LHCALL_SET_PTE		14
 #define LHCALL_SET_PMD		15
 #define LHCALL_LOAD_TLS		16
+
+/* Experimental hcalls for new I/O */
+#define LHCALL_NOTIFY	100 /* pfn */
 
 #define LG_CLOCK_MIN_DELTA	100UL
 #define LG_CLOCK_MAX_DELTA	ULONG_MAX

===================================================================

include/linux/lguest_launcher.h +24 -0
@@ -44,6 +44,8 @@ struct lguest_device_desc {
 #define LGUEST_DEVICE_T_CONSOLE	1
 #define LGUEST_DEVICE_T_NET	2
 #define LGUEST_DEVICE_T_BLOCK	3
+#define LGUEST_DEVICE_T_VIRTNET	8
+#define LGUEST_DEVICE_T_VIRTBLK	9
 
 	u16 features;
 #define LGUEST_NET_F_NOCSUM		0x4000 /* Don't bother checksumming */
@@ -70,4 +72,26 @@ enum lguest_req
 	LHREQ_IRQ, /* + irq */
 	LHREQ_BREAK, /* + on/off flag (on blocks until someone does off) */
 };
+
+/* This marks a buffer as being the start (and active) */
+#define LGUEST_DESC_F_HEAD	1
+/* This marks a buffer as continuing via the next field. */
+#define LGUEST_DESC_F_NEXT	2
+
+/* Virtio descriptors */
+struct lguest_desc
+{
+	unsigned long pfn;
+	unsigned long len;
+	u16 offset;
+	u16 flags;
+	/* We chain unused descriptors via this, too */
+	u32 next;
+};
+
+struct lguest_used
+{
+	unsigned int id;
+	unsigned int len;
+};
 #endif /* _ASM_LGUEST_USER */

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1182000514.6237.273.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#25 lore raw

Re: [PATCH] Lguest implemention of virtio draft III

Quoted 2 lines · Rusty Russell

This is a bonus patch for those wondering how a virtio implementation can look.

Thanks, I assumed it was something like this, but having the code in front of me makes it a lot easier to comment on it.

Quoted 55 lines
drivers/lguest/lguest_virtio.c new file +55 -0
+static struct lguest_driver lguest_virtnet_drv = {
+	.name = "lguestvirtnet",
+	.owner = THIS_MODULE,
+	.device_type = LGUEST_DEVICE_T_VIRTNET,
+	.probe = lguest_virtnet_probe,
+};
+
+static __init int lguest_virtnet_init(void)
+{
+	return register_lguest_driver(&lguest_virtnet_drv);
+}
+device_initcall(lguest_virtnet_init);
+
+/* Example block driver code. */
+#include <linux/virtio_blk.h>
+#include <linux/genhd.h>
+#include <linux/blkdev.h>
+static int lguest_virtblk_probe(struct lguest_device *lgdev)
+{
+	struct lguest_virtio_device *lgv;
+	struct gendisk *disk;
+	unsigned long sectors;
+	int err;
+
+	lgv = lg_new_virtio(lgdev);
+	if (!lgv)
+		return -ENOMEM;
+
+	/* Page is initially used to pass capacity. */
+	sectors = *(unsigned long *)lgv->in.desc;
+	*(unsigned long *)lgv->in.desc = 0;
+
+	lgv->priv = disk = virtblk_probe(&lgv->vdev);
+	if (IS_ERR(lgv->priv)) {
+		err = PTR_ERR(lgv->priv);
+		goto destroy;
+	}
+	set_capacity(disk, sectors);
+	blk_queue_max_hw_segments(disk->queue, NUM_DESCS-1);
+
+	err = lg_setup_interrupt(lgv, disk->disk_name);
+	if (err)
+		goto unprobe;
+	add_disk(disk);
+	return 0;
+
+unprobe:
+	virtblk_remove(disk);
+destroy:
+	lg_destroy_virtio(lgv);
+	return err;
+}
+
+static struct lguest_driver lguest_virtblk_drv = {
+	.name = "lguestvirtblk",

drivers/lguest/lguest_virtio.c line 509

Quoted 10 lines
drivers/lguest/lguest_virtio.c new file +10 -0
+	.owner = THIS_MODULE,
+	.device_type = LGUEST_DEVICE_T_VIRTBLK,
+	.probe = lguest_virtblk_probe,
+};
+
+static __init int lguest_virtblk_init(void)
+{
+	return register_lguest_driver(&lguest_virtblk_drv);
+}
+device_initcall(lguest_virtblk_init);

This is the part that I find puzzling. My idea would be that the drivers and devices you register are _not_ specific to the type of virtio bus.

The problem I see with your approach is that drivers for specific devices can't be loaded as separate modules, right now all of them have to be loaded or statically linked into the kernel in order to load the lguest_virtio module.

I'm sure you know how this works for other buses, but I'll explain anyway so that everyone knows what I'm talking about. What I have in mind is to have a single 'struct bus_type virtio_bus_type', which is used by all virtio implementations. The 'struct virtio_device' is derived from 'struct device', meaning that a 'struct device' is embedded in it. Each virtio implementation can either use the virtio_device directly, or derive its own kvm_virtio_device/lguest_virtio_device/... from it. When lguest scans its devices, it registers the lguest_virtio_device with the generic virtio_bus_type, not an lguest specific one. Similarly, the block/net/... drivers each register themselves as a virtio_driver with the virtio_bus_type, not with an implementation specific guest.

Do you see a problem with the approach that I described here, or did you just not bother implementing it so far?

	Arnd <><

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

#26 lore raw

Re: [kvm-devel] [PATCH] Lguest implemention of virtio draft III

In reply to Rusty Russell
Quoted 2 lines · Rusty Russell

This is a bonus patch for those wondering how a virtio implementation can look.

Thanks, I assumed it was something like this, but having the code in front of me makes it a lot easier to comment on it.

Quoted 55 lines
drivers/lguest/lguest_virtio.c new file +55 -0
+static struct lguest_driver lguest_virtnet_drv = {
+	.name = "lguestvirtnet",
+	.owner = THIS_MODULE,
+	.device_type = LGUEST_DEVICE_T_VIRTNET,
+	.probe = lguest_virtnet_probe,
+};
+
+static __init int lguest_virtnet_init(void)
+{
+	return register_lguest_driver(&lguest_virtnet_drv);
+}
+device_initcall(lguest_virtnet_init);
+
+/* Example block driver code. */
+#include <linux/virtio_blk.h>
+#include <linux/genhd.h>
+#include <linux/blkdev.h>
+static int lguest_virtblk_probe(struct lguest_device *lgdev)
+{
+	struct lguest_virtio_device *lgv;
+	struct gendisk *disk;
+	unsigned long sectors;
+	int err;
+
+	lgv = lg_new_virtio(lgdev);
+	if (!lgv)
+		return -ENOMEM;
+
+	/* Page is initially used to pass capacity. */
+	sectors = *(unsigned long *)lgv->in.desc;
+	*(unsigned long *)lgv->in.desc = 0;
+
+	lgv->priv = disk = virtblk_probe(&lgv->vdev);
+	if (IS_ERR(lgv->priv)) {
+		err = PTR_ERR(lgv->priv);
+		goto destroy;
+	}
+	set_capacity(disk, sectors);
+	blk_queue_max_hw_segments(disk->queue, NUM_DESCS-1);
+
+	err = lg_setup_interrupt(lgv, disk->disk_name);
+	if (err)
+		goto unprobe;
+	add_disk(disk);
+	return 0;
+
+unprobe:
+	virtblk_remove(disk);
+destroy:
+	lg_destroy_virtio(lgv);
+	return err;
+}
+
+static struct lguest_driver lguest_virtblk_drv = {
+	.name = "lguestvirtblk",

drivers/lguest/lguest_virtio.c line 509

Quoted 10 lines
drivers/lguest/lguest_virtio.c new file +10 -0
+	.owner = THIS_MODULE,
+	.device_type = LGUEST_DEVICE_T_VIRTBLK,
+	.probe = lguest_virtblk_probe,
+};
+
+static __init int lguest_virtblk_init(void)
+{
+	return register_lguest_driver(&lguest_virtblk_drv);
+}
+device_initcall(lguest_virtblk_init);

This is the part that I find puzzling. My idea would be that the drivers and devices you register are _not_ specific to the type of virtio bus.

The problem I see with your approach is that drivers for specific devices can't be loaded as separate modules, right now all of them have to be loaded or statically linked into the kernel in order to load the lguest_virtio module.

I'm sure you know how this works for other buses, but I'll explain anyway so that everyone knows what I'm talking about. What I have in mind is to have a single 'struct bus_type virtio_bus_type', which is used by all virtio implementations. The 'struct virtio_device' is derived from 'struct device', meaning that a 'struct device' is embedded in it. Each virtio implementation can either use the virtio_device directly, or derive its own kvm_virtio_device/lguest_virtio_device/... from it. When lguest scans its devices, it registers the lguest_virtio_device with the generic virtio_bus_type, not an lguest specific one. Similarly, the block/net/... drivers each register themselves as a virtio_driver with the virtio_bus_type, not with an implementation specific guest.

Do you see a problem with the approach that I described here, or did you just not bother implementing it so far?

	Arnd <><
#27 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

In reply to a message not in the archive
Quoted 14 lines · Rusty Russell
drivers/block/virtio_blk.c new file +13 -0
+static bool do_read(request_queue_t *q, struct virtio_blk *vblk,
+		    struct virtblk_req *vbr)
+{
+	unsigned long num;
+
+	vbr->out_hdr.type |= VIRTIO_BLK_T_READ;
+
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,

drivers/block/virtio_blk.c line 181

Quoted 10 lines
drivers/block/virtio_blk.c new file +10 -0
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 46754451.2010305-atKUWr5tajBWk0Htik3J/[email protected]

#29 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

Quoted 12 lines · Avi Kivity
Quoted 11 lines · Rusty Russell
drivers/block/virtio_blk.c new file +11 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
Quoted 8 lines

drivers/block/virtio_blk.c line 181

Quoted 5 lines
drivers/block/virtio_blk.c new file +5 -0
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

The strict separation of in and out in virtio is to support both untrusted inter-guest I/O (admittedly not useful for block devices) and socket-style hypercall interfaces.

Thanks,
Rusty.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1182154095.19064.24.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#31 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

Quoted 25 lines · Rusty Russell
Quoted 13 lines · Avi Kivity
Quoted 11 lines · Rusty Russell
drivers/block/virtio_blk.c new file +11 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
Quoted 9 lines

drivers/block/virtio_blk.c line 181

Quoted 6 lines
drivers/block/virtio_blk.c new file +5 -0
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Quoted 5 lines

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

So you map the first sg entry for output, and the rest for input.

Less pretty, but more efficient.

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 46764BB5.6070704-atKUWr5tajBWk0Htik3J/[email protected]

#33 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

Quoted 26 lines · Avi Kivity
Quoted 25 lines · Rusty Russell
Quoted 22 lines · Avi Kivity
Quoted 7 lines · Rusty Russell
(quoted diff) +6 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,

Review

Quoted 10 lines
(quoted diff) +10 -0
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Quoted 7 lines
Quoted 3 lines

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

So you map the first sg entry for output, and the rest for input.

Less pretty, but more efficient.

At the moment there are two kinds of sgs: input and output. You're proposing instead that we have a single kind, and a flag on each segment which indicates direction. That may make block a little efficient, depending on particular implementation of virtio.

Yet network really wants in and out separate. I'm not even sure how we'd implement a mixed-sg scheme efficiently for that case. At the moment it's very straightforward.

Of course, I'm carrying two implementations and two drivers, so it's natural for me to try to resist changes 8)

Cheers,
Rusty.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 1182234466.19064.51.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#35 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

Quoted 41 lines · Rusty Russell
Quoted 30 lines · Avi Kivity
Quoted 28 lines · Rusty Russell
Quoted 25 lines · Avi Kivity
Quoted 8 lines · Rusty Russell
(quoted diff) +6 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,

Review

Quoted 10 lines
(quoted diff) +10 -0
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Quoted 8 lines
Quoted 4 lines

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

So you map the first sg entry for output, and the rest for input.

Less pretty, but more efficient.

Quoted 10 lines

At the moment there are two kinds of sgs: input and output. You're proposing instead that we have a single kind, and a flag on each segment which indicates direction. That may make block a little efficient, depending on particular implementation of virtio.

Yet network really wants in and out separate. I'm not even sure how we'd implement a mixed-sg scheme efficiently for that case. At the moment it's very straightforward.

I think the differences are deeper than that. Network wants two queues, disk wants one queue. The directionality of the dma transfer is orthogonal to that.

For a disk, the initiator is always the guest; you issue a request and expend a completion within a few milliseconds. Read and write requests are similar in nature; you want them in a single queue because you want the elevator to run on both read and write request. When idling, the queue is empty.

For a network interface, the guest initiates transmits, but the host initiates receives. Yes, the guest preloads the queue with rx descriptors, but it can't expect a response in any given time, hence the need for two queues. When idling, the tx queue is empty, the rx queue is full.

So I'd suggest having a struct virtio_queue to represent a queue (so a network interface would have two of them), and the dma direction to be specified separately:

+ unsigned long (*add_buf)(struct virtio_queue *q,
+ const struct scatterlist out_sg[], int out_nr,
+ const struct scatterlist in_sg[], int in_nr,
+ void *data);

I think this fits better with the Xen model (which has one ring for block, two each for network and console, if memory serves).

Quoted 2 lines

Of course, I'm carrying two implementations and two drivers, so it's natural for me to try to resist changes 8

Life is rough on the frontier. But all the glory will be yours!

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
#36 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

In reply to Rusty Russell
Quoted 40 lines · Rusty Russell
Quoted 29 lines · Avi Kivity
Quoted 27 lines · Rusty Russell
Quoted 24 lines · Avi Kivity
Quoted 8 lines · Rusty Russell
(quoted diff) +6 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,

Review

Quoted 10 lines
(quoted diff) +10 -0
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Quoted 8 lines
Quoted 4 lines

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

So you map the first sg entry for output, and the rest for input.

Less pretty, but more efficient.

Quoted 10 lines

At the moment there are two kinds of sgs: input and output. You're proposing instead that we have a single kind, and a flag on each segment which indicates direction. That may make block a little efficient, depending on particular implementation of virtio.

Yet network really wants in and out separate. I'm not even sure how we'd implement a mixed-sg scheme efficiently for that case. At the moment it's very straightforward.

I think the differences are deeper than that. Network wants two queues, disk wants one queue. The directionality of the dma transfer is orthogonal to that.

For a disk, the initiator is always the guest; you issue a request and expend a completion within a few milliseconds. Read and write requests are similar in nature; you want them in a single queue because you want the elevator to run on both read and write request. When idling, the queue is empty.

For a network interface, the guest initiates transmits, but the host initiates receives. Yes, the guest preloads the queue with rx descriptors, but it can't expect a response in any given time, hence the need for two queues. When idling, the tx queue is empty, the rx queue is full.

So I'd suggest having a struct virtio_queue to represent a queue (so a network interface would have two of them), and the dma direction to be specified separately:

+ unsigned long (*add_buf)(struct virtio_queue *q,
+ const struct scatterlist out_sg[], int out_nr,
+ const struct scatterlist in_sg[], int in_nr,
+ void *data);

I think this fits better with the Xen model (which has one ring for block, two each for network and console, if memory serves).

Quoted 2 lines

Of course, I'm carrying two implementations and two drivers, so it's natural for me to try to resist changes 8

Life is rough on the frontier. But all the glory will be yours!

Signature · 1 line
error compiling committee.c: too many arguments to function
#37 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

In reply to Avi Kivity
Quoted 26 lines · Avi Kivity
Quoted 25 lines · Rusty Russell
Quoted 22 lines · Avi Kivity
Quoted 7 lines · Rusty Russell
(quoted diff) +6 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,

Review

Quoted 10 lines
(quoted diff) +10 -0
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Quoted 7 lines
Quoted 3 lines

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

So you map the first sg entry for output, and the rest for input.

Less pretty, but more efficient.

At the moment there are two kinds of sgs: input and output. You're proposing instead that we have a single kind, and a flag on each segment which indicates direction. That may make block a little efficient, depending on particular implementation of virtio.

Yet network really wants in and out separate. I'm not even sure how we'd implement a mixed-sg scheme efficiently for that case. At the moment it's very straightforward.

Of course, I'm carrying two implementations and two drivers, so it's natural for me to try to resist changes 8)

Cheers,
Rusty.

#38 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

In reply to Rusty Russell
Quoted 24 lines · Rusty Russell
Quoted 12 lines · Avi Kivity
Quoted 10 lines · Rusty Russell
drivers/block/virtio_blk.c new file +10 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
Quoted 9 lines

drivers/block/virtio_blk.c line 181

Quoted 7 lines
drivers/block/virtio_blk.c new file +6 -0
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Quoted 5 lines

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

So you map the first sg entry for output, and the rest for input.

Less pretty, but more efficient.

Signature · 1 line
error compiling committee.c: too many arguments to function
#39 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

In reply to Avi Kivity
Quoted 12 lines · Avi Kivity
Quoted 11 lines · Rusty Russell
drivers/block/virtio_blk.c new file +11 -0
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
Quoted 8 lines

drivers/block/virtio_blk.c line 181

Quoted 5 lines
drivers/block/virtio_blk.c new file +5 -0
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

We could actually do this for write, but not for read (where the length & sector need to be sent to other end, and the data comes from other end).

The strict separation of in and out in virtio is to support both untrusted inter-guest I/O (admittedly not useful for block devices) and socket-style hypercall interfaces.

Thanks,
Rusty.

#40 lore raw

PATCH 1/1 Lguest implemention of virtio draft III

In reply to Rusty Russell

This is a bonus patch for those wondering how a virtio implementation can look. I have two, this is the more efficient one (needs some modification for inter-guest though: it assumes the other end does all the accessing of our memory. It's currently tacked on to the existing lguest I/O mechanism as a demonstration, rather than replacing it.

It shows that it's possible to implement virtio without internal locking.

Userspace server-side code isn't included.
===
This allows zero-copy from guest <-> host. It uses a page of descriptors, a page to say what descriptors to use, and a page to say what's been used: one each set for inbufs and one for outbufs.

TODO:
1) More polishing
2) Get rid of old I/O
3) Inter-guest I/O implementation

---
 drivers/lguest/Makefile         |    2 
 drivers/lguest/hypercalls.c     |    4 
 drivers/lguest/lguest_virtio.c  |  511 +++++++++++++++++++++++++++++++++++++++
 include/linux/lguest.h          |    3 
 include/linux/lguest_launcher.h |   24 +
 6 files changed, 948 insertions(+), 5 deletions(-)
drivers/lguest/Makefile +1 -1
@@ -1,5 +1,5 @@
 # Guest requires the paravirt_ops replacement and the bus driver.
-obj-$(CONFIG_LGUEST_GUEST) += lguest.o lguest_asm.o lguest_bus.o
+obj-$(CONFIG_LGUEST_GUEST) += lguest.o lguest_asm.o lguest_bus.o lguest_virtio.o
 
 # Host requires the other files, which can be a module.
 obj-$(CONFIG_LGUEST)	+= lg.o

===================================================================

drivers/lguest/hypercalls.c +4 -0
@@ -86,6 +86,10 @@ static void do_hcall(struct lguest *lg, 
 		break;
 	case LHCALL_HALT:
 		lg->halted = 1;
+		break;
+	case LHCALL_NOTIFY:
+		lg->pending_key = regs->edx << PAGE_SHIFT;
+		lg->dma_is_pending = 1;
 		break;
 	default:
 		kill_guest(lg, "Bad hypercall %li\n", regs->eax);

===================================================================

drivers/lguest/lguest_virtio.c new file +511 -0

511 lines not on this page · raw message

===================================================================

include/linux/lguest.h +3 -0
@@ -23,6 +23,9 @@
 #define LHCALL_SET_PTE		14
 #define LHCALL_SET_PMD		15
 #define LHCALL_LOAD_TLS		16
+
+/* Experimental hcalls for new I/O */
+#define LHCALL_NOTIFY	100 /* pfn */
 
 #define LG_CLOCK_MIN_DELTA	100UL
 #define LG_CLOCK_MAX_DELTA	ULONG_MAX

===================================================================

include/linux/lguest_launcher.h +24 -0
@@ -44,6 +44,8 @@ struct lguest_device_desc {
 #define LGUEST_DEVICE_T_CONSOLE	1
 #define LGUEST_DEVICE_T_NET	2
 #define LGUEST_DEVICE_T_BLOCK	3
+#define LGUEST_DEVICE_T_VIRTNET	8
+#define LGUEST_DEVICE_T_VIRTBLK	9
 
 	u16 features;
 #define LGUEST_NET_F_NOCSUM		0x4000 /* Don't bother checksumming */
@@ -70,4 +72,26 @@ enum lguest_req
 	LHREQ_IRQ, /* + irq */
 	LHREQ_BREAK, /* + on/off flag (on blocks until someone does off) */
 };
+
+/* This marks a buffer as being the start (and active) */
+#define LGUEST_DESC_F_HEAD	1
+/* This marks a buffer as continuing via the next field. */
+#define LGUEST_DESC_F_NEXT	2
+
+/* Virtio descriptors */
+struct lguest_desc
+{
+	unsigned long pfn;
+	unsigned long len;
+	u16 offset;
+	u16 flags;
+	/* We chain unused descriptors via this, too */
+	u32 next;
+};
+
+struct lguest_used
+{
+	unsigned int id;
+	unsigned int len;
+};
 #endif /* _ASM_LGUEST_USER */
#41 lore raw

Re: [PATCH RFC 3/3] Virtio draft III: example block driver

In reply to Rusty Russell
Quoted 14 lines · Rusty Russell
drivers/block/virtio_blk.c new file +13 -0
+static bool do_read(request_queue_t *q, struct virtio_blk *vblk,
+		    struct virtblk_req *vbr)
+{
+	unsigned long num;
+
+	vbr->out_hdr.type |= VIRTIO_BLK_T_READ;
+
+	/* Set up for reply. */
+	vblk->sg[0].page = virt_to_page(&vbr->in_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->in_hdr);
+	vblk->sg[0].length = sizeof(vbr->in_hdr);
+	num = blk_rq_map_sg(q, vbr->req, vblk->sg+1);
+	vbr->out_hdr.id = vblk->vdev->ops->add_inbuf(vblk->vdev, vblk->sg,

drivers/block/virtio_blk.c line 181

Quoted 10 lines
drivers/block/virtio_blk.c new file +10 -0
+						     1+num, vbr);
+	if (IS_ERR_VALUE(vbr->out_hdr.id))
+		goto full;
+
+	vblk->sg[0].page = virt_to_page(&vbr->out_hdr);
+	vblk->sg[0].offset = offset_in_page(&vbr->out_hdr);
+	vblk->sg[0].length = sizeof(vbr->out_hdr);
+
+	vbr->out_id = vblk->vdev->ops->add_outbuf(vblk->vdev, vblk->sg, 1,
+						  vbr);

This strikes me as wasteful. Why not set up a single descriptor which contains both placement and the data itself?

Signature · 1 line
error compiling committee.c: too many arguments to function
#42 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to a message not in the archive

I've started to look at these patches to figure out how easily I could convert the ibmveth driver to use this new infrastructure and have ran into a few issues. So far the two biggest issues I've encountered are:

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.
2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

-Brian

Quoted 329 lines · Rusty Russell

Net driver using virtio

TODO:
1) GSO.
2) Checksum options.
3) Big packets.

---
 drivers/net/Makefile       |    2 
 drivers/net/virtio_net.c   |  277 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_net.h |   12 +
 3 files changed, 290 insertions(+), 1 deletion(-)

===================================================================

drivers/net/Makefile +1 -1
@@ -37,7 +37,7 @@ obj-$(CONFIG_CASSINI) += cassini.o
 
 obj-$(CONFIG_MACE) += mace.o
 obj-$(CONFIG_BMAC) += bmac.o
-
+obj-y += virtio_net.o
 obj-$(CONFIG_DGRS) += dgrs.o
 obj-$(CONFIG_VORTEX) += 3c59x.o
 obj-$(CONFIG_TYPHOON) += typhoon.o

===================================================================

drivers/net/virtio_net.c new file +277 -0
@@ -0,0 +1,277 @@
+/* A simple network driver using virtio.
+ *
+ * Copyright 2007 Rusty Russell <rusty-8n+1lVoiYb80n/[email protected]> IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+//#define DEBUG
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/module.h>
+#include <linux/virtio.h>
+#include <linux/scatterlist.h>
+
+/* FIXME: Make dynamic */
+#define MAX_PACKET_LEN (ETH_HLEN+ETH_DATA_LEN)
+
+#define SKB_ID(skb) (*(unsigned long *)(skb)->cb)
+
+struct virtnet_info
+{
+	struct virtio_device *vdev;
+	struct net_device *ndev;
+
+	/* Number of input buffers, and max we've ever had. */
+	unsigned int num, max;
+
+	/* Receive & send queues. */
+	struct sk_buff_head recv;
+	struct sk_buff_head send;
+};
+
+static bool skb_xmit_done(struct virtio_device *vdev)
+{
+	struct virtnet_info *vi = vdev->priv;
+
+	/* In case we were waiting for output buffers. */
+	netif_wake_queue(vi->ndev);
+	return true;
+}
+
+static void receive_skb(struct net_device *dev, struct sk_buff *skb,
+			unsigned len)
+{
+	if (unlikely(len < ETH_HLEN)) {
+		pr_debug("%s: short packet %i\n", dev->name, len);
+		dev->stats.rx_length_errors++;
+		dev_kfree_skb(skb);
+		return;
+	}
+	BUG_ON(len > MAX_PACKET_LEN);
+
+	skb_trim(skb, len);
+	skb->protocol = eth_type_trans(skb, dev);
+	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
+		 ntohs(skb->protocol), skb->len, skb->pkt_type);
+	dev->stats.rx_bytes += skb->len;
+	dev->stats.rx_packets++;
+	netif_rx(skb);
+}
+
+static void try_fill_recv(struct virtnet_info *vi)
+{
+	struct sk_buff *skb;
+	struct scatterlist sg[MAX_SKB_FRAGS];
+	unsigned long sgnum, id;
+
+	for (;;) {
+		skb = netdev_alloc_skb(vi->ndev, MAX_PACKET_LEN);
+		if (unlikely(!skb))
+			break;
+
+		skb_put(skb, MAX_PACKET_LEN);
+		sgnum = skb_to_sgvec(skb, sg, 0, skb->len);
+		skb_queue_head(&vi->recv, skb);
+
+		id = vi->vdev->ops->add_inbuf(vi->vdev, sg, sgnum, skb);
+		if (IS_ERR_VALUE(id)) {
+			skb_unlink(skb, &vi->recv);
+			kfree_skb(skb);
+			break;
+		}
+		vi->num++;
+	}
+	if (unlikely(vi->num > vi->max))
+		vi->max = vi->num;
+	vi->vdev->ops->sync(vi->vdev, VIRTIO_IN);
+}
+
+static bool skb_recv_done(struct virtio_device *vdev)
+{
+	struct virtnet_info *vi = vdev->priv;
+
+	netif_rx_schedule(vi->ndev);
+	/* Suppress further interrupts. */
+	return false;
+}
+
+static int virtnet_poll(struct net_device *dev, int *budget)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	struct sk_buff *skb = NULL;
+	unsigned int len, received = 0;
+
+again:
+	while (received < dev->quota &&
+	       (skb = vi->vdev->ops->get_inbuf(vi->vdev, &len)) != NULL) {
+		__skb_unlink(skb, &vi->recv);
+		receive_skb(vi->ndev, skb, len);
+		vi->num--;
+		received++;
+	}
+
+        dev->quota -= received;
+        *budget -= received;
+
+	/* FIXME: If we oom and completely run out of inbufs, we need
+	 * to start a timer trying to fill more. */
+	if (vi->num < vi->max / 2)
+		try_fill_recv(vi);
+
+	/* Still more work to do? */
+	if (skb)
+		return 1; /* not done */
+
+	netif_rx_complete(dev);
+	if (unlikely(!vi->vdev->ops->restart_in(vi->vdev))
+	    && netif_rx_reschedule(dev, received))
+		goto again;
+
+	return 0;
+}
+
+static void free_old_xmit_skbs(struct virtnet_info *vi)
+{
+	struct sk_buff *skb;
+	unsigned int len;
+
+	while ((skb = vi->vdev->ops->get_outbuf(vi->vdev, &len)) != NULL) {
+		pr_debug("Sent skb %p\n", skb);
+		__skb_unlink(skb, &vi->send);
+		vi->ndev->stats.tx_bytes += len;
+		vi->ndev->stats.tx_packets++;
+		kfree_skb(skb);
+	}
+}
+
+static int start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	unsigned long num, id;
+	struct scatterlist sg[MAX_SKB_FRAGS];
+	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
+
+	pr_debug("%s: xmit %p %02x:%02x:%02x:%02x:%02x:%02x\n",
+		 dev->name, skb,
+		 dest[0], dest[1], dest[2], dest[3], dest[4], dest[5]);
+
+	free_old_xmit_skbs(vi);
+
+	num = skb_to_sgvec(skb, sg, 0, skb->len);
+	__skb_queue_head(&vi->send, skb);
+	id = vi->vdev->ops->add_outbuf(vi->vdev, sg, num, skb);
+	if (IS_ERR_VALUE(id)) {
+		pr_debug("%s: virtio not prepared to send\n", dev->name);
+		skb_unlink(skb, &vi->send);
+		netif_stop_queue(dev);
+		return NETDEV_TX_BUSY;
+	}
+	SKB_ID(skb) = id;
+	vi->vdev->ops->sync(vi->vdev, VIRTIO_OUT);
+
+	return 0;
+}
+
+static int virtnet_open(struct net_device *dev)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+
+	try_fill_recv(vi);
+
+	/* If we didn't even get one input buffer, we're useless. */
+	if (vi->num == 0)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int virtnet_close(struct net_device *dev)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	struct sk_buff *skb;
+
+	/* networking core has neutered skb_xmit_done/skb_recv_done, so don't
+	 * worry about races vs. get_buf(). */
+	while ((skb = __skb_dequeue(&vi->recv)) != NULL) {
+		vi->vdev->ops->detach_inbuf(vi->vdev, SKB_ID(skb));
+		kfree_skb(skb);
+		vi->num--;
+	}
+	while ((skb = __skb_dequeue(&vi->send)) != NULL) {
+		vi->vdev->ops->detach_outbuf(vi->vdev, SKB_ID(skb));
+		kfree_skb(skb);
+	}
+	BUG_ON(vi->num != 0);
+	return 0;
+}
+
+static struct virtio_driver_ops virtnet_ops = {
+	.in = skb_recv_done,
+	.out = skb_xmit_done,
+};
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN])
+{
+	int err;
+	struct net_device *dev;
+	struct virtnet_info *vi;
+
+	dev = alloc_etherdev(sizeof(struct virtnet_info));
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	SET_MODULE_OWNER(dev);
+
+	ether_setup(dev);
+	memcpy(dev->dev_addr, mac, ETH_ALEN);
+	dev->open = virtnet_open;
+	dev->stop = virtnet_close;
+	dev->poll = virtnet_poll;
+	dev->hard_start_xmit = start_xmit;
+	dev->weight = 16;
+	SET_NETDEV_DEV(dev, vdev->dev);
+
+	vi = netdev_priv(dev);
+	vi->vdev = vdev;
+	vi->ndev = dev;
+	vdev->priv = vi;
+	vdev->driver_ops = &virtnet_ops;
+	skb_queue_head_init(&vi->recv);
+	skb_queue_head_init(&vi->send);
+
+	err = register_netdev(dev);
+	if (err) {
+		pr_debug("virtio_net: registering device failed\n");
+		goto free;
+	}
+	pr_debug("virtnet: registered device %s\n", dev->name);
+	return dev;
+
+free:
+	free_netdev(dev);
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL_GPL(virtnet_probe);
+
+void virtnet_remove(struct net_device *dev)
+{
+	unregister_netdev(dev);
+	free_netdev(dev);
+}
+EXPORT_SYMBOL_GPL(virtnet_remove);
+
+MODULE_DESCRIPTION("Virtio network driver");
+MODULE_LICENSE("GPL");

===================================================================

include/linux/virtio_net.h new file +12 -0
@@ -0,0 +1,12 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+struct net_device;
+struct virtio_device;
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN]);
+void virtnet_remove(struct net_device *dev);
+
+#endif /* _LINUX_VIRTIO_NET_H */

_______________________________________________
Virtualization mailing list Virtualization-cunTk1MwBs9QetFLy7KEm3xJsTq8ys+cHZ5vskTnxNA@public.gmane.org
https://lists.linux-foundation.org/mailman/listinfo/virtualization

Signature · 10 lines
Brian King
eServer Storage I/O
IBM Linux Technology Center


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 467FDEAD.4030204-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org

#44 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

Quoted 8 lines · Brian King

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.
2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff. Then again the network layer really should be using struct scaterrlist for skb fragments - on a lot of iommu a dma_map_sg is a lot more efficient than a lot of dma_map_page calls, not to mention the benefit of a common data structure for things like network attached storage protocols that have to talk to both worlds.

And yes, this is getting a little out of scope for the virtualization discussion.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): [email protected]

#46 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

Quoted 5 lines · Christoph Hellwig
Quoted 3 lines · Brian King

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.

I'm probably missing something. What is wrong with a scatterlist of length 1?

Quoted 9 lines
Quoted 7 lines

2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff.

But virtio is a generic layer that is useful for more than just networking.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 46803999.4040500-atKUWr5tajBWk0Htik3J/[email protected]

#48 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

Quoted 9 lines · Avi Kivity
Quoted 5 lines · Christoph Hellwig
Quoted 3 lines · Brian King

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.

I'm probably missing something. What is wrong with a scatterlist of length 1?

Absolutely nothing, as long as the interface guarantees I don't see a larger scatterlist. With the current implementation it looks to me like this is always true, but if we were to stick to using a scatterlist, it would make sense to allow the user to specify a max sg length supported. This value could be different for in vs. out buffers.

Quoted 10 lines
Quoted 8 lines
Quoted 7 lines

2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff.

But virtio is a generic layer that is useful for more than just networking.

Agreed. However, this one is a huge issue for ibmveth. ibmveth must write a tag to the first 8 bytes of each rx buffer before passing the buffer to the pSeries hypervisor. The hypervisor then writes this tag to the rx queue when the buffer is used. This first 8 bytes of the rx buffer is not overlayed with data. Additionally, the hypervisor may offset further than 8 bytes into the buffer for the start of the payload if it so desires. The offset is also written to the rx queue.

Another option which would solve this problem would be if the get_inbuf interface also returned an offset into the original data buffer where the payload starts.

-Brian

Signature · 9 lines
Brian King
Linux on Power Virtualization
IBM Linux Technology Center

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 46803E0E.7080103-23VcF4HTsmIX0ybBhKVfKdBPR1lH4CV8@public.gmane.org

#50 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

Quoted 11 lines · Brian King
Quoted 3 lines

I'm probably missing something. What is wrong with a scatterlist of length 1?

Absolutely nothing, as long as the interface guarantees I don't see a larger scatterlist. With the current implementation it looks to me like this is always true, but if we were to stick to using a scatterlist, it would make sense to allow the user to specify a max sg length supported. This value could be different for in vs. out buffers.

Okay; so virtio net should have configurable scatterlist max sizes.

Quoted 20 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff.

But virtio is a generic layer that is useful for more than just networking.

Agreed. However, this one is a huge issue for ibmveth. ibmveth must write a tag to the first 8 bytes of each rx buffer before passing the buffer to the pSeries hypervisor. The hypervisor then writes this tag to the rx queue when the buffer is used. This first 8 bytes of the rx buffer is not overlayed with data. Additionally, the hypervisor may offset further than 8 bytes into the buffer for the start of the payload if it so desires. The offset is also written to the rx queue.

Quoted 5 lines

Another option which would solve this problem would be if the get_inbuf interface also returned an offset into the original data buffer where the payload starts.

I guess. Rusty was concerned that virtio will explode to the sum of all options on all hypervisors, and we now see it begin.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

#51 lore raw

Re: [kvm-devel] [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Brian King
Quoted 10 lines · Brian King
Quoted 2 lines

I'm probably missing something. What is wrong with a scatterlist of length 1?

Absolutely nothing, as long as the interface guarantees I don't see a larger scatterlist. With the current implementation it looks to me like this is always true, but if we were to stick to using a scatterlist, it would make sense to allow the user to specify a max sg length supported. This value could be different for in vs. out buffers.

Okay; so virtio net should have configurable scatterlist max sizes.

Quoted 20 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines

2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Quoted 8 lines
Quoted 6 lines
Quoted 4 lines

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff.

But virtio is a generic layer that is useful for more than just networking.

Agreed. However, this one is a huge issue for ibmveth. ibmveth must write a tag to the first 8 bytes of each rx buffer before passing the buffer to the pSeries hypervisor. The hypervisor then writes this tag to the rx queue when the buffer is used. This first 8 bytes of the rx buffer is not overlayed with data. Additionally, the hypervisor may offset further than 8 bytes into the buffer for the start of the payload if it so desires. The offset is also written to the rx queue.

Quoted 5 lines

Another option which would solve this problem would be if the get_inbuf interface also returned an offset into the original data buffer where the payload starts.

I guess. Rusty was concerned that virtio will explode to the sum of all options on all hypervisors, and we now see it begin.

#52 lore raw

Re: [kvm-devel] [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Avi Kivity
Quoted 9 lines · Avi Kivity
Quoted 5 lines · Christoph Hellwig
Quoted 3 lines · Brian King

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.

I'm probably missing something. What is wrong with a scatterlist of length 1?

Absolutely nothing, as long as the interface guarantees I don't see a larger scatterlist. With the current implementation it looks to me like this is always true, but if we were to stick to using a scatterlist, it would make sense to allow the user to specify a max sg length supported. This value could be different for in vs. out buffers.

Quoted 10 lines
Quoted 8 lines
Quoted 7 lines

2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff.

But virtio is a generic layer that is useful for more than just networking.

Agreed. However, this one is a huge issue for ibmveth. ibmveth must write a tag to the first 8 bytes of each rx buffer before passing the buffer to the pSeries hypervisor. The hypervisor then writes this tag to the rx queue when the buffer is used. This first 8 bytes of the rx buffer is not overlayed with data. Additionally, the hypervisor may offset further than 8 bytes into the buffer for the start of the payload if it so desires. The offset is also written to the rx queue.

Another option which would solve this problem would be if the get_inbuf interface also returned an offset into the original data buffer where the payload starts.

-Brian

Signature · 3 lines
Brian King
Linux on Power Virtualization
IBM Linux Technology Center
#53 lore raw

Re: [kvm-devel] [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Christoph Hellwig
Quoted 5 lines · Christoph Hellwig
Quoted 3 lines · Brian King

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.

I'm probably missing something. What is wrong with a scatterlist of length 1?

Quoted 9 lines
Quoted 7 lines

2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff.

But virtio is a generic layer that is useful for more than just networking.

#54 lore raw

Re: [kvm-devel] [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Brian King
Quoted 8 lines · Brian King

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.
2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It probably should pass the sk_buff. Then again the network layer really should be using struct scaterrlist for skb fragments - on a lot of iommu a dma_map_sg is a lot more efficient than a lot of dma_map_page calls, not to mention the benefit of a common data structure for things like network attached storage protocols that have to talk to both worlds.

And yes, this is getting a little out of scope for the virtualization discussion.

#55 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Brian King
Quoted 4 lines · Brian King

I've started to look at these patches to figure out how easily I could convert the ibmveth driver to use this new infrastructure and have ran into a few issues. So far the two biggest issues I've encountered are:

Hi Brian,

Thanks for looking at this! Sorry for the slow reply, I've been
travelling.

Quoted 8 lines

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.
2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It would be easier, but not suitable for block and other devices which use the same interface. Fortunately, I think there are solutions to your issues.

1. 1500 byte packets will always be 1 sg long (ie. you should BUG_ON() if that ever happens). The issue is MTU > PAGE_SIZE, where I was planning to allocate discontig pages. I can't quite see what your maximum supported MTU is, though?

2. The header problem is difficult (I can't resist pointing out that if you had sg receive capability, it would be trivial 8). The two possibilities are to have get_buf take a "unsigned long *off" as well, or to have ibm_veth do a memmove.

memmove sounds horrible at first glance, but since the hypervisor has just copied the packet I'm not sure we'll notice in practice. Benchmarking should tell...

BTW, after Avi's comments I have a new virtio draft, but still debugging my implementation. It doesn't effect this discussion, but would involve churn for an actual implementation if you've gotten that far...

Thanks,
Rusty.

A message that is not in the archive (sent off-list or not yet received): 1183029641.12401.36.camel-bi+AKbBUZKY6gyzm1THtWbp2dZbC/[email protected]

#57 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

Quoted 4 lines · Rusty Russell

1. 1500 byte packets will always be 1 sg long (ie. you should BUG_ON() if that ever happens). The issue is MTU > PAGE_SIZE, where I was planning to allocate discontig pages. I can't quite see what your maximum supported MTU is, though?

The current maximum supported MTU for ibmveth is 64k. Additionally, the firmware interface for ibmveth allows for multiple rx "pools" of different sizes to be allocated and given to the hypervisor. Depending on the incoming packet size, the smallest buffer is then chosen. Just something to keep in mind as you look at adding large frame support. I'm not sure what requirements other users would have...

Quoted 8 lines

2. The header problem is difficult (I can't resist pointing out that if you had sg receive capability, it would be trivial 8). The two possibilities are to have get_buf take a "unsigned long *off" as well, or to have ibm_veth do a memmove.

memmove sounds horrible at first glance, but since the hypervisor has just copied the packet I'm not sure we'll notice in practice. Benchmarking should tell...

memmove definitely does not sound optimal. I would definitely prefer returning an offset on get_buf, but when I get something up and running I could certainly run some benchmarks.

Quoted 3 lines

BTW, after Avi's comments I have a new virtio draft, but still debugging my implementation. It doesn't effect this discussion, but would involve churn for an actual implementation if you've gotten that far...

I've started working on some code, but only to get a better handle on the API and see what issues ibmveth might run into. I was expecting some code churn yet at this stage..

-Brian

Signature · 9 lines
Brian King
Linux on Power Virtualization
IBM Linux Technology Center

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/
#58 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Rusty Russell
Quoted 4 lines · Rusty Russell

1. 1500 byte packets will always be 1 sg long (ie. you should BUG_ON() if that ever happens). The issue is MTU > PAGE_SIZE, where I was planning to allocate discontig pages. I can't quite see what your maximum supported MTU is, though?

The current maximum supported MTU for ibmveth is 64k. Additionally, the firmware interface for ibmveth allows for multiple rx "pools" of different sizes to be allocated and given to the hypervisor. Depending on the incoming packet size, the smallest buffer is then chosen. Just something to keep in mind as you look at adding large frame support. I'm not sure what requirements other users would have...

Quoted 8 lines

2. The header problem is difficult (I can't resist pointing out that if you had sg receive capability, it would be trivial 8). The two possibilities are to have get_buf take a "unsigned long *off" as well, or to have ibm_veth do a memmove.

memmove sounds horrible at first glance, but since the hypervisor has just copied the packet I'm not sure we'll notice in practice. Benchmarking should tell...

memmove definitely does not sound optimal. I would definitely prefer returning an offset on get_buf, but when I get something up and running I could certainly run some benchmarks.

Quoted 3 lines

BTW, after Avi's comments I have a new virtio draft, but still debugging my implementation. It doesn't effect this discussion, but would involve churn for an actual implementation if you've gotten that far...

I've started working on some code, but only to get a better handle on the API and see what issues ibmveth might run into. I was expecting some code churn yet at this stage..

-Brian

Signature · 3 lines
Brian King
Linux on Power Virtualization
IBM Linux Technology Center
#59 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Brian King
Quoted 4 lines · Brian King

I've started to look at these patches to figure out how easily I could convert the ibmveth driver to use this new infrastructure and have ran into a few issues. So far the two biggest issues I've encountered are:

Hi Brian,

Thanks for looking at this! Sorry for the slow reply, I've been
travelling.

Quoted 8 lines

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.
2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

It would be easier, but not suitable for block and other devices which use the same interface. Fortunately, I think there are solutions to your issues.

1. 1500 byte packets will always be 1 sg long (ie. you should BUG_ON() if that ever happens). The issue is MTU > PAGE_SIZE, where I was planning to allocate discontig pages. I can't quite see what your maximum supported MTU is, though?

2. The header problem is difficult (I can't resist pointing out that if you had sg receive capability, it would be trivial 8). The two possibilities are to have get_buf take a "unsigned long *off" as well, or to have ibm_veth do a memmove.

memmove sounds horrible at first glance, but since the hypervisor has just copied the packet I'm not sure we'll notice in practice. Benchmarking should tell...

BTW, after Avi's comments I have a new virtio draft, but still debugging my implementation. It doesn't effect this discussion, but would involve churn for an actual implementation if you've gotten that far...

Thanks,
Rusty.

#60 lore raw

RFC PATCH 3/3 Virtio draft III: example block driver

In reply to Rusty Russell

The block driver uses outbufs with sg[0] being the request information (struct virtio_blk_outhdr) with the type, sector and inbuf id. For a write, the rest of the sg will contain the data to be written.

The first segment of the inbuf is a result code (struct virtio_blk_inhdr). For a read, the rest of the sg points to the input buffer.

We accept the normal SCSI ioctls: they get handed through to the other side which can then handle it or reply that it's unsupported.

Although we try to reply -ENOTTY on unsupported commands, the block layer in its infinite wisdom suppressed the error so ioctl(fd, CDROMEJECT) returns success to userspace.

---
 drivers/block/Makefile     |    1 
 drivers/block/virtio_blk.c |  385 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/Kbuild       |    1 
 include/linux/virtio_blk.h |   39 ++++
 4 files changed, 426 insertions(+)

===================================================================

drivers/block/Makefile +1 -0
@@ -20,6 +20,7 @@ obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.
 obj-$(CONFIG_BLK_CPQ_CISS_DA)  += cciss.o
 obj-$(CONFIG_BLK_DEV_DAC960)	+= DAC960.o
 obj-$(CONFIG_CDROM_PKTCDVD)	+= pktcdvd.o
+obj-y				+= virtio_blk.o
 
 obj-$(CONFIG_BLK_DEV_UMEM)	+= umem.o
 obj-$(CONFIG_BLK_DEV_NBD)	+= nbd.o

===================================================================

drivers/block/virtio_blk.c new file +383 -0

383 lines not on this page · raw message

===================================================================

include/linux/Kbuild +1 -0
@@ -341,6 +341,7 @@ unifdef-y += utsname.h
 unifdef-y += utsname.h
 unifdef-y += videodev2.h
 unifdef-y += videodev.h
+unifdef-y += virtio_blk.h
 unifdef-y += wait.h
 unifdef-y += wanrouter.h
 unifdef-y += watchdog.h

===================================================================

include/linux/virtio_blk.h new file +39 -0
@@ -0,0 +1,39 @@
+#ifndef _LINUX_VIRTIO_BLK_H
+#define _LINUX_VIRTIO_BLK_H
+#include <linux/types.h>
+
+#define VIRTIO_BLK_T_READ	0
+#define VIRTIO_BLK_T_WRITE 1
+#define VIRTIO_BLK_T_SCSI_CMD 3
+#define VIRTIO_BLK_T_BARRIER 0x80000000 /* Barrier before this op. */
+
+/* This is the first element of the scatter-gather list. */
+struct virtio_blk_outhdr
+{
+	/* VIRTIO_BLK_T* */
+	__u32 type;
+	/* io priority. */
+	__u32 ioprio;
+	/* Sector (ie. 512 byte offset) */
+	__u64 sector;
+	/* Where to put reply. */
+	__u64 id;
+};
+
+#define VIRTIO_BLK_S_OK		0
+#define VIRTIO_BLK_S_IOERR	1
+#define VIRTIO_BLK_S_UNSUPP	2
+
+struct virtio_blk_inhdr
+{
+	unsigned char status;
+};
+
+#ifdef __KERNEL__
+struct gendisk;
+struct virtio_device;
+
+struct gendisk *virtblk_probe(struct virtio_device *vdev);
+void virtblk_remove(struct gendisk *disk);
+#endif /* __KERNEL__ */
+#endif /* _LINUX_VIRTIO_BLK_H */
#61 lore raw

Re: [PATCH RFC 2/3] Virtio draft III: example net driver

In reply to Rusty Russell

I've started to look at these patches to figure out how easily I could convert the ibmveth driver to use this new infrastructure and have ran into a few issues. So far the two biggest issues I've encountered are:

1. The add_inbuf interface passes an sg list. This causes problems for ibmveth since its rx buffers cannot be scatterlists.
2. The user of this API does not have access to the sk_buf. This causes issues for ibmveth since it needs to reserve the first 8 bytes of the rx buffer for use by the firmware. It currently uses skb_reserve to do this.

Would it be simpler to just pass an sk_buf rather than the scatterlist on these interfaces for virtio_net users?

-Brian

Quoted 329 lines · Rusty Russell

Net driver using virtio

TODO:
1) GSO.
2) Checksum options.
3) Big packets.

---
 drivers/net/Makefile       |    2 
 drivers/net/virtio_net.c   |  277 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_net.h |   12 +
 3 files changed, 290 insertions(+), 1 deletion(-)

===================================================================

drivers/net/Makefile +1 -1
@@ -37,7 +37,7 @@ obj-$(CONFIG_CASSINI) += cassini.o
 
 obj-$(CONFIG_MACE) += mace.o
 obj-$(CONFIG_BMAC) += bmac.o
-
+obj-y += virtio_net.o
 obj-$(CONFIG_DGRS) += dgrs.o
 obj-$(CONFIG_VORTEX) += 3c59x.o
 obj-$(CONFIG_TYPHOON) += typhoon.o

===================================================================

drivers/net/virtio_net.c new file +277 -0
@@ -0,0 +1,277 @@
+/* A simple network driver using virtio.
+ *
+ * Copyright 2007 Rusty Russell <[email protected]> IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+//#define DEBUG
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/module.h>
+#include <linux/virtio.h>
+#include <linux/scatterlist.h>
+
+/* FIXME: Make dynamic */
+#define MAX_PACKET_LEN (ETH_HLEN+ETH_DATA_LEN)
+
+#define SKB_ID(skb) (*(unsigned long *)(skb)->cb)
+
+struct virtnet_info
+{
+	struct virtio_device *vdev;
+	struct net_device *ndev;
+
+	/* Number of input buffers, and max we've ever had. */
+	unsigned int num, max;
+
+	/* Receive & send queues. */
+	struct sk_buff_head recv;
+	struct sk_buff_head send;
+};
+
+static bool skb_xmit_done(struct virtio_device *vdev)
+{
+	struct virtnet_info *vi = vdev->priv;
+
+	/* In case we were waiting for output buffers. */
+	netif_wake_queue(vi->ndev);
+	return true;
+}
+
+static void receive_skb(struct net_device *dev, struct sk_buff *skb,
+			unsigned len)
+{
+	if (unlikely(len < ETH_HLEN)) {
+		pr_debug("%s: short packet %i\n", dev->name, len);
+		dev->stats.rx_length_errors++;
+		dev_kfree_skb(skb);
+		return;
+	}
+	BUG_ON(len > MAX_PACKET_LEN);
+
+	skb_trim(skb, len);
+	skb->protocol = eth_type_trans(skb, dev);
+	pr_debug("Receiving skb proto 0x%04x len %i type %i\n",
+		 ntohs(skb->protocol), skb->len, skb->pkt_type);
+	dev->stats.rx_bytes += skb->len;
+	dev->stats.rx_packets++;
+	netif_rx(skb);
+}
+
+static void try_fill_recv(struct virtnet_info *vi)
+{
+	struct sk_buff *skb;
+	struct scatterlist sg[MAX_SKB_FRAGS];
+	unsigned long sgnum, id;
+
+	for (;;) {
+		skb = netdev_alloc_skb(vi->ndev, MAX_PACKET_LEN);
+		if (unlikely(!skb))
+			break;
+
+		skb_put(skb, MAX_PACKET_LEN);
+		sgnum = skb_to_sgvec(skb, sg, 0, skb->len);
+		skb_queue_head(&vi->recv, skb);
+
+		id = vi->vdev->ops->add_inbuf(vi->vdev, sg, sgnum, skb);
+		if (IS_ERR_VALUE(id)) {
+			skb_unlink(skb, &vi->recv);
+			kfree_skb(skb);
+			break;
+		}
+		vi->num++;
+	}
+	if (unlikely(vi->num > vi->max))
+		vi->max = vi->num;
+	vi->vdev->ops->sync(vi->vdev, VIRTIO_IN);
+}
+
+static bool skb_recv_done(struct virtio_device *vdev)
+{
+	struct virtnet_info *vi = vdev->priv;
+
+	netif_rx_schedule(vi->ndev);
+	/* Suppress further interrupts. */
+	return false;
+}
+
+static int virtnet_poll(struct net_device *dev, int *budget)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	struct sk_buff *skb = NULL;
+	unsigned int len, received = 0;
+
+again:
+	while (received < dev->quota &&
+	       (skb = vi->vdev->ops->get_inbuf(vi->vdev, &len)) != NULL) {
+		__skb_unlink(skb, &vi->recv);
+		receive_skb(vi->ndev, skb, len);
+		vi->num--;
+		received++;
+	}
+
+        dev->quota -= received;
+        *budget -= received;
+
+	/* FIXME: If we oom and completely run out of inbufs, we need
+	 * to start a timer trying to fill more. */
+	if (vi->num < vi->max / 2)
+		try_fill_recv(vi);
+
+	/* Still more work to do? */
+	if (skb)
+		return 1; /* not done */
+
+	netif_rx_complete(dev);
+	if (unlikely(!vi->vdev->ops->restart_in(vi->vdev))
+	    && netif_rx_reschedule(dev, received))
+		goto again;
+
+	return 0;
+}
+
+static void free_old_xmit_skbs(struct virtnet_info *vi)
+{
+	struct sk_buff *skb;
+	unsigned int len;
+
+	while ((skb = vi->vdev->ops->get_outbuf(vi->vdev, &len)) != NULL) {
+		pr_debug("Sent skb %p\n", skb);
+		__skb_unlink(skb, &vi->send);
+		vi->ndev->stats.tx_bytes += len;
+		vi->ndev->stats.tx_packets++;
+		kfree_skb(skb);
+	}
+}
+
+static int start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	unsigned long num, id;
+	struct scatterlist sg[MAX_SKB_FRAGS];
+	const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
+
+	pr_debug("%s: xmit %p %02x:%02x:%02x:%02x:%02x:%02x\n",
+		 dev->name, skb,
+		 dest[0], dest[1], dest[2], dest[3], dest[4], dest[5]);
+
+	free_old_xmit_skbs(vi);
+
+	num = skb_to_sgvec(skb, sg, 0, skb->len);
+	__skb_queue_head(&vi->send, skb);
+	id = vi->vdev->ops->add_outbuf(vi->vdev, sg, num, skb);
+	if (IS_ERR_VALUE(id)) {
+		pr_debug("%s: virtio not prepared to send\n", dev->name);
+		skb_unlink(skb, &vi->send);
+		netif_stop_queue(dev);
+		return NETDEV_TX_BUSY;
+	}
+	SKB_ID(skb) = id;
+	vi->vdev->ops->sync(vi->vdev, VIRTIO_OUT);
+
+	return 0;
+}
+
+static int virtnet_open(struct net_device *dev)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+
+	try_fill_recv(vi);
+
+	/* If we didn't even get one input buffer, we're useless. */
+	if (vi->num == 0)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static int virtnet_close(struct net_device *dev)
+{
+	struct virtnet_info *vi = netdev_priv(dev);
+	struct sk_buff *skb;
+
+	/* networking core has neutered skb_xmit_done/skb_recv_done, so don't
+	 * worry about races vs. get_buf(). */
+	while ((skb = __skb_dequeue(&vi->recv)) != NULL) {
+		vi->vdev->ops->detach_inbuf(vi->vdev, SKB_ID(skb));
+		kfree_skb(skb);
+		vi->num--;
+	}
+	while ((skb = __skb_dequeue(&vi->send)) != NULL) {
+		vi->vdev->ops->detach_outbuf(vi->vdev, SKB_ID(skb));
+		kfree_skb(skb);
+	}
+	BUG_ON(vi->num != 0);
+	return 0;
+}
+
+static struct virtio_driver_ops virtnet_ops = {
+	.in = skb_recv_done,
+	.out = skb_xmit_done,
+};
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN])
+{
+	int err;
+	struct net_device *dev;
+	struct virtnet_info *vi;
+
+	dev = alloc_etherdev(sizeof(struct virtnet_info));
+	if (!dev)
+		return ERR_PTR(-ENOMEM);
+
+	SET_MODULE_OWNER(dev);
+
+	ether_setup(dev);
+	memcpy(dev->dev_addr, mac, ETH_ALEN);
+	dev->open = virtnet_open;
+	dev->stop = virtnet_close;
+	dev->poll = virtnet_poll;
+	dev->hard_start_xmit = start_xmit;
+	dev->weight = 16;
+	SET_NETDEV_DEV(dev, vdev->dev);
+
+	vi = netdev_priv(dev);
+	vi->vdev = vdev;
+	vi->ndev = dev;
+	vdev->priv = vi;
+	vdev->driver_ops = &virtnet_ops;
+	skb_queue_head_init(&vi->recv);
+	skb_queue_head_init(&vi->send);
+
+	err = register_netdev(dev);
+	if (err) {
+		pr_debug("virtio_net: registering device failed\n");
+		goto free;
+	}
+	pr_debug("virtnet: registered device %s\n", dev->name);
+	return dev;
+
+free:
+	free_netdev(dev);
+	return ERR_PTR(err);
+}
+EXPORT_SYMBOL_GPL(virtnet_probe);
+
+void virtnet_remove(struct net_device *dev)
+{
+	unregister_netdev(dev);
+	free_netdev(dev);
+}
+EXPORT_SYMBOL_GPL(virtnet_remove);
+
+MODULE_DESCRIPTION("Virtio network driver");
+MODULE_LICENSE("GPL");

===================================================================

include/linux/virtio_net.h new file +12 -0
@@ -0,0 +1,12 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+struct net_device;
+struct virtio_device;
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN]);
+void virtnet_remove(struct net_device *dev);
+
+#endif /* _LINUX_VIRTIO_NET_H */

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linux-foundation.org/mailman/listinfo/virtualization

Signature · 3 lines
Brian King
eServer Storage I/O
IBM Linux Technology Center
#62 lore raw

RFC PATCH 2/3 Virtio draft III: example net driver

In reply to Rusty Russell

Net driver using virtio

TODO:
1) GSO.
2) Checksum options.
3) Big packets.

---
 drivers/net/Makefile       |    2 
 drivers/net/virtio_net.c   |  277 ++++++++++++++++++++++++++++++++++++++++++++
 include/linux/virtio_net.h |   12 +
 3 files changed, 290 insertions(+), 1 deletion(-)

===================================================================

drivers/net/Makefile +1 -1
@@ -37,7 +37,7 @@ obj-$(CONFIG_CASSINI) += cassini.o
 
 obj-$(CONFIG_MACE) += mace.o
 obj-$(CONFIG_BMAC) += bmac.o
-
+obj-y += virtio_net.o
 obj-$(CONFIG_DGRS) += dgrs.o
 obj-$(CONFIG_VORTEX) += 3c59x.o
 obj-$(CONFIG_TYPHOON) += typhoon.o

===================================================================

drivers/net/virtio_net.c new file +277 -0

277 lines not on this page · raw message

===================================================================

include/linux/virtio_net.h new file +12 -0
@@ -0,0 +1,12 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+#include <linux/types.h>
+#include <linux/etherdevice.h>
+struct net_device;
+struct virtio_device;
+
+struct net_device *virtnet_probe(struct virtio_device *vdev,
+				 const u8 mac[ETH_ALEN]);
+void virtnet_remove(struct net_device *dev);
+
+#endif /* _LINUX_VIRTIO_NET_H */
#63 lore raw

Re: [PATCH RFC 0/3] Virtio draft III

In reply to a message not in the archive
Quoted 7 lines · Rusty Russell

In this episode, Rusty tries to NAPI-ize the driver and discovers that virtio callbacks are a bad idea: NAPI needs to turn interrupts off and still be able to query for new incoming packets.

Changes to core:
1) Back to "interrupt" model with get_inbuf()/get_outbuf() calls.

Seems to be these are just two different ways of iterating over the pending buffers, and one could be implemented in terms of the other in a few lines. I think that new new way is better, though, due to less indirection.

btw, I'm not sure that the name 'virtio' is correct. Probably any network or block driver which has support for batching (the vast majority) could use this to good effect.

Signature · 8 lines
error compiling committee.c: too many arguments to function


-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take
control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

A message that is not in the archive (sent off-list or not yet received): 467541DF.5060907-atKUWr5tajBWk0Htik3J/[email protected]

#65 lore raw

Re: [Xen-devel] Re: [PATCH RFC 0/3] Virtio draft III

Quoted 5 lines · Avi Kivity
Quoted 4 lines · Rusty Russell

In this episode, Rusty tries to NAPI-ize the driver and discovers that virtio callbacks are a bad idea: NAPI needs to turn interrupts off and still be able to query for new incoming packets.

Quoted 8 lines
Quoted 3 lines

Changes to core:
1) Back to "interrupt" model with get_inbuf()/get_outbuf() calls.

Seems to be these are just two different ways of iterating over the pending buffers, and one could be implemented in terms of the other in a few lines. I think that new new way is better, though, due to less indirection.

Yes, the lguest implementation points them to a common routine. We could do some batching here too, but it seemed like premature optimization.

Quoted 3 lines

btw, I'm not sure that the name 'virtio' is correct. Probably any network or block driver which has support for batching (the vast majority) could use this to good effect.

Trying to avoid boiling the ocean 8)

Cheers,
Rusty.

-------------------------------------------------------------------------
This SF.net email is sponsored by DB2 Express
Download DB2 Express C - the FREE version of DB2 express and take control of your XML. No limits. Just data. Click to get it now.
http://sourceforge.net/powerbar/db2/

#66 lore raw

Re: [Xen-devel] Re: [kvm-devel] [PATCH RFC 0/3] Virtio draft III

In reply to Avi Kivity
Quoted 5 lines · Avi Kivity
Quoted 4 lines · Rusty Russell

In this episode, Rusty tries to NAPI-ize the driver and discovers that virtio callbacks are a bad idea: NAPI needs to turn interrupts off and still be able to query for new incoming packets.

Quoted 8 lines
Quoted 3 lines

Changes to core:
1) Back to "interrupt" model with get_inbuf()/get_outbuf() calls.

Seems to be these are just two different ways of iterating over the pending buffers, and one could be implemented in terms of the other in a few lines. I think that new new way is better, though, due to less indirection.

Yes, the lguest implementation points them to a common routine. We could do some batching here too, but it seemed like premature optimization.

Quoted 3 lines

btw, I'm not sure that the name 'virtio' is correct. Probably any network or block driver which has support for batching (the vast majority) could use this to good effect.

Trying to avoid boiling the ocean 8)

Cheers,
Rusty.

#67 lore raw

RFC PATCH 1/3 Virtio draft III: virtio.h

In reply to Rusty Russell

Draft III of virtio interface

This attempts to implement a "virtual I/O" layer which should allow common drivers to be efficiently used across most virtual I/O mechanisms. It will no-doubt need further enhancement.

The details of probing the device are left to hypervisor-specific
code: it simple constructs the "struct virtio_device" and hands it to the probe function (eg. virtnet_probe() or virtblk_probe()).

The virtio drivers add and get input and output buffers; as the buffers are used up the driver "interrupt" callbacks are invoked.

I have written two virtio device drivers (net and block) and two virtio implementations (for lguest): a read-write socket-style implementation, and a more efficient descriptor-based implementation.

---
 include/linux/virtio.h |  115 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 115 insertions(+)

===================================================================

include/linux/virtio.h new file +115 -0
@@ -0,0 +1,115 @@
+#ifndef _LINUX_VIRTIO_H
+#define _LINUX_VIRTIO_H
+#include <linux/types.h>
+#include <linux/scatterlist.h>
+#include <linux/spinlock.h>
+
+/**
+ * virtio_device - description and routines to drive a virtual device.
+ * @dev: the underlying struct device.
+ * @ops: the operations for this virtual device.
+ * @driver_ops: set by the driver for callbacks.
+ * @priv: private pointer for the driver to use.
+ */
+struct virtio_device {
+	struct device *dev;
+	struct virtio_ops *ops;
+	struct virtio_driver_ops *driver_ops;
+	void *priv;
+};
+
+/**
+ * virtio_driver_ops - driver callbacks for a virtual device.
+ * @in: inbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further inbuf callbacks.
+ * @out: outbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further outbuf callbacks.
+ */
+struct virtio_driver_ops {
+	bool (*in)(struct virtio_device *dev);
+	bool (*out)(struct virtio_device *dev);
+};
+
+enum virtio_dir {
+	VIRTIO_IN = 0x1,
+	VIRTIO_OUT = 0x2,
+};
+
+/**
+ * virtio_ops - virtio abstraction layer
+ * @add_outbuf: prepare to send data to the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	data: the token returned by the get_outbuf function.
+ *      Returns a unique id or an error.
+ * @add_inbuf: prepare to receive data from the other end:
+ *	vdev: the virtio_device
+ *	sg: the description of the buffer(s).
+ *	num: the size of the sg array.
+ *	data: the token returned by the get_inbuf function.
+ *      Returns a unique id or an error (eg. -ENOSPC).
+ * @sync: update after add_inbuf and/or add_outbuf
+ *	vdev: the virtio_device we're talking about.
+ *	inout: VIRTIO_IN and/or VIRTIO_OUT
+ *	After one or more add_inbuf/add_outbuf calls, invoke this to kick
+ *	the virtio layer.
+ * @get_outbuf: get the next used outbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length written into the outbuf
+ *	Returns NULL or the "data" token handed to add_outbuf (which has been
+ *	detached).
+ * @get_inbuf: get the next used inbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length read from the inbuf
+ *	Returns NULL or the "data" token handed to add_inbuf (which has been
+ *	detached).
+ * @detach_outbuf: make sure sent sg can no longer be read.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_outbuf.
+ *	This is usually used for shutdown.  Don't try to detach twice.
+ * @detach_inbuf: make sure sent sg can no longer be written to.
+ *	vdev: the virtio_device we're talking about.
+ *	id: the id returned from add_inbuf.
+ *	This is usually used for shutdown.  Don't try to detach twice.
+ * @restart_in: restart calls to driver_ops->in after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	inbufs, to avoid a race.
+ * @restart_out: restart calls to driver_ops->out after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	outbufs, to avoid a race.
+ *
+ * Locking rules are straightforward: the driver is responsible for
+ * locking.  Outbuf operations can be called in parallel to inbuf
+ * operations, but no two outbuf operations nor two inbuf operations
+ * may be invoked simultaneously.
+ *
+ * All operations can be called in any context.
+ */
+struct virtio_ops {
+	unsigned long (*add_outbuf)(struct virtio_device *vdev,
+				    const struct scatterlist sg[],
+				    unsigned int num,
+				    void *data);
+
+	unsigned long (*add_inbuf)(struct virtio_device *vdev,
+				   struct scatterlist sg[],
+				   unsigned int num,
+				   void *data);
+
+	void (*sync)(struct virtio_device *vdev, enum virtio_dir inout);
+
+	void *(*get_outbuf)(struct virtio_device *vdev, unsigned int *len);
+	void *(*get_inbuf)(struct virtio_device *vdev, unsigned int *len);
+
+	void (*detach_outbuf)(struct virtio_device *vdev, unsigned long id);
+	void (*detach_inbuf)(struct virtio_device *vdev, unsigned long id);
+
+	bool (*restart_in)(struct virtio_device *vdev);
+	bool (*restart_out)(struct virtio_device *vdev);
+};
+#endif /* _LINUX_VIRTIO_H */
#68 lore raw

Re: [kvm-devel] [PATCH RFC 0/3] Virtio draft III

In reply to Rusty Russell
Quoted 7 lines · Rusty Russell

In this episode, Rusty tries to NAPI-ize the driver and discovers that virtio callbacks are a bad idea: NAPI needs to turn interrupts off and still be able to query for new incoming packets.

Changes to core:
1) Back to "interrupt" model with get_inbuf()/get_outbuf() calls.

Seems to be these are just two different ways of iterating over the pending buffers, and one could be implemented in terms of the other in a few lines. I think that new new way is better, though, due to less indirection.

btw, I'm not sure that the name 'virtio' is correct. Probably any network or block driver which has support for batching (the vast majority) could use this to good effect.

Signature · 1 line
error compiling committee.c: too many arguments to function
#69 lore raw

cover Virtio draft III

In reply to Rusty Russell

In this episode, Rusty tries to NAPI-ize the driver and discovers that virtio callbacks are a bad idea: NAPI needs to turn interrupts off and still be able to query for new incoming packets.

Changes to core:
1) Back to "interrupt" model with get_inbuf()/get_outbuf() calls.
2) Clearer rules for locking: in calls cannot overlap, out calls cannot overlap, but in can overlap out.
3) Methods for suppressing/enabling "interrupt" calls.

Changes to example net driver:
1) NAPI, locking is now correct (and there is none)

Changes to example block driver:
1) Relay SCSI ioctls (particularly CDROMEJECT) for optional server support (VIRTIO_BLK_T_SCSI_CMD).
2) /dev/vb -> /dev/vd.
3) Barrier support.
4) Header w/ definitions can be included from userspace.

Here's the inter-diff for the three:

include/linux/virtio.h +61 -21
diff -u b/include/linux/virtio.h b/include/linux/virtio.h
@@ -7,69 +7,109 @@
 /**
  * virtio_device - description and routines to drive a virtual device.
- * @lock: the lock to hold before calling any functions.
  * @dev: the underlying struct device.
  * @ops: the operations for this virtual device.
+ * @driver_ops: set by the driver for callbacks.
  * @priv: private pointer for the driver to use.
  */
 struct virtio_device {
-	spinlock_t lock;
 	struct device *dev;
 	struct virtio_ops *ops;
+	struct virtio_driver_ops *driver_ops;
 	void *priv;
 };
 
 /**
+ * virtio_driver_ops - driver callbacks for a virtual device.
+ * @in: inbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further inbuf callbacks.
+ * @out: outbufs have been completed.
+ *	Usually called from an interrupt handler.
+ *	Return false to suppress further outbuf callbacks.
+ */
+struct virtio_driver_ops {
+	bool (*in)(struct virtio_device *dev);
+	bool (*out)(struct virtio_device *dev);
+};
+
+enum virtio_dir {
+	VIRTIO_IN = 0x1,
+	VIRTIO_OUT = 0x2,
+};
+
+/**
  * virtio_ops - virtio abstraction layer
  * @add_outbuf: prepare to send data to the other end:
  *	vdev: the virtio_device
  *	sg: the description of the buffer(s).
  *	num: the size of the sg array.
- *	cb: the function to call once the outbuf is finished & detached.
- *	data: the token to hand to the cb function.
- *      Returns a unique id or an error.  Note that the callback will be
- *	called with the lock held, and possibly in an interrupt handler.
+ *	data: the token returned by the get_outbuf function.
+ *      Returns a unique id or an error.
  * @add_inbuf: prepare to receive data from the other end:
  *	vdev: the virtio_device
  *	sg: the description of the buffer(s).
  *	num: the size of the sg array.
- *	cb: the function to call once the inbuf is finished & detached.
- *	data: the token to hand to the cb function.
- *      Returns a unique id or an error (eg. -ENOSPC).  Note that the
- *	callback will be called with the lock held, and possibly in an
- *	interrupt handler.
- * @sync: update after add_inbuf/add_outbuf
+ *	data: the token returned by the get_inbuf function.
+ *      Returns a unique id or an error (eg. -ENOSPC).
+ * @sync: update after add_inbuf and/or add_outbuf
  *	vdev: the virtio_device we're talking about.
+ *	inout: VIRTIO_IN and/or VIRTIO_OUT
  *	After one or more add_inbuf/add_outbuf calls, invoke this to kick
  *	the virtio layer.
+ * @get_outbuf: get the next used outbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length written into the outbuf
+ *	Returns NULL or the "data" token handed to add_outbuf (which has been
+ *	detached).
+ * @get_inbuf: get the next used inbuf.
+ *	vdev: the virtio_device we're talking about.
+ *	len: the length read from the inbuf
+ *	Returns NULL or the "data" token handed to add_inbuf (which has been
+ *	detached).
  * @detach_outbuf: make sure sent sg can no longer be read.
  *	vdev: the virtio_device we're talking about.
  *	id: the id returned from add_outbuf.
- *	This is not necessary (or valid!) if the outbuf callback has
- *	already fired.
+ *	This is usually used for shutdown.  Don't try to detach twice.
  * @detach_inbuf: make sure sent sg can no longer be written to.
  *	vdev: the virtio_device we're talking about.
  *	id: the id returned from add_inbuf.
- *	This is not necessary (or valid!) if the outbuf callback has
- *	already fired.
+ *	This is usually used for shutdown.  Don't try to detach twice.
+ * @restart_in: restart calls to driver_ops->in after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	inbufs, to avoid a race.
+ * @restart_out: restart calls to driver_ops->out after it returned false.
+ *	vdev: the virtio_device we're talking about.
+ *	This returns "false" (and doesn't re-enable) if there are pending
+ *	outbufs, to avoid a race.
+ *
+ * Locking rules are straightforward: the driver is responsible for
+ * locking.  Outbuf operations can be called in parallel to inbuf
+ * operations, but no two outbuf operations nor two inbuf operations
+ * may be invoked simultaneously.
+ *
+ * All operations can be called in any context.
  */
 struct virtio_ops {
 	unsigned long (*add_outbuf)(struct virtio_device *vdev,
 				    const struct scatterlist sg[],
 				    unsigned int num,
-				    void (*cb)(struct virtio_device *vdev,
-					       void *data, unsigned len),
 				    void *data);
 
 	unsigned long (*add_inbuf)(struct virtio_device *vdev,
 				   struct scatterlist sg[],
 				   unsigned int num,
-				   void (*cb)(struct virtio_device *vdev,
-					      void *data, unsigned len),
 				   void *data);
 
-	void (*sync)(struct virtio_device *vdev);
+	void (*sync)(struct virtio_device *vdev, enum virtio_dir inout);
+
+	void *(*get_outbuf)(struct virtio_device *vdev, unsigned int *len);
+	void *(*get_inbuf)(struct virtio_device *vdev, unsigned int *len);
 
 	void (*detach_outbuf)(struct virtio_device *vdev, unsigned long id);
 	void (*detach_inbuf)(struct virtio_device *vdev, unsigned long id);
+
+	bool (*restart_in)(struct virtio_device *vdev);
+	bool (*restart_out)(struct virtio_device *vdev);
 };
 #endif /* _LINUX_VIRTIO_H */
drivers/net/virtio_net.c +92 -58

248 lines not on this page · raw message

drivers/block/virtio_blk.c +152 -66

363 lines not on this page · raw message

include/linux/virtio_blk.h +21 -10
diff -u b/include/linux/virtio_blk.h b/include/linux/virtio_blk.h
@@ -3,26 +3,37 @@
 #include <linux/types.h>
-struct gendisk;
-struct virtio_device;
-struct hd_geometry;
+
+#define VIRTIO_BLK_T_READ	0
+#define VIRTIO_BLK_T_WRITE 1
+#define VIRTIO_BLK_T_SCSI_CMD 3
+#define VIRTIO_BLK_T_BARRIER 0x80000000 /* Barrier before this op. */
 
 /* This is the first element of the scatter-gather list. */
 struct virtio_blk_outhdr
 {
-	/* 0 == read, 1 == write */
-	u32 type;
+	/* VIRTIO_BLK_T* */
+	__u32 type;
+	/* io priority. */
+	__u32 ioprio;
 	/* Sector (ie. 512 byte offset) */
-	unsigned long sector;
+	__u64 sector;
 	/* Where to put reply. */
-	unsigned long id;
+	__u64 id;
 };
 
+#define VIRTIO_BLK_S_OK		0
+#define VIRTIO_BLK_S_IOERR	1
+#define VIRTIO_BLK_S_UNSUPP	2
+
 struct virtio_blk_inhdr
 {
-	/* 1 = OK, 0 = not ok. */
-	unsigned long status;
+	unsigned char status;
 };
 
+#ifdef __KERNEL__
+struct gendisk;
+struct virtio_device;
+
 struct gendisk *virtblk_probe(struct virtio_device *vdev);
 void virtblk_remove(struct gendisk *disk);
-
+#endif /* __KERNEL__ */
 #endif /* _LINUX_VIRTIO_BLK_H */

only in patch2:
unchanged:

include/linux/Kbuild +1 -0
@@ -341,6 +341,7 @@ unifdef-y += utsname.h
 unifdef-y += utsname.h
 unifdef-y += videodev2.h
 unifdef-y += videodev.h
+unifdef-y += virtio_blk.h
 unifdef-y += wait.h
 unifdef-y += wanrouter.h
 unifdef-y += watchdog.h