ATA over Ethernet driver for 2.6.9

The included patch allows the Linux kernel to use the ATA over Ethernet (AoE) network protocol to communicate with any block device that handles the AoE protocol. The Coraid EtherDrive (R) Storage Blade is the first hardware using AoE.

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

diff -urpN linux-2.6.9/drivers/block/Kconfig linux-2.6.9-aoe/drivers/block/Kconfig

drivers/block/Kconfig +1 -0

@@ -357,5 +357,6 @@ config LBD
 	  bigger than 2TB.  Otherwise say N.
 
 source "drivers/s390/block/Kconfig"
+source "drivers/block/aoe/Kconfig"
 
 endmenu

diff -urpN linux-2.6.9/drivers/block/aoe/Kconfig linux-2.6.9-aoe/drivers/block/aoe/Kconfig

diff -urpN linux-2.6.9/drivers/block/aoe/all.h linux-2.6.9-aoe/drivers/block/aoe/all.h

drivers/block/aoe/all.h +0 -0

+#define nil NULL

+#define nelem(A) (sizeof (A) / sizeof (A)[0])

drivers/block/aoe/aoeblk.c +8 -0

+void
+aoeblk_exit(void)
+{
+	unregister_blkdev(AOE_MAJOR, DEVICE_NAME);
+}
+
+int
+aoeblk_init(void)

drivers/block/aoe/aoechr.c +53 -0

+u16
+nhget16(uchar *p)
+{
+	u16 n;
+
+	n = p[0];
+	n <<= 8;
+	return n |= p[1];
+}
+
+u32
+nhget32(uchar *p)
+{
+	u32 n;
+
+	n = nhget16(p);
+	n <<= 16;
+	return n |= nhget16(p+2);
+}
+
+void
+hnput16(uchar *p, u16 n)
+{
+	p[1] = n;
+	p[0] = n >>= 8;
+}
+
+void
+hnput32(uchar *p, u32 n)
+{
+	hnput16(p+2, n);
+	hnput16(p, n >>= 16);
+}
+
+u16
+lhget16(uchar *p)
+{
+	u16 n;
+
+	n = p[1];
+	n <<= 8;
+	return n |= p[0];
+}
+
+u32
+lhget32(uchar *p)
+{
+	u32 n;
+
+	n = lhget16(p+2);
+	n <<= 16;
+	return n |= lhget16(p);
+}

The included patch allows the Linux kernel to use the ATA over Ethernet (AoE) network protocol to communicate with any block device that handles the AoE protocol. The Coraid EtherDrive (R) Storage Blade is the first hardware using AoE.

AoE devices on the LAN are accessable as block devices and can be used with filesystems, Software RAID, LVM, etc.

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

This patch is released under the terms of the GPL.

(We also have an AoE driver for the 2.4 kernel that we plan to release soon.)

Quoted 11 lines · Ed L Cashin

The included patch allows the Linux kernel to use the ATA over Ethernet (AoE) network protocol to communicate with any block device that handles the AoE protocol. The Coraid EtherDrive (R) Storage Blade is the first hardware using AoE.

AoE devices on the LAN are accessable as block devices and can be used with filesystems, Software RAID, LVM, etc.

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

Quoted 4 lines

This patch is released under the terms of the GPL.

(We also have an AoE driver for the 2.4 kernel that we plan to release soon.)

Is there a free server for this?

On Mon, 2004-12-06 10:51:46 -0500, Ed L Cashin <[email protected]>

Quoted 2 lines

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

So AoE is out of scope for many uses...

After all, especially keeping in mind that AoE isn't routeable, my thinking is that this had better written as a (E)NBD server process running in userspace. This way, you'd use the in-kernel NBD driver (or the ENBD which isn't in the kernel) and you the the routing stuff for free :)

Quoted 15 lines · Adam Heath

Quoted 13 lines · Ed L Cashin

The included patch allows the Linux kernel to use the ATA over Ethernet (AoE) network protocol to communicate with any block device that handles the AoE protocol. The Coraid EtherDrive (R) Storage Blade is the first hardware using AoE.

AoE devices on the LAN are accessable as block devices and can be used with filesystems, Software RAID, LVM, etc.

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

This patch is released under the terms of the GPL.

Quoted 4 lines

Quoted 2 lines

(We also have an AoE driver for the 2.4 kernel that we plan to release soon.)

Is there a free server for this?

Are you asking whether anybody has written software that allows a network host to export block storage using AoE? Not yet, as far as I know.

Quoted 4 lines

Quoted 2 lines

(We also have an AoE driver for the 2.4 kernel that we plan to release soon.)

Is there a free server for this?

Are you asking whether anybody has written software that allows a network host to export block storage using AoE? Not yet, as far as I know.

Quoted 2 lines

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

So AoE is out of scope for many uses...

drivers/block/aoe/aoeutils.c line 182

Quoted 4 lines

drivers/block/aoe/aoeutils.c +4 -0

+	n = lhget32(p+4);
+	n <<= 32;
+	return n |= lhget32(p);
+}

There are function available for this, look at the endianess header files.

After all, especially keeping in mind that AoE isn't routeable, my thinking is that this had better written as a (E)NBD server process running in userspace. This way, you'd use the in-kernel NBD driver (or the ENBD which isn't in the kernel) and you the the routing stuff for free :)

Documentation/aoe.txt +7 -0

+USING DEVICE NODES
+
+  "cat /dev/etherd/stat" shows the status of discovered AoE devices on
+  your LAN:
+
+	root@nai root# cat /dev/etherd/stat
+	/dev/etherd/e15.3       eth0    up

+typedef struct Buf Buf;

drivers/block/aoe/aoeblk.c +7 -0

+static int
+aoeblk_release(struct inode *inode, struct file *filp)
+{
+	Aoedev *d;
+	ulong flags;
+
+	d = (Aoedev *) inode->i_bdev->bd_disk->private_data;

drivers/block/aoe/aoeblk.c +11 -0

+static int
+aoeblk_ioctl(struct inode *inode, struct file *filp, uint cmd, ulong arg)
+{
+	Aoedev *d;
+
+	if (!arg)
+		return -EINVAL;
+
+	d = (Aoedev *) inode->i_bdev->bd_disk->private_data;
+	if ((d->flags & DEVFL_UP) == 0) {
+		printk(KERN_ERR "aoe: aoeblk_ioctl: disk not up\n");

drivers/block/aoe/aoechr.c +5 -0

+int
+aoechr_open(struct inode *inode, struct file *filp)
+{
+	int n;
+

+#include "u.h"

Quoted 7 lines

The included patch allows the Linux kernel to use the ATA over Ethernet (AoE) network protocol to communicate with any block device that handles the AoE protocol. The Coraid EtherDrive (R) Storage Blade is the first hardware using AoE.

Like IP, AoE is an ethernet-level network protocol, registered with the IEEE. Unlike IP, AoE is not routable.

So AoE is out of scope for many uses...

drivers/block/aoe/all.h 2004-12-06 10:40:00.000000000 -0500 renamed from drivers/block/aoe/all.h 1969-12-31 19:00:00.000000000 -0500 +12 -0

@@ -0,0 +1,156 @@
+#include <linux/ctype.h>
+#include <linux/string.h>
+#include <linux/hdreg.h>
+#include <linux/blkdev.h>
+#include <linux/types.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+#include <linux/blkpg.h>
+#include <linux/slab.h>
+#include <linux/skbuff.h>
+#include <linux/ioctl.h>
+#include <linux/if.h>

drivers/block/aoe/all.h 2004-12-06 10:40:00.000000000 -0500 renamed from drivers/block/aoe/all.h 1969-12-31 19:00:00.000000000 -0500 +4 -0

+enum {
+       DEVFL_UP = 1,           /* device is installed in system and ready for AoE->ATA commands */
+       DEVFL_TKILL = (1<<1),   /* flag for timer to know when to kill self */
+       DEVFL_EXT = (1<<2),     /* device accepts lba48 commands */

Well, at least it has a chance to correctly work in low-memory conditions, and it might be possible to swap over AoE.
ARPs are real problem there.
Pavel

Quoted 4 lines · Pavel Machek

Well, at least it has a chance to correctly work in low-memory conditions, and it might be possible to swap over AoE.
ARPs are real problem there.

Well, how about caching the hw address of the AoE device somewhere in the data structure describing the block device? Then you won't need to ARP for it, and I guess that address isn't likely to change while the device is in use?

drivers/block/aoe/aoeblk.c line 115

Quoted 1 line

drivers/block/aoe/aoeblk.c +1 -0

+	printk(KERN_INFO "aoe: aoeblk_ioctl: unknown ioctl %d\n", cmd);

So I can flood the syslog by sending improper ioctls to the driver? That's not nice...

drivers/block/aoe/all.h line 70

Quoted 4 lines

drivers/block/aoe/all.h +4 -0

+typedef struct Bufq Bufq;
+struct Bufq {
+	Buf *head, *tail;
+};

What's wrong with the in-kernel list structures that you need to create your own?

Greg KH <[email protected]> writes:

...

Quoted 7 lines

drivers/block/aoe/all.h line 70

Quoted 4 lines

drivers/block/aoe/all.h +4 -0

+typedef struct Bufq Bufq;
+struct Bufq {
+	Buf *head, *tail;
+};

What's wrong with the in-kernel list structures that you need to create your own?

The Buf structures are singly linked (one next pointer). We could convert it to use list.h, but that would mean having another pointer per list element.

Quoted 12 lines · Greg KH

Documentation/aoe.txt line 13

Quoted 7 lines · Ed L Cashin

Documentation/aoe.txt +7 -0

+CREATING DEVICE NODES
+
+  Two scripts are in scripts/aoe for assisting in creating device
+  nodes for using the aoe driver.  Usage is as follows.
+
+    rm -rf /dev/etherd
+    sh scripts/mkdevs /dev/etherd

If you use the /sys/class interface properly, and udev, you don't need this script at all. Care to add sysfs support to the driver so that people don't have to rely on this?

Right now genhd.c seems to be giving us some automatic sysfs support for the block devices:

root@makki root# ls /sys/block/

etherd!e15.3  etherd!e5.2  etherd!e5.6  fd0  md0    ram11  ram15  ram5  ram9
etherd!e3.0   etherd!e5.3  etherd!e5.7  hda  ram0   ram12  ram2   ram6
etherd!e5.0   etherd!e5.4  etherd!e5.8  hdb  ram1   ram13  ram3   ram7
etherd!e5.1   etherd!e5.5  etherd!e5.9  hdc  ram10  ram14  ram4   ram8

Will using class_simple_device_add register our char devices so that udev can create the device nodes dynamically? I'm looking at the example in netlink_dev.c.

Greg KH <[email protected]> writes:

...

Quoted 4 lines

drivers/block/aoe/aoeblk.c line 115

Quoted 1 line

drivers/block/aoe/aoeblk.c +1 -0

+	printk(KERN_INFO "aoe: aoeblk_ioctl: unknown ioctl %d\n", cmd);

So I can flood the syslog by sending improper ioctls to the driver? That's not nice...

Wouldn't root be the only user who could do that? When would this happen?

Quoted 5 lines

Well, at least it has a chance to correctly work in low-memory
conditions,
and it might be possible to swap over AoE.
ARPs are real problem there.
> Well, how about caching the hw address of the AoE device

somewhere in the data structure describing the block device?s Then you won't need to ARP for it, and I guess that address isn't likely to change while the device is in use?

drivers/block/aoe/all.h line 90

Quoted 7 lines

drivers/block/aoe/all.h +7 -0

+struct Aoedev {
+	Aoedev *next;
+	uchar addr[6];		/* remote mac addr */
+	ushort flags;
+	ulong sysminor;
+	ulong aoemajor;
+	ulong aoeminor;

sounds like the wrong type, why not use dev_t ?

drivers/block/aoe/all.h line 91

Quoted 1 line

drivers/block/aoe/all.h +1 -0

+	ulong nopen;		/* user count */

why do you need this ?

drivers/block/aoe/aoeblk.c line 34

Quoted 10 lines

drivers/block/aoe/aoeblk.c +10 -0

+static int
+aoeblk_release(struct inode *inode, struct file *filp)
+{
+	Aoedev *d;
+	ulong flags;
+
+	d = (Aoedev *) inode->i_bdev->bd_disk->private_data;
+
+	spin_lock_irqsave(&d->lock, flags);
+	if (--d->nopen == 0)

eh why not just a ->release function instead that uses the blocklayer refcounting instead of doing your own ?

Quoted 5 lines

drivers/block/aoe/aoeblk.c +5 -0

+int
+aoeblk_make_request(request_queue_t *q, struct bio *bio)
+{
+	Aoedev *d;
+	Buf *buf;

drivers/block/aoe/aoeblk.c line 56

Quoted 6 lines

drivers/block/aoe/aoeblk.c +6 -0

+	struct sk_buff *sl;
+	ulong flags;
+
+	blk_queue_bounce(q, &bio);
+
+	buf = kallocz(sizeof *buf, GFP_KERNEL);

this is deadlocky; you HAVE to use a mempool for allocations here!