Skip to contentSkip to conversation

libibverbs: Add I/O memory registration verbs

RFC PATCH In review Tom Tucker 4 participants 7 messages Jul 29, 2010 3 messages not in the archive InfiniBand

lore
Share & tools
lore

Tom Tucker

author
#0 lore raw

cover libibverbs: Add I/O memory registration verbs

This patch series adds support for the new I/O memory registration services to libibverbs.

---

Tom Tucker (3):
libibverbs: Add reg/unreg I/O memory verbs
libibverbs: Add reg/unreg I/O memory commands to kern ABI
libibverbs: Fix libtool configure warning

 configure.in                  |    1 +
 include/infiniband/driver.h   |    6 ++++++
 include/infiniband/kern-abi.h |   32 +++++++++++++++++++++++++++++++-
 include/infiniband/verbs.h    |   14 ++++++++++++++
 src/cmd.c                     |   41 +++++++++++++++++++++++++++++++++++++++++
 src/libibverbs.map            |    5 +++++
 src/verbs.c                   |   35 +++++++++++++++++++++++++++++++++++
 7 files changed, 133 insertions(+), 1 deletions(-)
Signature · 5 lines
Signed-off-by: Tom Tucker <tom-/Yg/[email protected]>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20100729163030.14901.87547.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org

author
#2 lore raw

RFC PATCH 1/3 libibverbs: Fix libtool configure warning

From: Tom Tucker <[email protected]>

Added AC_PROG_LIBTOOL to configure.in to fix an autogen.sh warning about LIBTOOL configuration.

Signed-off-by Tom Tucker <tom-/Yg/[email protected]>

---

 configure.in |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)
configure.in +1 -0
index 927c406..2bb4aee 100644
@@ -12,6 +12,7 @@ dnl Checks for programs
 AC_PROG_CC
 AC_GNU_SOURCE
 AC_PROG_LN_S
+AC_PROG_LIBTOOL
 
 LT_INIT

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

author
#3 lore raw

RFC PATCH 2/3 libibverbs: Add reg/unreg I/O memory commands to kern ABI

In reply to a message not in the archive

From: Tom Tucker <[email protected]>

Add the commands to register and unregister I/O memory to the Kernel ABI.

Signed-off-by Tom Tucker <tom-/Yg/[email protected]>

---

 include/infiniband/kern-abi.h |   32 +++++++++++++++++++++++++++++++-
 src/cmd.c                     |   41 +++++++++++++++++++++++++++++++++++++++++
 src/libibverbs.map            |    5 +++++
 3 files changed, 77 insertions(+), 1 deletions(-)
include/infiniband/kern-abi.h +31 -1
index 0db083a..56b538f 100644
@@ -85,7 +85,9 @@ enum {
 	IB_USER_VERBS_CMD_MODIFY_SRQ,
 	IB_USER_VERBS_CMD_QUERY_SRQ,
 	IB_USER_VERBS_CMD_DESTROY_SRQ,
-	IB_USER_VERBS_CMD_POST_SRQ_RECV
+	IB_USER_VERBS_CMD_POST_SRQ_RECV,
+	IB_USER_VERBS_CMD_REG_IO_MR,
+	IB_USER_VERBS_CMD_DEREG_IO_MR,
 };
 
 /*

@@ -271,6 +273,32 @@ struct ibv_dereg_mr {
 	__u32 mr_handle;
 };
 
+struct ibv_reg_io_mr {
+        __u32 command;
+        __u16 in_words;
+        __u16 out_words;
+        __u64 response;
+        __u64 start;
+        __u64 length;
+        __u64 hca_va;
+        __u32 pd_handle;
+        __u32 access_flags;
+        __u64 driver_data[0];
+};
+
+struct ibv_reg_io_mr_resp {
+        __u32 mr_handle;
+        __u32 lkey;
+        __u32 rkey;
+};
+
+struct ibv_dereg_io_mr {
+        __u32 command;
+        __u16 in_words;
+        __u16 out_words;
+        __u32 mr_handle;
+};
+
 struct ibv_create_comp_channel {
 	__u32 command;
 	__u16 in_words;

@@ -803,6 +831,8 @@ enum {
 	 * trick opcodes in IBV_INIT_CMD() doesn't break.
 	 */
 	IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL_V2 = -1,
+	IB_USER_VERBS_CMD_REG_IO_MR_V2 = -1,
+	IB_USER_VERBS_CMD_DEREG_IO_MR_V2 = -1,
 };
 
 struct ibv_destroy_cq_v1 {
src/cmd.c +41 -0

47 lines not on this page · raw message

src/libibverbs.map +5 -0
index 1827da0..bc8a251 100644
@@ -84,6 +84,11 @@ IBVERBS_1.1 {
 		ibv_open_device;
 		ibv_close_device;
 
+                ibv_reg_io_mr;
+                ibv_cmd_reg_io_mr;
+                ibv_dereg_io_mr;
+                ibv_cmd_dereg_io_mr;
+
 		ibv_init_ah_from_wc;
 		ibv_create_ah_from_wc;
 		ibv_copy_ah_attr_from_kern;

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

author
#4 lore raw

RFC PATCH 3/3 libibverbs: Add reg/unreg I/O memory verbs

In reply to a message not in the archive

From: Tom Tucker <[email protected]>

Add the ibv_reg_io_mr and ibv_dereg_io_mr verbs.

Signed-off-by Tom Tucker <tom-/Yg/[email protected]>

---

 include/infiniband/driver.h |    6 ++++++
 include/infiniband/verbs.h  |   14 ++++++++++++++
 src/verbs.c                 |   35 +++++++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 0 deletions(-)
include/infiniband/driver.h +6 -0
index 9a81416..37c0ed1 100644
@@ -82,6 +82,12 @@ int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
 		   size_t cmd_size,
 		   struct ibv_reg_mr_resp *resp, size_t resp_size);
 int ibv_cmd_dereg_mr(struct ibv_mr *mr);
+int ibv_cmd_reg_io_mr(struct ibv_pd *pd, void *addr, size_t length,
+		      uint64_t hca_va, int access,
+		      struct ibv_mr *mr, struct ibv_reg_io_mr *cmd,
+		      size_t cmd_size,
+		      struct ibv_reg_io_mr_resp *resp, size_t resp_size);
+int ibv_cmd_dereg_io_mr(struct ibv_mr *mr);
 int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
 		      struct ibv_comp_channel *channel,
 		      int comp_vector, struct ibv_cq *cq,
include/infiniband/verbs.h +14 -0
index 0f1cb2e..a0d969a 100644
@@ -640,6 +640,9 @@ struct ibv_context_ops {
 					    size_t length,
 					    int access);
 	int			(*dereg_mr)(struct ibv_mr *mr);
+        struct ibv_mr *         (*reg_io_mr)(struct ibv_pd *pd, void *addr, size_t length,
+					     int access);
+        int                     (*dereg_io_mr)(struct ibv_mr *mr);
 	struct ibv_mw *		(*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type);
 	int			(*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw,
 					   struct ibv_mw_bind *mw_bind);

@@ -801,6 +804,17 @@ struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
 int ibv_dereg_mr(struct ibv_mr *mr);
 
 /**
+ * ibv_reg_io_mr - Register a physical memory region
+ */
+struct ibv_mr *ibv_reg_io_mr(struct ibv_pd *pd, void *addr,
+                             size_t length, int access);
+
+/**
+ * ibv_dereg_io_mr - Deregister a physical memory region
+ */
+int ibv_dereg_io_mr(struct ibv_mr *mr);
+
+/**
  * ibv_create_comp_channel - Create a completion event channel
  */
 struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context);
src/verbs.c +35 -0
index ba3c0a4..7d215c1 100644
@@ -189,6 +189,41 @@ int __ibv_dereg_mr(struct ibv_mr *mr)
 }
 default_symver(__ibv_dereg_mr, ibv_dereg_mr);
 
+struct ibv_mr *__ibv_reg_io_mr(struct ibv_pd *pd, void *addr,
+                               size_t length, int access)
+{
+        struct ibv_mr *mr;
+
+        if (ibv_dontfork_range(addr, length))
+                return NULL;
+
+        mr = pd->context->ops.reg_io_mr(pd, addr, length, access);
+        if (mr) {
+                mr->context = pd->context;
+                mr->pd      = pd;
+                mr->addr    = addr;
+                mr->length  = length;
+        } else
+                ibv_dofork_range(addr, length);
+
+        return mr;
+}
+default_symver(__ibv_reg_io_mr, ibv_reg_io_mr);
+
+int __ibv_dereg_io_mr(struct ibv_mr *mr)
+{
+        int ret;
+        void *addr      = mr->addr;
+        size_t length   = mr->length;
+
+        ret = mr->context->ops.dereg_io_mr(mr);
+        if (!ret)
+                ibv_dofork_range(addr, length);
+
+        return ret;
+}
+default_symver(__ibv_dereg_io_mr, ibv_dereg_io_mr);
+
 static struct ibv_comp_channel *ibv_create_comp_channel_v2(struct ibv_context *context)
 {
 	struct ibv_abi_compat_v2 *t = context->abi_compat;

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20100729163202.14901.44211.stgit-T4OLL4TyM9aNDNWfRnPdfg@public.gmane.org

#6 lore raw

Re: [RFC PATCH 3/3] libibverbs: Add reg/unreg I/O memory verbs

How does an application know when to call ibv_reg_io_mr() instead of ibv_reg_mr()? It isn't going to know that some address returned by mmap() is going to have the VM_PFNMAP flag set.

How does an application know that the HCA supports ibv_reg_io_mr() or not? (see below)
I think returning ENOTSUP or something would be good.

Quoted 34 lines · Tom Tucker

From: Tom Tucker <[email protected]>

Add the ibv_reg_io_mr and ibv_dereg_io_mr verbs.

Signed-off-by Tom Tucker <tom-/Yg/[email protected]>

---

 include/infiniband/driver.h |    6 ++++++
 include/infiniband/verbs.h  |   14 ++++++++++++++
 src/verbs.c                 |   35 +++++++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 0 deletions(-)
include/infiniband/driver.h +6 -0
index 9a81416..37c0ed1 100644
@@ -82,6 +82,12 @@ int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
 		   size_t cmd_size,
 		   struct ibv_reg_mr_resp *resp, size_t resp_size);
 int ibv_cmd_dereg_mr(struct ibv_mr *mr);
+int ibv_cmd_reg_io_mr(struct ibv_pd *pd, void *addr, size_t length,
+		      uint64_t hca_va, int access,
+		      struct ibv_mr *mr, struct ibv_reg_io_mr *cmd,
+		      size_t cmd_size,
+		      struct ibv_reg_io_mr_resp *resp, size_t resp_size);
+int ibv_cmd_dereg_io_mr(struct ibv_mr *mr);
 int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
 		      struct ibv_comp_channel *channel,
 		      int comp_vector, struct ibv_cq *cq,
include/infiniband/verbs.h +0 -0
index 0f1cb2e..a0d969a 100644

include/infiniband/verbs.h struct ibv_context_ops { · line 648

Quoted 9 lines
include/infiniband/verbs.h +3 -0
@@ -640,6 +640,9 @@ struct ibv_context_ops {
 					    size_t length,
 					    int access);
 	int			(*dereg_mr)(struct ibv_mr *mr);
+        struct ibv_mr *         (*reg_io_mr)(struct ibv_pd *pd, void *addr, size_t length,
+					     int access);
+        int                     (*dereg_io_mr)(struct ibv_mr *mr);
 	struct ibv_mw *		(*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type);
 	int			(*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw,
 					   struct ibv_mw_bind *mw_bind);

Doesn't adding these in the middle of the struct break the libibverbs to libxxxverbs.so binary interface? Shouldn't they be added at the end of the struct? I'm not sure how the versioning works between libibverbs and device plugins. Don't we need to protect against libibverbs being upgraded but the libxxxverbs.so being older?

Quoted 25 lines
include/infiniband/verbs.h +11 -0
@@ -801,6 +804,17 @@ struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
 int ibv_dereg_mr(struct ibv_mr *mr);
 
 /**
+ * ibv_reg_io_mr - Register a physical memory region
+ */
+struct ibv_mr *ibv_reg_io_mr(struct ibv_pd *pd, void *addr,
+                             size_t length, int access);
+
+/**
+ * ibv_dereg_io_mr - Deregister a physical memory region
+ */
+int ibv_dereg_io_mr(struct ibv_mr *mr);
+
+/**
  * ibv_create_comp_channel - Create a completion event channel
  */
 struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context);
src/verbs.c +0 -0
index ba3c0a4..7d215c1 100644
@@ -189,6 +189,41 @@ int __ibv_dereg_mr(struct ibv_mr *mr)
 }
 default_symver(__ibv_dereg_mr, ibv_dereg_mr);

src/verbs.c int __ibv_dereg_mr(struct ibv_mr *mr) · line 200

Quoted 10 lines
src/verbs.c +9 -0
 
+struct ibv_mr *__ibv_reg_io_mr(struct ibv_pd *pd, void *addr,
+                               size_t length, int access)
+{
+        struct ibv_mr *mr;
+
+        if (ibv_dontfork_range(addr, length))
+                return NULL;
+
+        mr = pd->context->ops.reg_io_mr(pd, addr, length, access);

Won't reg_io_mr pointer be NULL for other HCAs?
What happens if the device doesn't yet implement this function?

Quoted 31 lines
src/verbs.c +26 -1
+        if (mr) {
+                mr->context = pd->context;
+                mr->pd      = pd;
+                mr->addr    = addr;
+                mr->length  = length;
+        } else
+                ibv_dofork_range(addr, length);
+
+        return mr;
+}
+default_symver(__ibv_reg_io_mr, ibv_reg_io_mr);
+
+int __ibv_dereg_io_mr(struct ibv_mr *mr)
+{
+        int ret;
+        void *addr      = mr->addr;
+        size_t length   = mr->length;
+
+        ret = mr->context->ops.dereg_io_mr(mr);
+        if (!ret)
+                ibv_dofork_range(addr, length);
+
+        return ret;
+}
+default_symver(__ibv_dereg_io_mr, ibv_dereg_io_mr);
+
 static struct ibv_comp_channel *ibv_create_comp_channel_v2(struct ibv_context *context)
 {
 	struct ibv_abi_compat_v2 *t = context->abi_compat;
 
--
Quoted 4 lines

To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 1280434056.6820.136.camel-/[email protected]

#8 lore raw

Re: [RFC PATCH 3/3] libibverbs: Add reg/unreg I/O memory verbs

Quoted 9 lines · Ralph Campbell
Quoted 9 lines
include/infiniband/verbs.h +3 -0
@@ -640,6 +640,9 @@ struct ibv_context_ops {
 					    size_t length,
 					    int access);
 	int			(*dereg_mr)(struct ibv_mr *mr);
+        struct ibv_mr *         (*reg_io_mr)(struct ibv_pd *pd, void *addr, size_t length,
+					     int access);
+        int                     (*dereg_io_mr)(struct ibv_mr *mr);
 	struct ibv_mw *		(*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type);
 	int			(*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw,
Quoted 8 lines

include/infiniband/verbs.h struct ibv_context_ops { · line 648

Quoted 1 line
include/infiniband/verbs.h +0 -0
 					   struct ibv_mw_bind *mw_bind);

Doesn't adding these in the middle of the struct break the libibverbs to libxxxverbs.so binary interface? Shouldn't they be added at the end of the struct? I'm not sure how the versioning works between libibverbs and device plugins. Don't we need to protect against libibverbs being upgraded but the libxxxverbs.so being older?

Adding them at all, anywhere, breaks the ABI, and needs a soname bump of libibverbs. Last time this was done a versioned symbol approach was used to try and keep handling a 1.0 ibv_context structure, but that only works some times and breaks in corner cases :(

Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#9 lore raw

Re: [RFC PATCH 3/3] libibverbs: Add reg/unreg I/O memory verbs

In reply to a message not in the archive
Quoted 5 lines · Ralph Campbell

How does an application know when to call ibv_reg_io_mr() instead of ibv_reg_mr()? It isn't going to know that some address returned by mmap() is going to have the VM_PFNMAP flag set.

Please see my response to Jason.

Quoted 5 lines

How does an application know that the HCA supports ibv_reg_io_mr() or not? (see below)
I think returning ENOTSUP or something would be good.

There are bits in the devcaps that indicate if these verbs are supported. It should however return -ENOTSUPP if they are called without support. I copied ibv_reg_mr's which is inappropriate in this regard.

Quoted 44 lines · Tom Tucker
Quoted 43 lines

From: Tom Tucker<[email protected]>

Add the ibv_reg_io_mr and ibv_dereg_io_mr verbs.

Signed-off-by Tom Tucker<tom-/Yg/[email protected]>

---

  include/infiniband/driver.h |    6 ++++++
  include/infiniband/verbs.h  |   14 ++++++++++++++
  src/verbs.c                 |   35 +++++++++++++++++++++++++++++++++++
  3 files changed, 55 insertions(+), 0 deletions(-)
include/infiniband/driver.h +6 -0
index 9a81416..37c0ed1 100644
@@ -82,6 +82,12 @@ int ibv_cmd_reg_mr(struct ibv_pd *pd, void *addr, size_t length,
  		   size_t cmd_size,
  		   struct ibv_reg_mr_resp *resp, size_t resp_size);
  int ibv_cmd_dereg_mr(struct ibv_mr *mr);
+int ibv_cmd_reg_io_mr(struct ibv_pd *pd, void *addr, size_t length,
+		      uint64_t hca_va, int access,
+		      struct ibv_mr *mr, struct ibv_reg_io_mr *cmd,
+		      size_t cmd_size,
+		      struct ibv_reg_io_mr_resp *resp, size_t resp_size);
+int ibv_cmd_dereg_io_mr(struct ibv_mr *mr);
  int ibv_cmd_create_cq(struct ibv_context *context, int cqe,
  		      struct ibv_comp_channel *channel,
  		      int comp_vector, struct ibv_cq *cq,
include/infiniband/verbs.h +3 -0
index 0f1cb2e..a0d969a 100644
@@ -640,6 +640,9 @@ struct ibv_context_ops {
  					    size_t length,
  					    int access);
  	int			(*dereg_mr)(struct ibv_mr *mr);
+        struct ibv_mr *         (*reg_io_mr)(struct ibv_pd *pd, void *addr, size_t length,
+					     int access);
+        int                     (*dereg_io_mr)(struct ibv_mr *mr);
  	struct ibv_mw *		(*alloc_mw)(struct ibv_pd *pd, enum ibv_mw_type type);
  	int			(*bind_mw)(struct ibv_qp *qp, struct ibv_mw *mw,
Quoted 10 lines

include/infiniband/verbs.h struct ibv_context_ops { · line 648

Quoted 2 lines
include/infiniband/verbs.h +0 -0
  					   struct ibv_mw_bind *mw_bind);

Doesn't adding these in the middle of the struct break the libibverbs to libxxxverbs.so binary interface? Shouldn't they be added at the end of the struct? I'm not sure how the versioning works between libibverbs and device plugins. Don't we need to protect against libibverbs being upgraded but the libxxxverbs.so being older?

I would think it's broken regardless.

Quoted 26 lines
Quoted 26 lines
include/infiniband/verbs.h +11 -0
@@ -801,6 +804,17 @@ struct ibv_mr *ibv_reg_mr(struct ibv_pd *pd, void *addr,
  int ibv_dereg_mr(struct ibv_mr *mr);
 
  /**
+ * ibv_reg_io_mr - Register a physical memory region
+ */
+struct ibv_mr *ibv_reg_io_mr(struct ibv_pd *pd, void *addr,
+                             size_t length, int access);
+
+/**
+ * ibv_dereg_io_mr - Deregister a physical memory region
+ */
+int ibv_dereg_io_mr(struct ibv_mr *mr);
+
+/**
   * ibv_create_comp_channel - Create a completion event channel
   */
  struct ibv_comp_channel *ibv_create_comp_channel(struct ibv_context *context);
src/verbs.c +0 -0
index ba3c0a4..7d215c1 100644
@@ -189,6 +189,41 @@ int __ibv_dereg_mr(struct ibv_mr *mr)
  }
  default_symver(__ibv_dereg_mr, ibv_dereg_mr);
Quoted 14 lines

src/verbs.c int __ibv_dereg_mr(struct ibv_mr *mr)

Quoted 10 lines
src/verbs.c +9 -0
+struct ibv_mr *__ibv_reg_io_mr(struct ibv_pd *pd, void *addr,
+                               size_t length, int access)
+{
+        struct ibv_mr *mr;
+
+        if (ibv_dontfork_range(addr, length))
+                return NULL;
+
+        mr = pd->context->ops.reg_io_mr(pd, addr, length, access);

Won't reg_io_mr pointer be NULL for other HCAs?
What happens if the device doesn't yet implement this function?

Without a check, SEGV. See above.

Quoted 30 lines
Quoted 30 lines
src/verbs.c +26 -0
+        if (mr) {
+                mr->context = pd->context;
+                mr->pd      = pd;
+                mr->addr    = addr;
+                mr->length  = length;
+        } else
+                ibv_dofork_range(addr, length);
+
+        return mr;
+}
+default_symver(__ibv_reg_io_mr, ibv_reg_io_mr);
+
+int __ibv_dereg_io_mr(struct ibv_mr *mr)
+{
+        int ret;
+        void *addr      = mr->addr;
+        size_t length   = mr->length;
+
+        ret = mr->context->ops.dereg_io_mr(mr);
+        if (!ret)
+                ibv_dofork_range(addr, length);
+
+        return ret;
+}
+default_symver(__ibv_dereg_io_mr, ibv_dereg_io_mr);
+
  static struct ibv_comp_channel *ibv_create_comp_channel_v2(struct ibv_context *context)
  {

  	struct ibv_abi_compat_v2 *t = context->abi_compat;
Quoted 12 lines
Quoted 6 lines

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html