Skip to contentSkip to conversation

Add MAD helper functions

PATCH In review Hal Rosenstock 6 participants 12 messages Jul 11, 2005 – Jul 12, 2005

lore
Share & tools
lore
#0 lore raw

PATCH 3/27 Add MAD helper functions

Add new helper routines for allocating MADs for sending and formatting a send WR.

This patch depends on patch 2/27.

Signature · 178 lines
 core/mad.c       |   76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 include/ib_mad.h |   60 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+)
diff -uprN linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c
-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c	2005-07-09 12:58:23.000000000 -0400
+++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c	2005-07-09 14:31:49.000000000 -0400
@@ -763,6 +763,82 @@ out:
 	return ret;
 }
 
+static int get_buf_length(int hdr_len, int data_len)
+{
+	int seg_size, pad;
+
+	seg_size = sizeof(struct ib_mad) - hdr_len;
+	if (data_len && seg_size) {
+		pad = seg_size - data_len % seg_size;
+		if (pad == seg_size)
+			pad = 0;
+	} else
+		pad = seg_size;
+	return hdr_len + data_len + pad;
+}
+
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah,
+					    int hdr_len, int data_len,
+					    int gfp_mask)
+{
+	struct ib_mad_agent_private *mad_agent_priv;
+	struct ib_mad_send_buf *send_buf;
+	int buf_size;
+	void *buf;
+
+	mad_agent_priv = container_of(mad_agent,
+				      struct ib_mad_agent_private, agent);
+	buf_size = get_buf_length(hdr_len, data_len);
+
+	buf = kmalloc(sizeof *send_buf + buf_size, gfp_mask);
+	if (!buf)
+		return ERR_PTR(-ENOMEM);
+
+	send_buf = buf + buf_size;
+	memset(send_buf, 0, sizeof *send_buf);
+	send_buf->mad = buf;
+
+	send_buf->sge.addr = dma_map_single(mad_agent->device->dma_device,
+					    buf, buf_size, DMA_TO_DEVICE);
+	pci_unmap_addr_set(send_buf, mapping, send_buf->sge.addr);
+	send_buf->sge.length = buf_size;
+	send_buf->sge.lkey = mad_agent->mr->lkey;
+
+	send_buf->send_wr.wr_id = (unsigned long) send_buf;
+	send_buf->send_wr.sg_list = &send_buf->sge;
+	send_buf->send_wr.num_sge = 1;
+	send_buf->send_wr.opcode = IB_WR_SEND;
+	send_buf->send_wr.send_flags = IB_SEND_SIGNALED;
+	send_buf->send_wr.wr.ud.ah = ah;
+	send_buf->send_wr.wr.ud.mad_hdr = &send_buf->mad->mad_hdr;
+	send_buf->send_wr.wr.ud.remote_qpn = remote_qpn;
+	send_buf->send_wr.wr.ud.remote_qkey = IB_QP_SET_QKEY;
+	send_buf->send_wr.wr.ud.pkey_index = pkey_index;
+	send_buf->mad_agent = mad_agent;
+	atomic_inc(&mad_agent_priv->refcount);
+	return send_buf;
+}
+EXPORT_SYMBOL(ib_create_send_mad);
+
+void ib_free_send_mad(struct ib_mad_send_buf *send_buf)
+{
+	struct ib_mad_agent_private *mad_agent_priv;
+
+	mad_agent_priv = container_of(send_buf->mad_agent,
+				      struct ib_mad_agent_private, agent);
+
+	dma_unmap_single(send_buf->mad_agent->device->dma_device,
+			 pci_unmap_addr(send_buf, mapping),
+			 send_buf->sge.length, DMA_TO_DEVICE);
+	kfree(send_buf->mad);
+
+	if (atomic_dec_and_test(&mad_agent_priv->refcount))
+		wake_up(&mad_agent_priv->wait);
+}
+EXPORT_SYMBOL(ib_free_send_mad);
+
 static int ib_send_mad(struct ib_mad_agent_private *mad_agent_priv,
 		       struct ib_mad_send_wr_private *mad_send_wr)
 {
diff -uprN linux-2.6.13-rc2-mm1/drivers/infiniband2/include/ib_mad.h linux-2.6.13-rc2-mm1/drivers/infiniband3/include/ib_mad.h
-- linux-2.6.13-rc2-mm1/drivers/infiniband2/include/ib_mad.h	2005-07-09 12:06:49.000000000 -0400
+++ linux-2.6.13-rc2-mm1/drivers/infiniband3/include/ib_mad.h	2005-07-09 14:26:49.000000000 -0400
@@ -39,6 +39,8 @@
 #if !defined( IB_MAD_H )
 #define IB_MAD_H
 
+#include <linux/pci.h>
+
 #include <ib_verbs.h>
 
 /* Management base version */
@@ -73,6 +75,7 @@
 #define IB_QP0		0
 #define IB_QP1		__constant_htonl(1)
 #define IB_QP1_QKEY	0x80010000
+#define IB_QP_SET_QKEY	0x80000000
 
 struct ib_grh {
 	u32		version_tclass_flow;
@@ -124,6 +127,30 @@ struct ib_vendor_mad {
 	u8			data[216];
 } __attribute__ ((packed));
 
+/**
+ * ib_mad_send_buf - MAD data buffer and work request for sends.
+ * @mad: References an allocated MAD data buffer.  The size of the data
+ *   buffer is specified in the @send_wr.length field.
+ * @mapping: DMA mapping information.
+ * @mad_agent: MAD agent that allocated the buffer.
+ * @context: User-controlled context fields.
+ * @send_wr: An initialized work request structure used when sending the MAD.
+ *   The wr_id field of the work request is initialized to reference this
+ *   data structure.
+ * @sge: A scatter-gather list referenced by the work request.
+ *
+ * Users are responsible for initializing the MAD buffer itself, with the
+ * exception of specifying the payload length field in any RMPP MAD.
+ */
+struct ib_mad_send_buf {
+	struct ib_mad		*mad;
+	DECLARE_PCI_UNMAP_ADDR(mapping)
+	struct ib_mad_agent	*mad_agent;
+	void			*context[2];
+	struct ib_send_wr	send_wr;
+	struct ib_sge		sge;
+};
+
 struct ib_mad_agent;
 struct ib_mad_send_wc;
 struct ib_mad_recv_wc;
@@ -402,4 +429,37 @@ struct ib_mad_agent *ib_redirect_mad_qp(
 int ib_process_mad_wc(struct ib_mad_agent *mad_agent,
 		      struct ib_wc *wc);
 
+/**
+ * ib_create_send_mad - Allocate and initialize a data buffer and work request
+ *   for sending a MAD.
+ * @mad_agent: Specifies the registered MAD service to associate with the MAD.
+ * @remote_qpn: Specifies the QPN of the receiving node.
+ * @pkey_index: Specifies which PKey the MAD will be sent using.  This field
+ *   is valid only if the remote_qpn is QP 1.
+ * @ah: References the address handle used to transfer to the remote node.
+ * @hdr_len: Indicates the size of the data header of the MAD.  This length
+ *   should include the common MAD header, RMPP header, plus any class
+ *   specific header.
+ * @data_len: Indicates the size of any user-transfered data.  The call will
+ *   automatically adjust the allocated buffer size to account for any
+ *   additional padding that may be necessary.
+ * @gfp_mask: GFP mask used for the memory allocation.
+ *
+ * This is a helper routine that may be used to allocate a MAD.  Users are
+ * not required to allocate outbound MADs using this call.  The returned
+ * MAD send buffer will reference a data buffer usable for sending a MAD, along
+ * with an intialized work request structure.
+ */
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah,
+					    int hdr_len, int data_len,
+					    int gfp_mask);
+
+/**
+ * ib_free_send_mad - Returns data buffers used to send a MAD.
+ * @send_buf: Previously allocated send data buffer.
+ */
+void ib_free_send_mad(struct ib_mad_send_buf *send_buf);
+
 #endif /* IB_MAD_H */
#1 lore raw
Quoted 5 lines · Hal Rosenstock

Add new helper routines for allocating MADs for sending and formatting a send WR.

-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c +++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c

				   ^^^^^^^^^^^

Ick. You'd better have linux-2.6.13-rc2-mm1-[0123...].

Review

Quoted 5 lines
(quoted diff) +5 -0
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah,
+					    int hdr_len, int data_len,
+					    int gfp_mask)

unsigned int __nocast gfp_mask, please. 430 or so infiniband sparse warnings is not a reason to add more.

Quoted 3 lines
(quoted diff) +2 -0
+{
 
+	buf = kmalloc(sizeof *send_buf + buf_size, gfp_mask);
#2 lore raw
Quoted 8 lines · Alexey Dobriyan
Quoted 5 lines · Hal Rosenstock

Add new helper routines for allocating MADs for sending and formatting a send WR.

-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c +++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c

				   ^^^^^^^^^^^

Ick. You'd better have linux-2.6.13-rc2-mm1-[0123...].

Shall I resubmit with linux-2.6.13-rc2-mm1-[0123...] ?

Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah,
+					    int hdr_len, int data_len,
+					    int gfp_mask)

unsigned int __nocast gfp_mask, please. 430 or so infiniband sparse warnings is not a reason to add more.

I'll fix this in a subsequent patch. Is that OK ?

Thanks.

-- Hal

#3 lore raw
Quoted 11 lines · Hal Rosenstock
Quoted 8 lines · Alexey Dobriyan
Quoted 5 lines · Hal Rosenstock

Add new helper routines for allocating MADs for sending and formatting a send WR.

-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c +++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c

				   ^^^^^^^^^^^

Ick. You'd better have linux-2.6.13-rc2-mm1-[0123...].

Shall I resubmit with linux-2.6.13-rc2-mm1-[0123...] ?

Do these patches even apply with -p1 with these odd directories? I think it will try to find the addition context in a file which doesn't exist (because the directory, e.g. infiniband3, does not). I notice that every patch increments these values. I think you only want to differentiate between the kernels, not between particular directories in the kernel.

Thanks,
Nish

#4 lore raw
Quoted 8 lines · Nishanth Aravamudan
Quoted 7 lines · Hal Rosenstock
Quoted 5 lines · Alexey Dobriyan
Quoted 2 lines · Hal Rosenstock

-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c +++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c

				   ^^^^^^^^^^^

Ick. You'd better have linux-2.6.13-rc2-mm1-[0123...].

Quoted 7 lines
Quoted 1 line

Shall I resubmit with linux-2.6.13-rc2-mm1-[0123...] ?

Do these patches even apply with -p1 with these odd directories? I think it will try to find the addition context in a file which doesn't exist (because the directory, e.g. infiniband3, does not). I notice that every patch increments these values. I think you only want to differentiate between the kernels, not between particular directories in the kernel.

I will resubmit. Sorry.

-- Hal

#5 lore raw
In reply to Hal Rosenstock
Quoted 11 lines · Hal Rosenstock
Quoted 8 lines · Alexey Dobriyan
Quoted 5 lines · Hal Rosenstock

Add new helper routines for allocating MADs for sending and formatting a send WR.

-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c +++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c

				   ^^^^^^^^^^^

Ick. You'd better have linux-2.6.13-rc2-mm1-[0123...].

Shall I resubmit with linux-2.6.13-rc2-mm1-[0123...] ?

I'd wait for comments and then resubmit clean series which can be applied with patch -p1.

Quoted 10 lines
Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah,
+					    int hdr_len, int data_len,
+					    int gfp_mask)

unsigned int __nocast gfp_mask, please. 430 or so infiniband sparse warnings is not a reason to add more.

I'll fix this in a subsequent patch. Is that OK ?

If Andrew will fix patches locally, OK.

#6 lore raw
Quoted 25 lines · Alexey Dobriyan
Quoted 11 lines · Hal Rosenstock
Quoted 8 lines · Alexey Dobriyan
Quoted 5 lines · Hal Rosenstock

Add new helper routines for allocating MADs for sending and formatting a send WR.

-- linux-2.6.13-rc2-mm1/drivers/infiniband2/core/mad.c +++ linux-2.6.13-rc2-mm1/drivers/infiniband3/core/mad.c

				   ^^^^^^^^^^^

Ick. You'd better have linux-2.6.13-rc2-mm1-[0123...].

Shall I resubmit with linux-2.6.13-rc2-mm1-[0123...] ?

I'd wait for comments and then resubmit clean series which can be applied with patch -p1.

Quoted 9 lines
Quoted 8 lines

Review

Quoted 5 lines
(quoted diff) +5 -0
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah,
+					    int hdr_len, int data_len,
+					    int gfp_mask)

unsigned int __nocast gfp_mask, please. 430 or so infiniband sparse warnings is not a reason to add more.

Quoted 3 lines
Quoted 1 line

I'll fix this in a subsequent patch. Is that OK ?

If Andrew will fix patches locally, OK.

I will work this into the resubmitted patches.

-- Hal

#7 lore raw
In reply to Alexey Dobriyan
Quoted 4 lines · Alexey Dobriyan

unsigned int __nocast gfp_mask, please. 430 or so infiniband sparse warnings is not a reason to add more.

Can you please elaborate on the sparse warnings that you are seeing throughout the rest of infiniband?

Thanks,

-tduffy

#8 lore raw
Quoted 8 lines · Tom Duffy
Quoted 4 lines · Alexey Dobriyan

unsigned int __nocast gfp_mask, please. 430 or so infiniband sparse warnings is not a reason to add more.

Can you please elaborate on the sparse warnings that you are seeing throughout the rest of infiniband?

$ make allmodconfig >/dev/null
$ make C=2 CHECK="sparse -Wbitwise" drivers/infiniband/ 2>&1 | tee ../W_infiniband
	[snip]
$ grep -c "warning: " ../W_infiniband
430
#9 lore raw
Quoted 5 lines · Alexey Dobriyan
$ make allmodconfig >/dev/null
$ make C=2 CHECK="sparse -Wbitwise" drivers/infiniband/ 2>&1 | tee ../W_infiniband
	[snip]
$ grep -c "warning: " ../W_infiniband
430

These seem to be mostly coming from cpu_to_be*() and be*_to_cpu(). Is there a good rule of thumb for fixing these warnings?

Thanks,

-tduffy

#10 lore raw

Quoting r. Tom Duffy <[email protected]>:

Quoted 2 lines

These seem to be mostly coming from cpu_to_be*() and be*_to_cpu(). Is there a good rule of thumb for fixing these warnings?

Yes.
Use attributes like __be32 and friends appropriately.

Signature · 1 line
MST
#11 lore raw
Quoted 6 lines · Michael S. Tsirkin

Quoting r. Tom Duffy <[email protected]>:

Quoted 2 lines

These seem to be mostly coming from cpu_to_be*() and be*_to_cpu(). Is there a good rule of thumb for fixing these warnings?

Yes.
Use attributes like __be32 and friends appropriately.

ITYM types. ;-)

Tom, see, for example, drivers/infiniband/core/sysfs.c:
----------------------------------------------------------------------------
313 in_mad->mad_hdr.attr_id = cpu_to_be16(0x12); /* PortCounters */
----------------------------------------------------------------------------
drivers/infiniband/core/sysfs.c:313:32: warning: incorrect type in assignment (different base types)
drivers/infiniband/core/sysfs.c:313:32: expected unsigned short [unsigned] [usertype] attr_id
drivers/infiniband/core/sysfs.c:313:32: got restricted unsigned short [usertype] [force] <noident>
----------------------------------------------------------------------------
Grepping for attr_id in drivers/infiniband/ shows that:
1) in_mad->attr_id is set to IB_SMP_ATTR_NODE_INFO (network order)
2) mad->mad_hdr.attr_id is compared with IB_SMP_ATTR_PORT_INFO (network order)
3) *->mad_hdr.attr_id is set to big-endian value

All this suggests that struct ib_mad_hdr::attr_id should be __be16 instead of u16. So, if attr_id is really something big-endian (infiniband people should know), convert it. If not (unlikely) all those cpu_to_be16() and htons() are bogus.

HO-WE-VER, be careful and don't blindly shut up sparse. Its warnings exist to uncover real bugs.