Skip to contentSkip to conversation

IB/rdmavt: Finish adding core verbs support

PATCH New Dennis Dalessandro 1 participant 8 messages Feb 3, 2016 1 message not in the archive

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover IB/rdmavt: Finish adding core verbs support

Add the rest of the core verbs support to rdmavt. This includes query/modify port as well as get_port_immutable and query quid. SRQ functionality is also added as is support for memory affinity.

his patch series applies on the previously posted "staging/rdma/hfi1: Add rdmavt support to hfi1" and can be seen in context at:
https://github.com/ddalessa/kernel/tree/for-4.6

---

Dennis Dalessandro (2):
IB/rdmavt: Clean up distinction between port number and index IB/rdmavt: Add query gid support.

Harish Chegondi (1):
IB/rdmavt: Add support for query_port, modify_port and get_port_immutable

Ira Weiny (2):
IB/rdmavt: Add hardware driver send work request check IB/rdmavt: Properly pass gfp to hw driver function

Jubin John (1):
IB/rdmavt: Add srq functionality to rdmavt

Mitko Haralanov (1):
IB/rdmavt: Add Mem affinity support

 drivers/infiniband/sw/rdmavt/mad.c  |    8 +
 drivers/infiniband/sw/rdmavt/mad.h  |    2 
 drivers/infiniband/sw/rdmavt/mmap.c |    2 
 drivers/infiniband/sw/rdmavt/mr.c   |    2 
 drivers/infiniband/sw/rdmavt/qp.c   |   80 +++++++++--
 drivers/infiniband/sw/rdmavt/srq.c  |  257 ++++++++++++++++++++++++++++++++++-
 drivers/infiniband/sw/rdmavt/srq.h  |    1 
 drivers/infiniband/sw/rdmavt/vt.c   |  125 ++++++++++++-----
 drivers/infiniband/sw/rdmavt/vt.h   |   12 ++
 include/rdma/rdma_vt.h              |   17 ++
 10 files changed, 441 insertions(+), 65 deletions(-)
Signature · 5 lines
-Denny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A message that is not in the archive (sent off-list or not yet received): 20160203221136.1765.8947.stgit-9QXIwq+3FY+1XWohqUldA0EOCMrvLtNR@public.gmane.org

#2 lore raw

PATCH 1/7 IB/rdmavt: Add srq functionality to rdmavt

From: Jubin John <[email protected]>

Fill in srq function stubs with code derived from hfi1 and qib. Move necessary functions and data structure members as well.

---
 drivers/infiniband/sw/rdmavt/qp.c  |   37 +++++
 drivers/infiniband/sw/rdmavt/srq.c |  257 +++++++++++++++++++++++++++++++++++-
 drivers/infiniband/sw/rdmavt/srq.h |    1 
 drivers/infiniband/sw/rdmavt/vt.c  |    1 
 include/rdma/rdma_vt.h             |    3 
 5 files changed, 293 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rdmavt/qp.c +36 -1
index 8f85665..8efb0f5 100644
@@ -1510,7 +1510,42 @@ bail:
 int rvt_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
 		      struct ib_recv_wr **bad_wr)
 {
-	return -EOPNOTSUPP;
+	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
+	struct rvt_rwq *wq;
+	unsigned long flags;
+
+	for (; wr; wr = wr->next) {
+		struct rvt_rwqe *wqe;
+		u32 next;
+		int i;
+
+		if ((unsigned)wr->num_sge > srq->rq.max_sge) {
+			*bad_wr = wr;
+			return -EINVAL;
+		}
+
+		spin_lock_irqsave(&srq->rq.lock, flags);
+		wq = srq->rq.wq;
+		next = wq->head + 1;
+		if (next >= srq->rq.size)
+			next = 0;
+		if (next == wq->tail) {
+			spin_unlock_irqrestore(&srq->rq.lock, flags);
+			*bad_wr = wr;
+			return -ENOMEM;
+		}
+
+		wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head);
+		wqe->wr_id = wr->wr_id;
+		wqe->num_sge = wr->num_sge;
+		for (i = 0; i < wr->num_sge; i++)
+			wqe->sg_list[i] = wr->sg_list[i];
+		/* Make sure queue entry is written before the head index. */
+		smp_wmb();
+		wq->head = next;
+		spin_unlock_irqrestore(&srq->rq.lock, flags);
+	}
+	return 0;
 }
 
 void rvt_free_qpn(struct rvt_qpn_table *qpt, u32 qpn)
drivers/infiniband/sw/rdmavt/srq.c +252 -5

283 lines not on this page · raw message

drivers/infiniband/sw/rdmavt/srq.h +1 -0
index 9f07880..bf0eaaf 100644
@@ -49,6 +49,7 @@
  */
 
 #include <rdma/rdma_vt.h>
+void rvt_driver_srq_init(struct rvt_dev_info *rdi);
 struct ib_srq *rvt_create_srq(struct ib_pd *ibpd,
 			      struct ib_srq_init_attr *srq_init_attr,
 			      struct ib_udata *udata);
drivers/infiniband/sw/rdmavt/vt.c +1 -0
index 571463e..d45206c 100644
@@ -323,6 +323,7 @@ int rvt_register_device(struct rvt_dev_info *rdi)
 	CHECK_DRIVER_OVERRIDE(rdi, modify_srq);
 	CHECK_DRIVER_OVERRIDE(rdi, destroy_srq);
 	CHECK_DRIVER_OVERRIDE(rdi, query_srq);
+	rvt_driver_srq_init(rdi);
 
 	/* Multicast */
 	rvt_driver_mcast_init(rdi);
include/rdma/rdma_vt.h +3 -0
index f6569b2..1b77065 100644
@@ -299,6 +299,9 @@ struct rvt_dev_info {
 	int n_ahs_allocated;
 	spinlock_t n_ahs_lock; /* Protect ah allocated count */
 
+	u32 n_srqs_allocated;
+	spinlock_t n_srqs_lock; /* Protect srqs allocated count */
+
 	int flags;
 	struct rvt_ibport **ports;
 

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#3 lore raw

PATCH 2/7 IB/rdmavt: Add hardware driver send work request check

In reply to a message not in the archive

From: Ira Weiny <[email protected]>

Some hardware drivers requires additional checks on send WRs. Create an optional call back to allow hardware drivers to reject a send WR.

---
 drivers/infiniband/sw/rdmavt/qp.c |    4 ++++
 include/rdma/rdma_vt.h            |    2 ++
 2 files changed, 6 insertions(+), 0 deletions(-)
drivers/infiniband/sw/rdmavt/qp.c +4 -0
index 8efb0f5..34c92c0 100644
@@ -1387,6 +1387,10 @@ static int rvt_post_one_wr(struct rvt_qp *qp, struct ib_send_wr *wr)
 	if (next == qp->s_last)
 		return -ENOMEM;
 
+	if (rdi->driver_f.check_send_wr &&
+	    rdi->driver_f.check_send_wr(qp, wr))
+		return -EINVAL;
+
 	rkt = &rdi->lkey_table;
 	pd = ibpd_to_rvtpd(qp->ibqp.pd);
 	wqe = rvt_get_swqe_ptr(qp, qp->s_head);
include/rdma/rdma_vt.h +2 -0
index 1b77065..52dfa9c 100644
@@ -265,6 +265,8 @@ struct rvt_driver_provided {
 	void (*modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr,
 			  int attr_mask, struct ib_udata *udata);
 
+	int (*check_send_wr)(struct rvt_qp *qp, struct ib_send_wr *wr);
+
 	void (*notify_create_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
 	void (*notify_free_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
 };

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#4 lore raw

PATCH 3/7 IB/rdmavt: Add Mem affinity support

In reply to a message not in the archive

From: Mitko Haralanov <[email protected]>

Change verbs memory allocations to the device numa node. This keeps memory close to the device for optimal performance.

---
 drivers/infiniband/sw/rdmavt/mmap.c |    2 +-
 drivers/infiniband/sw/rdmavt/mr.c   |    2 +-
 drivers/infiniband/sw/rdmavt/qp.c   |   21 ++++++++++++---------
 3 files changed, 14 insertions(+), 11 deletions(-)
drivers/infiniband/sw/rdmavt/mmap.c +1 -1
index d6330d7..49180c4 100644
@@ -157,7 +157,7 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
 {
 	struct rvt_mmap_info *ip;
 
-	ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+	ip = kmalloc_node(sizeof(*ip), GFP_KERNEL, rdi->dparms.node);
 	if (!ip)
 		return ip;
 
drivers/infiniband/sw/rdmavt/mr.c +1 -1
index ee36be3..8bff6bb 100644
@@ -87,7 +87,7 @@ int rvt_driver_mr_init(struct rvt_dev_info *rdi)
 	}
 	lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table);
 	rdi->lkey_table.table = (struct rvt_mregion __rcu **)
-			       vmalloc(lk_tab_size);
+			       vmalloc_node(lk_tab_size, rdi->dparms.node);
 	if (!rdi->lkey_table.table)
 		return -ENOMEM;
 
drivers/infiniband/sw/rdmavt/qp.c +12 -9
index 34c92c0..a2fed05 100644
@@ -186,7 +186,8 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi)
 		return -EINVAL;
 
 	/* allocate parent object */
-	rdi->qp_dev = kzalloc(sizeof(*rdi->qp_dev), GFP_KERNEL);
+	rdi->qp_dev = kzalloc_node(sizeof(*rdi->qp_dev), GFP_KERNEL,
+				   rdi->dparms.node);
 	if (!rdi->qp_dev)
 		return -ENOMEM;
 
@@ -194,9 +195,9 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi)
 	rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size;
 	rdi->qp_dev->qp_table_bits = ilog2(rdi->dparms.qp_table_size);
 	rdi->qp_dev->qp_table =
-		kmalloc(rdi->qp_dev->qp_table_size *
-			sizeof(*rdi->qp_dev->qp_table),
-			GFP_KERNEL);
+		kmalloc_node(rdi->qp_dev->qp_table_size *
+			     sizeof(*rdi->qp_dev->qp_table),
+			     GFP_KERNEL, rdi->dparms.node);
 	if (!rdi->qp_dev->qp_table)
 		goto no_qp_table;
 
@@ -542,8 +543,9 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 				(init_attr->cap.max_send_wr + 1) * sz,
 				gfp, PAGE_KERNEL);
 		else
-			swq = vmalloc(
-				(init_attr->cap.max_send_wr + 1) * sz);
+			swq = vmalloc_node(
+				(init_attr->cap.max_send_wr + 1) * sz,
+				rdi->dparms.node);
 		if (!swq)
 			return ERR_PTR(-ENOMEM);
 
@@ -558,7 +560,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 		} else if (init_attr->cap.max_recv_sge > 1)
 			sg_list_sz = sizeof(*qp->r_sg_list) *
 				(init_attr->cap.max_recv_sge - 1);
-		qp = kzalloc(sz + sg_list_sz, gfp);
+		qp = kzalloc_node(sz + sg_list_sz, gfp, rdi->dparms.node);
 		if (!qp)
 			goto bail_swq;
 
@@ -592,9 +594,10 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd,
 						qp->r_rq.size * sz,
 						gfp, PAGE_KERNEL);
 			else
-				qp->r_rq.wq = vmalloc(
+				qp->r_rq.wq = vmalloc_node(
 						sizeof(struct rvt_rwq) +
-						qp->r_rq.size * sz);
+						qp->r_rq.size * sz,
+						rdi->dparms.node);
 			if (!qp->r_rq.wq)
 				goto bail_driver_priv;
 		}

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#5 lore raw

PATCH 4/7 IB/rdmavt: Clean up distinction between port number and index

In reply to a message not in the archive

IB core uses 1 relative indexing for ports. All of our data structures use 0 based indexing. Add an inline function that we can use whenever we need to validate a legal value and try to convert a port number to a port index at the entrance into rdmavt.

Try to follow the policy that when we are talking about a port from IB core point of view we refer to it as a port number. When port is an index into our arrays refer to it as a port index.

---
 drivers/infiniband/sw/rdmavt/mad.c |    8 ++++++--
 drivers/infiniband/sw/rdmavt/mad.h |    2 +-
 drivers/infiniband/sw/rdmavt/qp.c  |   17 +++++++++++------
 drivers/infiniband/sw/rdmavt/vt.c  |   36 ++++++++++++++++++++++--------------
 drivers/infiniband/sw/rdmavt/vt.h  |   12 ++++++++++++
 include/rdma/rdma_vt.h             |    4 ++--
 6 files changed, 54 insertions(+), 25 deletions(-)
drivers/infiniband/sw/rdmavt/mad.c +6 -2
index 5c720d3..2feae47 100644
@@ -47,12 +47,13 @@
 
 #include <rdma/ib_mad.h>
 #include "mad.h"
+#include "vt.h"
 
 /**
  * rvt_process_mad - process an incoming MAD packet
  * @ibdev: the infiniband device this packet came in on
  * @mad_flags: MAD flags
- * @port: the port number this packet came in on
+ * @port_num: the port number this packet came in on, 1 based from ib core
  * @in_wc: the work completion entry for this packet
  * @in_grh: the global route header for this packet
  * @in_mad: the incoming MAD
@@ -67,7 +68,7 @@
  *
  * This is called by the ib_mad module.
  */
-int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,
+int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
 		    const struct ib_mad_hdr *in, size_t in_mad_size,
 		    struct ib_mad_hdr *out, size_t *out_mad_size,
@@ -82,6 +83,9 @@ int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,
 	 *VT-DRIVER-API: ????
 	 *
 	 */
+	if (ibport_num_to_idx(ibdev, port_num) < 0)
+		return -EINVAL;
+
 	return IB_MAD_RESULT_FAILURE;
 }
 
drivers/infiniband/sw/rdmavt/mad.h +1 -1
index c89faf4..a9d6eec 100644
@@ -50,7 +50,7 @@
 
 #include <rdma/rdma_vt.h>
 
-int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,
+int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
 		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
 		    const struct ib_mad_hdr *in, size_t in_mad_size,
 		    struct ib_mad_hdr *out, size_t *out_mad_size,
drivers/infiniband/sw/rdmavt/qp.c +11 -6
index a2fed05..4eab466 100644
@@ -286,26 +286,31 @@ static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,
 	return (map - qpt->map) * RVT_BITS_PER_PAGE + off;
 }
 
-/*
- * Allocate the next available QPN or
- * zero/one for QP type IB_QPT_SMI/IB_QPT_GSI.
+/**
+ * alloc_qpn - Allocate the next available qpn or zero/one for QP type
+ *	       IB_QPT_SMI/IB_QPT_GSI
+ *@rdi:	rvt device info structure
+ *@qpt: queue pair number table pointer
+ *@port_num: IB port number, 1 based, comes from core
+ *
+ * Return: The queue pair number
  */
 static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
-		     enum ib_qp_type type, u8 port, gfp_t gfp)
+		     enum ib_qp_type type, u8 port_num, gfp_t gfp)
 {
 	u32 i, offset, max_scan, qpn;
 	struct rvt_qpn_map *map;
 	u32 ret;
 
 	if (rdi->driver_f.alloc_qpn)
-		return rdi->driver_f.alloc_qpn(rdi, qpt, type, port,
+		return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num,
 					       GFP_KERNEL);
 
 	if (type == IB_QPT_SMI || type == IB_QPT_GSI) {
 		unsigned n;
 
 		ret = type == IB_QPT_GSI;
-		n = 1 << (ret + 2 * (port - 1));
+		n = 1 << (ret + 2 * (port_num - 1));
 		spin_lock(&qpt->lock);
 		if (qpt->flags & n)
 			ret = -EINVAL;
drivers/infiniband/sw/rdmavt/vt.c +22 -14
index d45206c..9f9cb9a 100644
@@ -120,14 +120,17 @@ static int rvt_modify_device(struct ib_device *device,
 /**
  * rvt_query_port: Passes the query port call to the driver
  * @ibdev: Verbs IB dev
- * @port: port number
+ * @port_num: port number, 1 based from ib core
  * @props: structure to hold returned properties
  *
  * Returns 0 on success
  */
-static int rvt_query_port(struct ib_device *ibdev, u8 port,
+static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
 			  struct ib_port_attr *props)
 {
+	if (ibport_num_to_idx(ibdev, port_num) < 0)
+		return -EINVAL;
+
 	/*
 	 * VT-DRIVER-API: query_port_state()
 	 * driver returns pretty much everything in ib_port_attr
@@ -138,13 +141,13 @@ static int rvt_query_port(struct ib_device *ibdev, u8 port,
 /**
  * rvt_modify_port
  * @ibdev: Verbs IB dev
- * @port: Port number
+ * @port_num: Port number, 1 based from ib core
  * @port_modify_mask: How to change the port
  * @props: Structure to fill in
  *
  * Returns 0 on success
  */
-static int rvt_modify_port(struct ib_device *ibdev, u8 port,
+static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
 			   int port_modify_mask, struct ib_port_modify *props)
 {
 	/*
@@ -160,18 +163,21 @@ static int rvt_modify_port(struct ib_device *ibdev, u8 port,
 	 * TBD: send_trap() and post_mad_send() need examined to see where they
 	 * fit in.
 	 */
+	if (ibport_num_to_idx(ibdev, port_num) < 0)
+		return -EINVAL;
+
 	return -EOPNOTSUPP;
 }
 
 /**
  * rvt_query_pkey - Return a pkey from the table at a given index
  * @ibdev: Verbs IB dev
- * @port: Port number
+ * @port_num: Port number, 1 based from ib core
  * @intex: Index into pkey table
  *
  * Returns 0 on failure pkey otherwise
  */
-static int rvt_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
+static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
 			  u16 *pkey)
 {
 	/*
@@ -183,11 +189,11 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
 	int port_index;
 
-	if (index >= rvt_get_npkeys(rdi))
+	port_index = ibport_num_to_idx(ibdev, port_num);
+	if (port_index < 0)
 		return -EINVAL;
 
-	port_index = port - 1; /* IB ports start at 1 our array at 0 */
-	if ((port_index < 0) || (port_index >= rdi->dparms.nports))
+	if (index >= rvt_get_npkeys(rdi))
 		return -EINVAL;
 
 	*pkey = rvt_get_pkey(rdi, port_index, index);
@@ -197,13 +203,13 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 /**
  * rvt_query_gid - Return a gid from the table
  * @ibdev: Verbs IB dev
- * @port: Port number
+ * @port_num: Port number, 1 based from ib core
  * @index: = Index in table
  * @gid: Gid to return
  *
  * Returns 0 on success
  */
-static int rvt_query_gid(struct ib_device *ibdev, u8 port,
+static int rvt_query_gid(struct ib_device *ibdev, u8 port_num,
 			 int index, union ib_gid *gid)
 {
 	/*
@@ -211,6 +217,8 @@ static int rvt_query_gid(struct ib_device *ibdev, u8 port,
 	 * to craft the return value. This will work similar to how query_pkey()
 	 * is being done.
 	 */
+	if (ibport_num_to_idx(ibdev, port_num) < 0)
+		return -EINVAL;
 
 	return -EOPNOTSUPP;
 }
@@ -455,11 +463,11 @@ EXPORT_SYMBOL(rvt_unregister_device);
  * They persist until the driver goes away.
  */
 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
-		  int portnum, u16 *pkey_table)
+		  int port_index, u16 *pkey_table)
 {
 
-	rdi->ports[portnum] = port;
-	rdi->ports[portnum]->pkey_table = pkey_table;
+	rdi->ports[port_index] = port;
+	rdi->ports[port_index]->pkey_table = pkey_table;
 
 	return 0;
 }
drivers/infiniband/sw/rdmavt/vt.h +12 -0
index a5c36d3..e26f9e9 100644
@@ -88,4 +88,16 @@
 #define __rvt_pr_err(pdev, name, fmt, ...) \
 	dev_err(&pdev->dev, "%s: " fmt, name, ##__VA_ARGS__)
 
+static inline int ibport_num_to_idx(struct ib_device *ibdev, u8 port_num)
+{
+	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
+	int port_index;
+
+	port_index = port_num - 1; /* IB ports start at 1 our arrays at 0 */
+	if ((port_index < 0) || (port_index >= rdi->dparms.nports))
+		return -EINVAL;
+
+	return port_index;
+}
+
 #endif          /* DEF_RDMAVT_H */
include/rdma/rdma_vt.h +2 -2
index 52dfa9c..5d1c694 100644
@@ -256,7 +256,7 @@ struct rvt_driver_provided {
 	void (*notify_new_ah)(struct ib_device *, struct ib_ah_attr *,
 			      struct rvt_ah *);
 	int (*alloc_qpn)(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
-			 enum ib_qp_type type, u8 port, gfp_t gfp);
+			 enum ib_qp_type type, u8 port_num, gfp_t gfp);
 	/**
 	 * Return 0 if modification is valid, -errno otherwise
 	 */
@@ -408,7 +408,7 @@ int rvt_register_device(struct rvt_dev_info *rvd);
 void rvt_unregister_device(struct rvt_dev_info *rvd);
 int rvt_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr);
 int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
-		  int portnum, u16 *pkey_table);
+		  int port_index, u16 *pkey_table);
 int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
 		u32 len, u64 vaddr, u32 rkey, int acc);
 int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#6 lore raw

PATCH 5/7 IB/rdmavt: Add query gid support.

In reply to a message not in the archive

Addin query gid support. Rdmavt still relies on the driver to maintain the gid table. Rdmavt simply calls into the driver to retrive the guid for a particular port.

---
 drivers/infiniband/sw/rdmavt/vt.c |   17 ++++++++++++++---
 include/rdma/rdma_vt.h            |    2 ++
 2 files changed, 16 insertions(+), 3 deletions(-)
drivers/infiniband/sw/rdmavt/vt.c +14 -3
index 9f9cb9a..e017117 100644
@@ -210,17 +210,28 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,
  * Returns 0 on success
  */
 static int rvt_query_gid(struct ib_device *ibdev, u8 port_num,
-			 int index, union ib_gid *gid)
+			 int guid_index, union ib_gid *gid)
 {
+	struct rvt_dev_info *rdi;
+	struct rvt_ibport *rvp;
+	int port_index;
+
 	/*
 	 * Driver is responsible for updating the guid table. Which will be used
 	 * to craft the return value. This will work similar to how query_pkey()
 	 * is being done.
 	 */
-	if (ibport_num_to_idx(ibdev, port_num) < 0)
+	port_index = ibport_num_to_idx(ibdev, port_num);
+	if (port_index < 0)
 		return -EINVAL;
 
-	return -EOPNOTSUPP;
+	rdi = ib_to_rvt(ibdev);
+	rvp = rdi->ports[port_index];
+
+	gid->global.subnet_prefix = rvp->gid_prefix;
+
+	return rdi->driver_f.get_guid_be(rdi, rvp, guid_index,
+					 &gid->global.interface_id);
 }
 
 struct rvt_ucontext {
include/rdma/rdma_vt.h +2 -0
index 5d1c694..dabf4d5 100644
@@ -248,6 +248,8 @@ struct rvt_driver_provided {
 	u32 (*mtu_from_qp)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
 			   u32 pmtu);
 	int (*mtu_to_path_mtu)(u32 mtu);
+	int (*get_guid_be)(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,
+			   int guid_index, __be64 *guid);
 
 	/*--------------------*/
 	/* Optional functions */

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#7 lore raw

PATCH 6/7 IB/rdmavt: Add support for query_port, modify_port and get_port_immutable

In reply to a message not in the archive

From: Harish Chegondi <[email protected]>

rvt_query_port calls into the driver through a call back function query_port_state to populate the rest of ib_port_attr elements. rvt_modify_port calls into the driver if needed through a call back function shut_down_port()

---
 drivers/infiniband/sw/rdmavt/vt.c |   77 ++++++++++++++++++++++++++-----------
 include/rdma/rdma_vt.h            |    6 +++
 2 files changed, 61 insertions(+), 22 deletions(-)
drivers/infiniband/sw/rdmavt/vt.c +55 -22
index e017117..2ccf610 100644
@@ -128,14 +128,27 @@ static int rvt_modify_device(struct ib_device *device,
 static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
 			  struct ib_port_attr *props)
 {
-	if (ibport_num_to_idx(ibdev, port_num) < 0)
+	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
+	struct rvt_ibport *rvp;
+	int port_index = ibport_num_to_idx(ibdev, port_num);
+
+	if (port_index < 0)
 		return -EINVAL;
 
-	/*
-	 * VT-DRIVER-API: query_port_state()
-	 * driver returns pretty much everything in ib_port_attr
-	 */
-	return -EOPNOTSUPP;
+	rvp = rdi->ports[port_index];
+	memset(props, 0, sizeof(*props));
+	props->sm_lid = rvp->sm_lid;
+	props->sm_sl = rvp->sm_sl;
+	props->port_cap_flags = rvp->port_cap_flags;
+	props->max_msg_sz = 0x80000000;
+	props->pkey_tbl_len = rvt_get_npkeys(rdi);
+	props->bad_pkey_cntr = rvp->pkey_violations;
+	props->qkey_viol_cntr = rvp->qkey_violations;
+	props->subnet_timeout = rvp->subnet_timeout;
+	props->init_type_reply = 0;
+
+	/* Populate the remaining ib_port_attr elements */
+	return rdi->driver_f.query_port_state(rdi, port_num, props);
 }
 
 /**
@@ -150,23 +163,26 @@ static int rvt_query_port(struct ib_device *ibdev, u8 port_num,
 static int rvt_modify_port(struct ib_device *ibdev, u8 port_num,
 			   int port_modify_mask, struct ib_port_modify *props)
 {
-	/*
-	 * VT-DRIVER-API: set_link_state()
-	 * driver will set the link state using the IB enumeration
-	 *
-	 * VT-DRIVER-API: clear_qkey_violations()
-	 * clears driver private qkey counter
-	 *
-	 * VT-DRIVER-API: get_lid()
-	 * driver needs to return the LID
-	 *
-	 * TBD: send_trap() and post_mad_send() need examined to see where they
-	 * fit in.
-	 */
-	if (ibport_num_to_idx(ibdev, port_num) < 0)
+	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
+	struct rvt_ibport *rvp;
+	int ret = 0;
+	int port_index = ibport_num_to_idx(ibdev, port_num);
+
+	if (port_index < 0)
 		return -EINVAL;
 
-	return -EOPNOTSUPP;
+	rvp = rdi->ports[port_index];
+	rvp->port_cap_flags |= props->set_port_cap_mask;
+	rvp->port_cap_flags &= ~props->clr_port_cap_mask;
+
+	if (props->set_port_cap_mask || props->clr_port_cap_mask)
+		rdi->driver_f.cap_mask_chg(rdi, port_num);
+	if (port_modify_mask & IB_PORT_SHUTDOWN)
+		ret = rdi->driver_f.shut_down_port(rdi, port_num);
+	if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
+		rvp->qkey_violations = 0;
+
+	return ret;
 }
 
 /**
@@ -273,7 +289,24 @@ static int rvt_dealloc_ucontext(struct ib_ucontext *context)
 static int rvt_get_port_immutable(struct ib_device *ibdev, u8 port_num,
 				  struct ib_port_immutable *immutable)
 {
-	return -EOPNOTSUPP;
+	struct rvt_dev_info *rdi = ib_to_rvt(ibdev);
+	struct ib_port_attr attr;
+	int err, port_index;
+
+	port_index = ibport_num_to_idx(ibdev, port_num);
+	if (port_index < 0)
+		return -EINVAL;
+
+	err = rvt_query_port(ibdev, port_num, &attr);
+	if (err)
+		return err;
+
+	immutable->pkey_tbl_len = attr.pkey_tbl_len;
+	immutable->gid_tbl_len = attr.gid_tbl_len;
+	immutable->core_cap_flags = rdi->dparms.core_cap_flags;
+	immutable->max_mad_size = rdi->dparms.max_mad_size;
+
+	return 0;
 }
 
 /*
include/rdma/rdma_vt.h +6 -0
index dabf4d5..4242fea 100644
@@ -200,6 +200,8 @@ struct rvt_driver_params {
 	int psn_mask;
 	int psn_shift;
 	int psn_modify_mask;
+	u32 core_cap_flags;
+	u32 max_mad_size;
 };
 
 /* Protection domain */
@@ -250,6 +252,10 @@ struct rvt_driver_provided {
 	int (*mtu_to_path_mtu)(u32 mtu);
 	int (*get_guid_be)(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,
 			   int guid_index, __be64 *guid);
+	int (*query_port_state)(struct rvt_dev_info *rdi, u8 port_num,
+				struct ib_port_attr *props);
+	int (*shut_down_port)(struct rvt_dev_info *rdi, u8 port_num);
+	void (*cap_mask_chg)(struct rvt_dev_info *rdi, u8 port_num);
 
 	/*--------------------*/
 	/* Optional functions */

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html

#8 lore raw

PATCH 7/7 IB/rdmavt: Properly pass gfp to hw driver function

In reply to a message not in the archive

From: Ira Weiny <[email protected]>

alloc_qpn must use GFP and the hardware drivers should use it as well.

---
 drivers/infiniband/sw/rdmavt/qp.c |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)
drivers/infiniband/sw/rdmavt/qp.c +1 -2
index 4eab466..322de64 100644
@@ -303,8 +303,7 @@ static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
 	u32 ret;
 
 	if (rdi->driver_f.alloc_qpn)
-		return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num,
-					       GFP_KERNEL);
+		return rdi->driver_f.alloc_qpn(rdi, qpt, type, port_num, gfp);
 
 	if (type == IB_QPT_SMI || type == IB_QPT_GSI) {
 		unsigned n;

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html