IB/rdmavt: Finish adding core verbs support
Subsystems (2)
cover IB/rdmavt: Finish adding core verbs support
Add the rest of the core verbs support to rdmavt. This includes query/modify port as well as get_port_immutable and query quid. SRQ functionality is also added as is support for memory affinity.
his patch series applies on the previously posted "staging/rdma/hfi1: Add rdmavt support to hfi1" and can be seen in context at:
https://github.com/ddalessa/kernel/tree/for-4.6
---
Dennis Dalessandro (2):
IB/rdmavt: Clean up distinction between port number and index IB/rdmavt: Add query gid support.
Harish Chegondi (1):
IB/rdmavt: Add support for query_port, modify_port and get_port_immutable
Ira Weiny (2):
IB/rdmavt: Add hardware driver send work request check IB/rdmavt: Properly pass gfp to hw driver function
Jubin John (1):
IB/rdmavt: Add srq functionality to rdmavt
Mitko Haralanov (1):
IB/rdmavt: Add Mem affinity support
drivers/infiniband/sw/rdmavt/mad.c | 8 + drivers/infiniband/sw/rdmavt/mad.h | 2 drivers/infiniband/sw/rdmavt/mmap.c | 2 drivers/infiniband/sw/rdmavt/mr.c | 2 drivers/infiniband/sw/rdmavt/qp.c | 80 +++++++++-- drivers/infiniband/sw/rdmavt/srq.c | 257 ++++++++++++++++++++++++++++++++++- drivers/infiniband/sw/rdmavt/srq.h | 1 drivers/infiniband/sw/rdmavt/vt.c | 125 ++++++++++++----- drivers/infiniband/sw/rdmavt/vt.h | 12 ++ include/rdma/rdma_vt.h | 17 ++ 10 files changed, 441 insertions(+), 65 deletions(-)
Signature · 5 lines
-Denny -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
A message that is not in the archive (sent off-list or not yet received): 20160203221136.1765.8947.stgit-9QXIwq+3FY+1XWohqUldA0EOCMrvLtNR@public.gmane.org
PATCH 1/7 IB/rdmavt: Add srq functionality to rdmavt
From: Jubin John <[email protected]>
Fill in srq function stubs with code derived from hfi1 and qib. Move necessary functions and data structure members as well.
--- drivers/infiniband/sw/rdmavt/qp.c | 37 +++++ drivers/infiniband/sw/rdmavt/srq.c | 257 +++++++++++++++++++++++++++++++++++- drivers/infiniband/sw/rdmavt/srq.h | 1 drivers/infiniband/sw/rdmavt/vt.c | 1 include/rdma/rdma_vt.h | 3 5 files changed, 293 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rdmavt/qp.c +36 -1
@@ -1510,7 +1510,42 @@ bail:int rvt_post_srq_recv (struct ib_srq *ibsrq,struct ib_recv_wr *wr,struct ib_recv_wr **bad_wr) { -return -EOPNOTSUPP ; +struct rvt_srq *srq =ibsrq_to_rvtsrq (ibsrq); +struct rvt_rwq *wq; +unsigned long flags; + +for (; wr; wr = wr->next) { +struct rvt_rwqe *wqe; +u32 next; +int i; + +if ((unsigned )wr->num_sge > srq->rq.max_sge) { + *bad_wr = wr; +return -EINVAL ; + } + +spin_lock_irqsave (&srq->rq.lock, flags); + wq = srq->rq.wq; + next = wq->head +1 ; +if (next >= srq->rq.size) + next =0 ; +if (next == wq->tail) { +spin_unlock_irqrestore (&srq->rq.lock, flags); + *bad_wr = wr; +return -ENOMEM ; + } + + wqe =rvt_get_rwqe_ptr (&srq->rq, wq->head); + wqe->wr_id = wr->wr_id; + wqe->num_sge = wr->num_sge; +for (i =0 ; i < wr->num_sge; i++) + wqe->sg_list[i] = wr->sg_list[i]; +/* Make sure queue entry is written before the head index. */ +smp_wmb (); + wq->head = next; +spin_unlock_irqrestore (&srq->rq.lock, flags); + } +return 0 ; }void rvt_free_qpn (struct rvt_qpn_table *qpt,u32 qpn)
drivers/infiniband/sw/rdmavt/srq.c +252 -5
283 lines not on this page · raw message
drivers/infiniband/sw/rdmavt/srq.h +1 -0
@@ -49,6 +49,7 @@*/ #include <rdma/rdma_vt.h> +void rvt_driver_srq_init (struct rvt_dev_info *rdi);struct ib_srq *rvt_create_srq (struct ib_pd *ibpd,struct ib_srq_init_attr *srq_init_attr,struct ib_udata *udata);
drivers/infiniband/sw/rdmavt/vt.c +1 -0
@@ -323,6 +323,7 @@ int rvt_register_device(struct rvt_dev_info *rdi)CHECK_DRIVER_OVERRIDE (rdi, modify_srq);CHECK_DRIVER_OVERRIDE (rdi, destroy_srq);CHECK_DRIVER_OVERRIDE (rdi, query_srq); +rvt_driver_srq_init (rdi);/* Multicast */ rvt_driver_mcast_init (rdi);
include/rdma/rdma_vt.h +3 -0
@@ -299,6 +299,9 @@ struct rvt_dev_info {int n_ahs_allocated;spinlock_t n_ahs_lock;/* Protect ah allocated count */ +u32 n_srqs_allocated; +spinlock_t n_srqs_lock;/* Protect srqs allocated count */ +int flags;struct rvt_ibport **ports;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 2/7 IB/rdmavt: Add hardware driver send work request check
In reply to a message not in the archiveFrom: Ira Weiny <[email protected]>
Some hardware drivers requires additional checks on send WRs. Create an optional call back to allow hardware drivers to reject a send WR.
--- drivers/infiniband/sw/rdmavt/qp.c | 4 ++++ include/rdma/rdma_vt.h | 2 ++ 2 files changed, 6 insertions(+), 0 deletions(-)
drivers/infiniband/sw/rdmavt/qp.c +4 -0
@@ -1387,6 +1387,10 @@ static int rvt_post_one_wr(struct rvt_qp *qp, struct ib_send_wr *wr)if (next == qp->s_last)return -ENOMEM ; +if (rdi->driver_f.check_send_wr && + rdi->driver_f.check_send_wr (qp, wr)) +return -EINVAL ; + rkt = &rdi->lkey_table; pd =ibpd_to_rvtpd (qp->ibqp.pd); wqe =rvt_get_swqe_ptr (qp, qp->s_head);
include/rdma/rdma_vt.h +2 -0
@@ -265,6 +265,8 @@ struct rvt_driver_provided {void (*modify_qp)(struct rvt_qp *qp,struct ib_qp_attr *attr,int attr_mask,struct ib_udata *udata); +int (*check_send_wr)(struct rvt_qp *qp,struct ib_send_wr *wr); +void (*notify_create_mad_agent)(struct rvt_dev_info *rdi,int port_idx);void (*notify_free_mad_agent)(struct rvt_dev_info *rdi,int port_idx); };
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 3/7 IB/rdmavt: Add Mem affinity support
In reply to a message not in the archiveFrom: Mitko Haralanov <[email protected]>
Change verbs memory allocations to the device numa node. This keeps memory close to the device for optimal performance.
--- drivers/infiniband/sw/rdmavt/mmap.c | 2 +- drivers/infiniband/sw/rdmavt/mr.c | 2 +- drivers/infiniband/sw/rdmavt/qp.c | 21 ++++++++++++--------- 3 files changed, 14 insertions(+), 11 deletions(-)
drivers/infiniband/sw/rdmavt/mmap.c +1 -1
@@ -157,7 +157,7 @@ struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi, {struct rvt_mmap_info *ip; - ip =kmalloc (sizeof (*ip),GFP_KERNEL ); + ip =kmalloc_node (sizeof (*ip),GFP_KERNEL , rdi->dparms.node);if (!ip)return ip;
drivers/infiniband/sw/rdmavt/mr.c +1 -1
@@ -87,7 +87,7 @@ int rvt_driver_mr_init(struct rvt_dev_info *rdi) } lk_tab_size = rdi->lkey_table.max *sizeof (*rdi->lkey_table.table); rdi->lkey_table.table = (struct rvt_mregion __rcu **) -vmalloc (lk_tab_size); +vmalloc_node (lk_tab_size, rdi->dparms.node);if (!rdi->lkey_table.table)return -ENOMEM ;
drivers/infiniband/sw/rdmavt/qp.c +12 -9
@@ -186,7 +186,8 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi)return -EINVAL ;/* allocate parent object */ - rdi->qp_dev =kzalloc (sizeof (*rdi->qp_dev),GFP_KERNEL ); + rdi->qp_dev =kzalloc_node (sizeof (*rdi->qp_dev),GFP_KERNEL , + rdi->dparms.node);if (!rdi->qp_dev)return -ENOMEM ;
@@ -194,9 +195,9 @@ int rvt_driver_qp_init(struct rvt_dev_info *rdi) rdi->qp_dev->qp_table_size = rdi->dparms.qp_table_size; rdi->qp_dev->qp_table_bits =ilog2 (rdi->dparms.qp_table_size); rdi->qp_dev->qp_table = -kmalloc (rdi->qp_dev->qp_table_size * -sizeof (*rdi->qp_dev->qp_table), -GFP_KERNEL ); +kmalloc_node (rdi->qp_dev->qp_table_size * +sizeof (*rdi->qp_dev->qp_table), +GFP_KERNEL , rdi->dparms.node);if (!rdi->qp_dev->qp_table) goto no_qp_table;
@@ -542,8 +543,9 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, (init_attr->cap.max_send_wr +1 ) * sz, gfp, PAGE_KERNEL); else - swq =vmalloc ( - (init_attr->cap.max_send_wr +1 ) * sz); + swq =vmalloc_node ( + (init_attr->cap.max_send_wr +1 ) * sz, + rdi->dparms.node);if (!swq)return ERR_PTR (-ENOMEM );
@@ -558,7 +560,7 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, }else if (init_attr->cap.max_recv_sge >1 ) sg_list_sz =sizeof (*qp->r_sg_list) * (init_attr->cap.max_recv_sge -1 ); - qp =kzalloc (sz + sg_list_sz, gfp); + qp =kzalloc_node (sz + sg_list_sz, gfp, rdi->dparms.node);if (!qp) goto bail_swq;
@@ -592,9 +594,10 @@ struct ib_qp *rvt_create_qp(struct ib_pd *ibpd, qp->r_rq.size * sz, gfp, PAGE_KERNEL); else - qp->r_rq.wq = vmalloc( + qp->r_rq.wq = vmalloc_node(sizeof (struct rvt_rwq ) + - qp->r_rq.size * sz); + qp->r_rq.size * sz, + rdi->dparms.node);if (!qp->r_rq.wq) goto bail_driver_priv; }
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 4/7 IB/rdmavt: Clean up distinction between port number and index
In reply to a message not in the archiveIB core uses 1 relative indexing for ports. All of our data structures use 0 based indexing. Add an inline function that we can use whenever we need to validate a legal value and try to convert a port number to a port index at the entrance into rdmavt.
Try to follow the policy that when we are talking about a port from IB core point of view we refer to it as a port number. When port is an index into our arrays refer to it as a port index.
--- drivers/infiniband/sw/rdmavt/mad.c | 8 ++++++-- drivers/infiniband/sw/rdmavt/mad.h | 2 +- drivers/infiniband/sw/rdmavt/qp.c | 17 +++++++++++------ drivers/infiniband/sw/rdmavt/vt.c | 36 ++++++++++++++++++++++-------------- drivers/infiniband/sw/rdmavt/vt.h | 12 ++++++++++++ include/rdma/rdma_vt.h | 4 ++-- 6 files changed, 54 insertions(+), 25 deletions(-)
drivers/infiniband/sw/rdmavt/mad.c +6 -2
@@ -47,12 +47,13 @@#include <rdma/ib_mad.h> #include "mad.h" +#include "vt.h" /** * rvt_process_mad - process an incoming MAD packet * @ibdev: the infiniband device this packet came in on * @mad_flags: MAD flags -* @port: the port number this packet came in on +* @port_num: the port number this packet came in on, 1 based from ib core * @in_wc: the work completion entry for this packet * @in_grh: the global route header for this packet * @in_mad: the incoming MAD
@@ -67,7 +68,7 @@* * This is called by the ib_mad module. */ -int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, +int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad_hdr *in, size_t in_mad_size, struct ib_mad_hdr *out, size_t *out_mad_size,
@@ -82,6 +83,9 @@ int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port,*VT-DRIVER-API: ???? * */ +if (ibport_num_to_idx (ibdev, port_num) <0 ) +return -EINVAL ; +return IB_MAD_RESULT_FAILURE ; }
drivers/infiniband/sw/rdmavt/mad.h +1 -1
@@ -50,7 +50,7 @@#include <rdma/rdma_vt.h> -int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, +int rvt_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad_hdr *in, size_t in_mad_size, struct ib_mad_hdr *out, size_t *out_mad_size,
drivers/infiniband/sw/rdmavt/qp.c +11 -6
@@ -286,26 +286,31 @@ static inline unsigned mk_qpn(struct rvt_qpn_table *qpt,return (map - qpt->map) *RVT_BITS_PER_PAGE + off; } -/* -* Allocate the next available QPN or -* zero/one for QP type IB_QPT_SMI/IB_QPT_GSI. +/** +* alloc_qpn - Allocate the next available qpn or zero/one for QP type +* IB_QPT_SMI/IB_QPT_GSI +*@rdi: rvt device info structure +*@qpt: queue pair number table pointer +*@port_num: IB port number, 1 based, comes from core +* +* Return: The queue pair number */ static int alloc_qpn (struct rvt_dev_info *rdi,struct rvt_qpn_table *qpt, -enum ib_qp_type type,u8 port,gfp_t gfp) +enum ib_qp_type type,u8 port_num,gfp_t gfp) {u32 i, offset, max_scan, qpn;struct rvt_qpn_map *map;u32 ret;if (rdi->driver_f.alloc_qpn) -return rdi->driver_f.alloc_qpn (rdi, qpt, type, port, +return rdi->driver_f.alloc_qpn (rdi, qpt, type, port_num,GFP_KERNEL );if (type ==IB_QPT_SMI || type ==IB_QPT_GSI ) {unsigned n; ret = type ==IB_QPT_GSI ; - n =1 << (ret +2 * (port -1 )); + n =1 << (ret +2 * (port_num -1 ));spin_lock (&qpt->lock);if (qpt->flags & n) ret = -EINVAL ;
drivers/infiniband/sw/rdmavt/vt.c +22 -14
@@ -120,14 +120,17 @@ static int rvt_modify_device(struct ib_device *device,/** * rvt_query_port: Passes the query port call to the driver * @ibdev: Verbs IB dev -* @port: port number +* @port_num: port number, 1 based from ib core * @props: structure to hold returned properties * * Returns 0 on success */ -static int rvt_query_port (struct ib_device *ibdev,u8 port, +static int rvt_query_port (struct ib_device *ibdev,u8 port_num,struct ib_port_attr *props) { +if (ibport_num_to_idx (ibdev, port_num) <0 ) +return -EINVAL ; +/* * VT-DRIVER-API: query_port_state() * driver returns pretty much everything in ib_port_attr
@@ -138,13 +141,13 @@ static int rvt_query_port(struct ib_device *ibdev, u8 port,/** * rvt_modify_port * @ibdev: Verbs IB dev -* @port: Port number +* @port_num: Port number, 1 based from ib core * @port_modify_mask: How to change the port * @props: Structure to fill in * * Returns 0 on success */ -static int rvt_modify_port (struct ib_device *ibdev,u8 port, +static int rvt_modify_port (struct ib_device *ibdev,u8 port_num,int port_modify_mask,struct ib_port_modify *props) {/*
@@ -160,18 +163,21 @@ static int rvt_modify_port(struct ib_device *ibdev, u8 port,* TBD: send_trap() and post_mad_send() need examined to see where they * fit in. */ +if (ibport_num_to_idx (ibdev, port_num) <0 ) +return -EINVAL ; +return -EOPNOTSUPP ; }/** * rvt_query_pkey - Return a pkey from the table at a given index * @ibdev: Verbs IB dev -* @port: Port number +* @port_num: Port number, 1 based from ib core * @intex: Index into pkey table * * Returns 0 on failure pkey otherwise */ -static int rvt_query_pkey (struct ib_device *ibdev,u8 port,u16 index, +static int rvt_query_pkey (struct ib_device *ibdev,u8 port_num,u16 index,u16 *pkey) {/*
@@ -183,11 +189,11 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port, u16 index,struct rvt_dev_info *rdi =ib_to_rvt (ibdev);int port_index; -if (index >=rvt_get_npkeys (rdi)) + port_index =ibport_num_to_idx (ibdev, port_num); +if (port_index <0 )return -EINVAL ; - port_index = port -1 ;/* IB ports start at 1 our array at 0 */ -if ((port_index <0 ) || (port_index >= rdi->dparms.nports)) +if (index >=rvt_get_npkeys (rdi))return -EINVAL ; *pkey =rvt_get_pkey (rdi, port_index, index);
@@ -197,13 +203,13 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port, u16 index,/** * rvt_query_gid - Return a gid from the table * @ibdev: Verbs IB dev -* @port: Port number +* @port_num: Port number, 1 based from ib core * @index: = Index in table * @gid: Gid to return * * Returns 0 on success */ -static int rvt_query_gid (struct ib_device *ibdev,u8 port, +static int rvt_query_gid (struct ib_device *ibdev,u8 port_num,int index,union ib_gid *gid) {/*
@@ -211,6 +217,8 @@ static int rvt_query_gid(struct ib_device *ibdev, u8 port,* to craft the return value. This will work similar to how query_pkey() * is being done. */ +if (ibport_num_to_idx (ibdev, port_num) <0 ) +return -EINVAL ;return -EOPNOTSUPP ; }
@@ -455,11 +463,11 @@ EXPORT_SYMBOL(rvt_unregister_device);* They persist until the driver goes away. */ int rvt_init_port (struct rvt_dev_info *rdi,struct rvt_ibport *port, -int portnum,u16 *pkey_table) +int port_index,u16 *pkey_table) { - rdi->ports[portnum] = port; - rdi->ports[portnum]->pkey_table = pkey_table; + rdi->ports[port_index] = port; + rdi->ports[port_index]->pkey_table = pkey_table;return 0 ; }
drivers/infiniband/sw/rdmavt/vt.h +12 -0
@@ -88,4 +88,16 @@#define __rvt_pr_err (pdev, name, fmt, ...) \ dev_err(&pdev->dev, "%s: " fmt, name, ##__VA_ARGS__) +static inline int ibport_num_to_idx (struct ib_device *ibdev,u8 port_num) +{ +struct rvt_dev_info *rdi =ib_to_rvt (ibdev); +int port_index; + + port_index = port_num -1 ;/* IB ports start at 1 our arrays at 0 */ +if ((port_index <0 ) || (port_index >= rdi->dparms.nports)) +return -EINVAL ; + +return port_index; +} +#endif /* DEF_RDMAVT_H */
include/rdma/rdma_vt.h +2 -2
@@ -256,7 +256,7 @@ struct rvt_driver_provided {void (*notify_new_ah)(struct ib_device *,struct ib_ah_attr *,struct rvt_ah *);int (*alloc_qpn)(struct rvt_dev_info *rdi,struct rvt_qpn_table *qpt, -enum ib_qp_type type,u8 port,gfp_t gfp); +enum ib_qp_type type,u8 port_num,gfp_t gfp);/** * Return 0 if modification is valid, -errno otherwise */
@@ -408,7 +408,7 @@ int rvt_register_device(struct rvt_dev_info *rvd);void rvt_unregister_device (struct rvt_dev_info *rvd);int rvt_check_ah (struct ib_device *ibdev,struct ib_ah_attr *ah_attr);int rvt_init_port (struct rvt_dev_info *rdi,struct rvt_ibport *port, -int portnum,u16 *pkey_table); +int port_index,u16 *pkey_table);int rvt_rkey_ok (struct rvt_qp *qp,struct rvt_sge *sge,u32 len,u64 vaddr,u32 rkey,int acc); int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 5/7 IB/rdmavt: Add query gid support.
In reply to a message not in the archiveAddin query gid support. Rdmavt still relies on the driver to maintain the gid table. Rdmavt simply calls into the driver to retrive the guid for a particular port.
--- drivers/infiniband/sw/rdmavt/vt.c | 17 ++++++++++++++--- include/rdma/rdma_vt.h | 2 ++ 2 files changed, 16 insertions(+), 3 deletions(-)
drivers/infiniband/sw/rdmavt/vt.c +14 -3
@@ -210,17 +210,28 @@ static int rvt_query_pkey(struct ib_device *ibdev, u8 port_num, u16 index,* Returns 0 on success */ static int rvt_query_gid (struct ib_device *ibdev,u8 port_num, -int index,union ib_gid *gid) +int guid_index,union ib_gid *gid) { +struct rvt_dev_info *rdi; +struct rvt_ibport *rvp; +int port_index; +/* * Driver is responsible for updating the guid table. Which will be used * to craft the return value. This will work similar to how query_pkey() * is being done. */ -if (ibport_num_to_idx (ibdev, port_num) <0 ) + port_index =ibport_num_to_idx (ibdev, port_num); +if (port_index <0 )return -EINVAL ; -return -EOPNOTSUPP ; + rdi =ib_to_rvt (ibdev); + rvp = rdi->ports[port_index]; + + gid->global.subnet_prefix = rvp->gid_prefix; + +return rdi->driver_f.get_guid_be (rdi, rvp, guid_index, + &gid->global.interface_id); }struct rvt_ucontext {
include/rdma/rdma_vt.h +2 -0
@@ -248,6 +248,8 @@ struct rvt_driver_provided {u32 (*mtu_from_qp)(struct rvt_dev_info *rdi,struct rvt_qp *qp,u32 pmtu);int (*mtu_to_path_mtu)(u32 mtu); +int (*get_guid_be)(struct rvt_dev_info *rdi,struct rvt_ibport *rvp, +int guid_index,__be64 *guid);/*--------------------*/ /* Optional functions */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 6/7 IB/rdmavt: Add support for query_port, modify_port and get_port_immutable
In reply to a message not in the archiveFrom: Harish Chegondi <[email protected]>
rvt_query_port calls into the driver through a call back function query_port_state to populate the rest of ib_port_attr elements. rvt_modify_port calls into the driver if needed through a call back function shut_down_port()
--- drivers/infiniband/sw/rdmavt/vt.c | 77 ++++++++++++++++++++++++++----------- include/rdma/rdma_vt.h | 6 +++ 2 files changed, 61 insertions(+), 22 deletions(-)
drivers/infiniband/sw/rdmavt/vt.c +55 -22
@@ -128,14 +128,27 @@ static int rvt_modify_device(struct ib_device *device,static int rvt_query_port (struct ib_device *ibdev,u8 port_num,struct ib_port_attr *props) { -if (ibport_num_to_idx (ibdev, port_num) <0 ) +struct rvt_dev_info *rdi =ib_to_rvt (ibdev); +struct rvt_ibport *rvp; +int port_index =ibport_num_to_idx (ibdev, port_num); + +if (port_index <0 )return -EINVAL ; -/* -* VT-DRIVER-API: query_port_state() -* driver returns pretty much everything in ib_port_attr -*/ -return -EOPNOTSUPP ; + rvp = rdi->ports[port_index]; +memset (props,0 ,sizeof (*props)); + props->sm_lid = rvp->sm_lid; + props->sm_sl = rvp->sm_sl; + props->port_cap_flags = rvp->port_cap_flags; + props->max_msg_sz =0x80000000 ; + props->pkey_tbl_len =rvt_get_npkeys (rdi); + props->bad_pkey_cntr = rvp->pkey_violations; + props->qkey_viol_cntr = rvp->qkey_violations; + props->subnet_timeout = rvp->subnet_timeout; + props->init_type_reply =0 ; + +/* Populate the remaining ib_port_attr elements */ +return rdi->driver_f.query_port_state (rdi, port_num, props); }/**
@@ -150,23 +163,26 @@ static int rvt_query_port(struct ib_device *ibdev, u8 port_num,static int rvt_modify_port (struct ib_device *ibdev,u8 port_num,int port_modify_mask,struct ib_port_modify *props) { -/* -* VT-DRIVER-API: set_link_state() -* driver will set the link state using the IB enumeration -* -* VT-DRIVER-API: clear_qkey_violations() -* clears driver private qkey counter -* -* VT-DRIVER-API: get_lid() -* driver needs to return the LID -* -* TBD: send_trap() and post_mad_send() need examined to see where they -* fit in. -*/ -if (ibport_num_to_idx (ibdev, port_num) <0 ) +struct rvt_dev_info *rdi =ib_to_rvt (ibdev); +struct rvt_ibport *rvp; +int ret =0 ; +int port_index =ibport_num_to_idx (ibdev, port_num); + +if (port_index <0 )return -EINVAL ; -return -EOPNOTSUPP ; + rvp = rdi->ports[port_index]; + rvp->port_cap_flags |= props->set_port_cap_mask; + rvp->port_cap_flags &= ~props->clr_port_cap_mask; + +if (props->set_port_cap_mask || props->clr_port_cap_mask) + rdi->driver_f.cap_mask_chg (rdi, port_num); +if (port_modify_mask &IB_PORT_SHUTDOWN ) + ret = rdi->driver_f.shut_down_port (rdi, port_num); +if (port_modify_mask &IB_PORT_RESET_QKEY_CNTR ) + rvp->qkey_violations =0 ; + +return ret; }/**
@@ -273,7 +289,24 @@ static int rvt_dealloc_ucontext(struct ib_ucontext *context)static int rvt_get_port_immutable (struct ib_device *ibdev,u8 port_num,struct ib_port_immutable *immutable) { -return -EOPNOTSUPP ; +struct rvt_dev_info *rdi =ib_to_rvt (ibdev); +struct ib_port_attr attr; +int err, port_index; + + port_index =ibport_num_to_idx (ibdev, port_num); +if (port_index <0 ) +return -EINVAL ; + + err =rvt_query_port (ibdev, port_num, &attr); +if (err) +return err; + + immutable->pkey_tbl_len = attr.pkey_tbl_len; + immutable->gid_tbl_len = attr.gid_tbl_len; + immutable->core_cap_flags = rdi->dparms.core_cap_flags; + immutable->max_mad_size = rdi->dparms.max_mad_size; + +return 0 ; }/*
include/rdma/rdma_vt.h +6 -0
@@ -200,6 +200,8 @@ struct rvt_driver_params {int psn_mask;int psn_shift;int psn_modify_mask; +u32 core_cap_flags; +u32 max_mad_size; };/* Protection domain */
@@ -250,6 +252,10 @@ struct rvt_driver_provided {int (*mtu_to_path_mtu)(u32 mtu);int (*get_guid_be)(struct rvt_dev_info *rdi,struct rvt_ibport *rvp,int guid_index,__be64 *guid); +int (*query_port_state)(struct rvt_dev_info *rdi,u8 port_num, +struct ib_port_attr *props); +int (*shut_down_port)(struct rvt_dev_info *rdi,u8 port_num); +void (*cap_mask_chg)(struct rvt_dev_info *rdi,u8 port_num);/*--------------------*/ /* Optional functions */
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 7/7 IB/rdmavt: Properly pass gfp to hw driver function
In reply to a message not in the archiveFrom: Ira Weiny <[email protected]>
alloc_qpn must use GFP and the hardware drivers should use it as well.
--- drivers/infiniband/sw/rdmavt/qp.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-)
drivers/infiniband/sw/rdmavt/qp.c +1 -2
@@ -303,8 +303,7 @@ static int alloc_qpn(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,u32 ret;if (rdi->driver_f.alloc_qpn) -return rdi->driver_f.alloc_qpn (rdi, qpt, type, port_num, -GFP_KERNEL ); +return rdi->driver_f.alloc_qpn (rdi, qpt, type, port_num, gfp);if (type ==IB_QPT_SMI || type ==IB_QPT_GSI ) {unsigned n;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html