IB/hfi1: Driver fixes for 10/23/2017
Subsystems (2)
cover IB/hfi1: Driver fixes for 10/23/2017
Hi Doug,
Another round of fixes. These are all pretty minor. Mike M added some more tx opcode stats to debugfs, similar to what we did for the rx side.
Patches can can also be found in my GitHub repo at:
https://github.com/ddalessa/kernel/tree/for-4.15
---
Kamenee Arumugam (1):
IB/hfi1: Don't modify num_user_contexts module parameter
Michael J. Ruhl (1):
IB/hfi1: Race condition between user notification and driver state
Mike Marciniszyn (3):
IB/hfi1: Add tx_opcode_stats like the opcode_stats IB/hfi1: Insure int mask for in-kernel receive contexts is clear IB/hfi1: Take advantage of kvzalloc_node in sdma initialization
Patel Jay P (1):
Ib/hfi1: Return actual operational VLs in port info query
Sebastian Sanchez (1):
IB/hfi1: Validate PKEY for incoming GSI MAD packets
drivers/infiniband/hw/hfi1/chip.c | 93 ++++++++++++++++++++++++---------- drivers/infiniband/hw/hfi1/debugfs.c | 56 ++++++++++++++++++-- drivers/infiniband/hw/hfi1/hfi.h | 6 +- drivers/infiniband/hw/hfi1/init.c | 7 +++ drivers/infiniband/hw/hfi1/intr.c | 14 +++++ drivers/infiniband/hw/hfi1/mad.c | 86 +++++++++++++++++++++++++++++++ drivers/infiniband/hw/hfi1/sdma.c | 9 +-- drivers/infiniband/hw/hfi1/verbs.c | 24 +++++++++ 8 files changed, 248 insertions(+), 47 deletions(-)
--
-Denny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 1/7 IB/hfi1: Race condition between user notification and driver state
In reply to a message not in the archiveFrom: Michael J. Ruhl <[email protected]>
The handler for link init state (HLS_UP_INIT) notifies userspace (update_statusp()) before enabling the device (RCV_CTRL_RCV_PORT_ENABLE_SMASK) or setting the device state (ppd->host_link_state). This causes a race condition where the userspace thinks the interface is in the INIT state before the driver has set that state.
Rework the code path to eliminate the race.
Delay setting the init state until after a HW settling period.
--- drivers/infiniband/hw/hfi1/chip.c | 27 ++++++++++++++++++--------- drivers/infiniband/hw/hfi1/intr.c | 14 +++++++++++++- 2 files changed, 31 insertions(+), 10 deletions(-)
drivers/infiniband/hw/hfi1/chip.c +18 -9
@@ -10317,9 +10317,6 @@ static void force_logical_link_state_down(struct hfi1_pportdata *ppd)write_csr (dd,DC_LCB_CFG_ALLOW_LINK_UP ,0 );write_csr (dd,DC_LCB_CFG_IGNORE_LOST_RCLK ,0 ); -/* adjust ppd->statusp, if needed */ -update_statusp (ppd,IB_PORT_DOWN ); -dd_dev_info (ppd->dd,"logical state forced to LINK_DOWN\n" ); }
@@ -10401,6 +10398,7 @@ static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason)force_logical_link_state_down (ppd); ppd->host_link_state =HLS_LINK_COOLDOWN ;/* LCB access allowed */ +update_statusp (ppd,IB_PORT_DOWN );/* * The LNI has a mandatory wait time after the physical state
@@ -10662,6 +10660,7 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)handle_linkup_change (dd,1 ); ppd->host_link_state =HLS_UP_INIT ; +update_statusp (ppd,IB_PORT_INIT );break ;case HLS_UP_ARMED :if (ppd->host_link_state !=HLS_UP_INIT )
@@ -10683,6 +10682,7 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)break ; } ppd->host_link_state =HLS_UP_ARMED ; +update_statusp (ppd,IB_PORT_ARMED );/* * The simulator does not currently implement SMA messages, * so neighbor_normal is not set. Set it here when we first
@@ -10705,6 +10705,7 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)/* tell all engines to go running */ sdma_all_running (dd); ppd->host_link_state =HLS_UP_ACTIVE ; +update_statusp (ppd,IB_PORT_ACTIVE );/* Signal the IB layer that the port has went active */ event.device = &dd->verbs_dev.rdi.ibdev;
@@ -12720,6 +12721,17 @@ u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate)return "unknown" ; } +/** +* update_statusp - Update userspace status flag +* @ppd: Port data structure +* @state: port state information +* +* Actual port status is determined by the host_link_state value +* in the ppd. +* +* host_link_state MUST be updated before updating the user space +* statusp. +*/ static void update_statusp (struct hfi1_pportdata *ppd,u32 state) {/*
@@ -12745,9 +12757,11 @@ static void update_statusp(struct hfi1_pportdata *ppd, u32 state)break ; } } +dd_dev_info (ppd->dd,"logical state changed to %s (0x%x)\n" , +opa_lstate_name (state), state); } -/* +/** * wait_logical_linkstate - wait for an IB link state change to occur * @ppd: port device * @state: the state to wait for
@@ -12778,11 +12792,6 @@ static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state,msleep (20 ); } -update_statusp (ppd, state); -dd_dev_info (ppd->dd, -"logical state changed to %s (0x%x)\n" , -opa_lstate_name (state), - state);return 0 ; }
drivers/infiniband/hw/hfi1/intr.c +13 -1
@@ -53,6 +53,8 @@#include "common.h" #include "sdma.h" +#define LINK_UP_DELAY 500/* in microseconds */ +/** * format_hwmsg - format a single hwerror message * @msg message buffer
@@ -102,9 +104,16 @@ static void signal_ib_event(struct hfi1_pportdata *ppd, enum ib_event_type ev)ib_dispatch_event (&event); } -/* +/** +* handle_linkup_change - finish linkup/down state changes +* @dd: valid device +* @linkup: link state information +* * Handle a linkup or link down notification. +* The HW needs time to finish its link up state change. Give it that chance. +* * This is called outside an interrupt. +* */ void handle_linkup_change (struct hfi1_devdata *dd,u32 linkup) {
@@ -151,6 +160,9 @@ void handle_linkup_change(struct hfi1_devdata *dd, u32 linkup) ppd->neighbor_guid, ppd->neighbor_type, ppd->neighbor_port_number); +/* HW needs LINK_UP_DELAY to settle, give it that chance */ +udelay (LINK_UP_DELAY ); +/* physical link went up */ ppd->linkup =1 ; ppd->offline_disabled_reason =
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 2/7 Ib/hfi1: Return actual operational VLs in port info query
In reply to a message not in the archiveFrom: Patel Jay P <[email protected]>
__subn_get_opa_portinfo stores value returned by hfi1_get_ib_cfg() as operational vls. hfi1_get_ib_cfg() returns vls_operational field in hfi1_pportdata. The problem with this is that the value is always equal to vls_supported field in hfi1_pportdata.
The logic to calculate operational_vls is to set value passed by FM (in __subn_set_opa_portinfo routine). If no value is passed then default value is stored in operational_vls.
Field actual_vls_operational is calculated on the basis of buffer control table. Hence, modifying hfi1_get_ib_cfg() to return actual_operational_vls when used with HFI1_IB_CFG_OP_VLS parameter
--- drivers/infiniband/hw/hfi1/chip.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-)
drivers/infiniband/hw/hfi1/chip.c +1 -1
@@ -9967,7 +9967,7 @@ int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which) goto unimplemented;case HFI1_IB_CFG_OP_VLS : - val = ppd->vls_operational; + val = ppd->actual_vls_operational;break ;case HFI1_IB_CFG_VL_HIGH_CAP :/* VL arb high priority table size */ val =VL_ARB_HIGH_PRIO_TABLE_SIZE ;
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 3/7 IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveFrom: Sebastian Sanchez <[email protected]>
These are the use-cases where the pkey needs to be tested to see if a packet needs to be dropped.
a) Check if pkey is not FULL_MGMT_P_KEY or LIM_MGMT_P_KEY, drop the packet as it's not part of the management partition. Self-originated packets are an exception.
b) If pkey index points to FULL_MGMT_P_KEY and LIM_MGMT_P_KEY is in the table, the packet is coming from a management node, and the receiving node is also a management node, so it is safe for the packet to go through.
c) If pkey index points to FULL_MGMT_P_KEY and LIM_MGMT_P_KEY is NOT in the table, drop the packet as LIM_MGMT_P_KEY should always be in the pkey table. It could be a misconfiguration.
d) If pkey index points to LIM_MGMT_P_KEY and FULL_MGMT_P_KEY is NOT in the table, it is safe for the packet to go through since a non-management node is talking to another non-managment node.
e) If pkey index points to LIM_MGMT_P_KEY and FULL_MGMT_P_KEY is in the table, drop the packet because a non-management node is talking to a management node, and it could be an attack.
For the implementation, these rules can be simplied to only checking for (a) and (e). There's no need to check for rule (b) as the packet doesn't need to be dropped. Rule (c) is not possible in the driver as LIM_MGMT_P_KEY is always in the pkey table.
--- drivers/infiniband/hw/hfi1/mad.c | 86 +++++++++++++++++++++++++++++++++++++- 1 files changed, 84 insertions(+), 2 deletions(-)
drivers/infiniband/hw/hfi1/mad.c +84 -2
@@ -98,6 +98,16 @@ static inline void clear_opa_smp_data(struct opa_smp *smp)memset (data,0 , size); } +static inline u16 hfi1_lookup_pkey_value (struct hfi1_ibport *ibp,int pkey_idx) +{ +struct hfi1_pportdata *ppd =ppd_from_ibp (ibp); + +if (pkey_idx <ARRAY_SIZE (ppd->pkeys)) +return ppd->pkeys[pkey_idx]; + +return 0 ; +} +void hfi1_event_pkey_change (struct hfi1_devdata *dd,u8 port) {struct ib_event event;
@@ -4260,6 +4270,18 @@ void clear_linkup_counters(struct hfi1_devdata *dd) dd->err_info_xmit_constraint.status &= ~OPA_EI_STATUS_SMASK ; } +static int is_full_mgmt_pkey_in_table (struct hfi1_ibport *ibp) +{ +unsigned int i; +struct hfi1_pportdata *ppd =ppd_from_ibp (ibp); + +for (i =0 ; i <ARRAY_SIZE (ppd->pkeys); ++i) +if (ppd->pkeys[i] ==FULL_MGMT_P_KEY ) +return 1 ; + +return 0 ; +} +/* * is_local_mad() returns 1 if 'mad' is sent from, and destined to the * local node, 0 otherwise.
@@ -4327,6 +4349,63 @@ static int opa_local_smp_check(struct hfi1_ibport *ibp,return 1 ; } +/** +* hfi1_pkey_validation_pma - It validates PKEYs for incoming PMA MAD packets. +* @ibp: IB port data +* @in_mad: MAD packet with header and data +* @in_wc: Work completion data such as source LID, port number, etc. +* +* These are all the possible logic rules for validating a pkey: +* +* a) If pkey neither FULL_MGMT_P_KEY nor LIM_MGMT_P_KEY, +* and NOT self-originated packet: +* Drop MAD packet as it should always be part of the +* management partition unless it's a self-originated packet. +* +* b) If pkey_index -> FULL_MGMT_P_KEY, and LIM_MGMT_P_KEY in pkey table: +* The packet is coming from a management node and the receiving node +* is also a management node, so it is safe for the packet to go through. +* +* c) If pkey_index -> FULL_MGMT_P_KEY, and LIM_MGMT_P_KEY is NOT in pkey table: +* Drop the packet as LIM_MGMT_P_KEY should always be in the pkey table. +* It could be an FM misconfiguration. +* +* d) If pkey_index -> LIM_MGMT_P_KEY and FULL_MGMT_P_KEY is NOT in pkey table: +* It is safe for the packet to go through since a non-management node is +* talking to another non-management node. +* +* e) If pkey_index -> LIM_MGMT_P_KEY and FULL_MGMT_P_KEY in pkey table: +* Drop the packet because a non-management node is talking to a +* management node, and it could be an attack. +* +* For the implementation, these rules can be simplied to only checking +* for (a) and (e). There's no need to check for rule (b) as +* the packet doesn't need to be dropped. Rule (c) is not possible in +* the driver as LIM_MGMT_P_KEY is always in the pkey table. +* +* Return: +* 0 - pkey is okay, -EINVAL it's a bad pkey +*/ +static int hfi1_pkey_validation_pma (struct hfi1_ibport *ibp, +const struct opa_mad *in_mad, +const struct ib_wc *in_wc) +{ +u16 pkey_value =hfi1_lookup_pkey_value (ibp, in_wc->pkey_index); + +/* Rule (a) from above */ +if (!is_local_mad (ibp, in_mad, in_wc) && + pkey_value !=LIM_MGMT_P_KEY && + pkey_value !=FULL_MGMT_P_KEY ) +return -EINVAL ; + +/* Rule (e) from above */ +if (pkey_value ==LIM_MGMT_P_KEY && +is_full_mgmt_pkey_in_table (ibp)) +return -EINVAL ; + +return 0 ; +} + static int process_subn_opa(struct ib_device *ibdev, int mad_flags, u8 port, const struct opa_mad *in_mad, struct opa_mad *out_mad,
@@ -4666,8 +4745,11 @@ static int hfi1_process_opa_mad(struct ib_device *ibdev, int mad_flags, out_mad, &resp_len); goto bail;case IB_MGMT_CLASS_PERF_MGMT : - ret =process_perf_opa (ibdev, port, in_mad, out_mad, - &resp_len); + ret =hfi1_pkey_validation_pma (ibp, in_mad, in_wc); +if (ret) +return IB_MAD_RESULT_FAILURE ; + + ret =process_perf_opa (ibdev, port, in_mad, out_mad, &resp_len); goto bail;default :
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 4/7 IB/hfi1: Add tx_opcode_stats like the opcode_stats
In reply to a message not in the archiveFrom: Mike Marciniszyn <[email protected]>
This patch adds tx_opcode_stats to parallel the
(rx)opcode_stats in the debugfs.
--- drivers/infiniband/hw/hfi1/debugfs.c | 56 ++++++++++++++++++++++++++++++---- drivers/infiniband/hw/hfi1/hfi.h | 6 ++-- drivers/infiniband/hw/hfi1/init.c | 7 ++++ drivers/infiniband/hw/hfi1/verbs.c | 24 +++++++++++++++ 4 files changed, 83 insertions(+), 10 deletions(-)
drivers/infiniband/hw/hfi1/debugfs.c +49 -7
@@ -165,6 +165,17 @@ static void _opcode_stats_seq_stop(struct seq_file *s, void *v) { } +static int opcode_stats_show (struct seq_file *s,u8 i,u64 packets,u64 bytes) +{ +if (!packets && !bytes) +return SEQ_SKIP ; +seq_printf (s,"%02x %llu/%llu\n" , i, + (unsigned long long )packets, + (unsigned long long )bytes); + +return 0 ; +} +static int _opcode_stats_seq_show (struct seq_file *s,void *v) {loff_t *spos = v;
@@ -182,19 +193,49 @@ static int _opcode_stats_seq_show(struct seq_file *s, void *v) }hfi1_rcd_put (rcd); } -if (!n_packets && !n_bytes) -return SEQ_SKIP ; -seq_printf (s,"%02llx %llu/%llu\n" , i, - (unsigned long long )n_packets, - (unsigned long long )n_bytes); - -return 0 ; +return opcode_stats_show (s, i, n_packets, n_bytes); }DEBUGFS_SEQ_FILE_OPS (opcode_stats);DEBUGFS_SEQ_FILE_OPEN (opcode_stats)DEBUGFS_FILE_OPS (opcode_stats); +static void *_tx_opcode_stats_seq_start (struct seq_file *s,loff_t *pos) +{ +return _opcode_stats_seq_start (s, pos); +} + +static void *_tx_opcode_stats_seq_next (struct seq_file *s,void *v,loff_t *pos) +{ +return _opcode_stats_seq_next (s, v, pos); +} + +static void _tx_opcode_stats_seq_stop (struct seq_file *s,void *v) +{ +} + +static int _tx_opcode_stats_seq_show (struct seq_file *s,void *v) +{ +loff_t *spos = v; +loff_t i = *spos; +int j; +u64 n_packets =0 , n_bytes =0 ; +struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private; +struct hfi1_devdata *dd =dd_from_dev (ibd); + +for_each_possible_cpu (j) { +struct hfi1_opcode_stats_perctx *s = +per_cpu_ptr (dd->tx_opstats, j); + n_packets += s->stats[i].n_packets; + n_bytes += s->stats[i].n_bytes; + } +return opcode_stats_show (s, i, n_packets, n_bytes); +} + +DEBUGFS_SEQ_FILE_OPS (tx_opcode_stats); +DEBUGFS_SEQ_FILE_OPEN (tx_opcode_stats) +DEBUGFS_FILE_OPS (tx_opcode_stats); +static void *_ctx_stats_seq_start (struct seq_file *s,loff_t *pos) {struct hfi1_ibdev *ibd = (struct hfi1_ibdev *)s->private;
@@ -1363,6 +1404,7 @@ void hfi1_dbg_ibdev_init(struct hfi1_ibdev *ibd)return ; }DEBUGFS_SEQ_FILE_CREATE (opcode_stats, ibd->hfi1_ibdev_dbg, ibd); +DEBUGFS_SEQ_FILE_CREATE (tx_opcode_stats, ibd->hfi1_ibdev_dbg, ibd);DEBUGFS_SEQ_FILE_CREATE (ctx_stats, ibd->hfi1_ibdev_dbg, ibd);DEBUGFS_SEQ_FILE_CREATE (qp_stats, ibd->hfi1_ibdev_dbg, ibd);DEBUGFS_SEQ_FILE_CREATE (sdes, ibd->hfi1_ibdev_dbg, ibd);
drivers/infiniband/hw/hfi1/hfi.h +3 -3
@@ -164,9 +164,7 @@ struct hfi1_ib_stats {* Below contains all data related to a single context (formerly called port). */ -#ifdef CONFIG_DEBUG_FS struct hfi1_opcode_stats_perctx ; -#endif struct ctxt_eager_bufs {ssize_t size;/* total size of eager buffers */
@@ -283,7 +281,7 @@ struct hfi1_ctxtdata {u64 imask;/* clear interrupt mask */ int ireg;/* clear interrupt register */ unsigned numa_id;/* numa node of this context */ -/* verbs stats per CTX */ +/* verbs rx_stats per rcd */ struct hfi1_opcode_stats_perctx *opstats;/* Is ASPM interrupt supported for this context */
@@ -1276,6 +1274,8 @@ struct hfi1_devdata {/* receive context data */ struct hfi1_ctxtdata **rcd;u64 __percpu *int_counter; +/* verbs tx opcode stats */ +struct hfi1_opcode_stats_perctx __percpu *tx_opstats;/* device (not port) flags, basically device capabilities */ u16 flags;/* Number of physical ports available */
drivers/infiniband/hw/hfi1/init.c +7 -0
@@ -1217,6 +1217,7 @@ static void __hfi1_free_devdata(struct kobject *kobj)free_percpu (dd->int_counter);free_percpu (dd->rcv_limit);free_percpu (dd->send_schedule); +free_percpu (dd->tx_opstats);rvt_dealloc_device (&dd->verbs_dev.rdi); }
@@ -1306,6 +1307,12 @@ struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, size_t extra) goto bail; } + dd->tx_opstats = alloc_percpu(struct hfi1_opcode_stats_perctx); +if (!dd->tx_opstats) { + ret = -ENOMEM ; + goto bail; + } +kobject_init (&dd->kobj, &hfi1_devdata_type);return dd;
drivers/infiniband/hw/hfi1/verbs.c +24 -0
@@ -796,6 +796,27 @@ static noinline int build_verbs_ulp_payload(return ret; } +/** +* update_tx_opstats - record stats by opcode +* @qp; the qp +* @ps: transmit packet state +* @plen: the plen in dwords +* +* This is a routine to record the tx opstats after a +* packet has been presented to the egress mechanism. +*/ +static void update_tx_opstats (struct rvt_qp *qp,struct hfi1_pkt_state *ps, +u32 plen) +{ +#ifdef CONFIG_DEBUG_FS +struct hfi1_devdata *dd =dd_from_ibdev (qp->ibqp.device); +struct hfi1_opcode_stats_perctx *s =get_cpu_ptr (dd->tx_opstats); + +inc_opstats (plen *4 , &s->stats[ps->opcode]); +put_cpu_ptr (s); +#endif +} +/* * Build the number of DMA descriptors needed to send length bytes of data. *
@@ -940,6 +961,8 @@ int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, goto bail_ecomm;return ret; } + +update_tx_opstats (qp, ps, plen);trace_sdma_output_ibhdr (dd_from_ibdev (qp->ibqp.device), &ps->s_txreq->phdr.hdr,ib_is_sc5 (sc5));return ret;
@@ -1137,6 +1160,7 @@ int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps,seg_pio_copy_end (pbuf); } +update_tx_opstats (qp, ps, plen);trace_pio_output_ibhdr (dd_from_ibdev (qp->ibqp.device), &ps->s_txreq->phdr.hdr,ib_is_sc5 (sc5));
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 5/7 IB/hfi1: Insure int mask for in-kernel receive contexts is clear
In reply to a message not in the archiveFrom: Mike Marciniszyn <[email protected]>
The only use for the urg interrupt is for priority PSM packets.
There is no reason for this interrupt to be enabled for kernel contexts.
--- drivers/infiniband/hw/hfi1/chip.c | 35 ++++++++++++++++++++++++++++++++--- 1 files changed, 32 insertions(+), 3 deletions(-)
drivers/infiniband/hw/hfi1/chip.c +32 -3
@@ -12938,6 +12938,32 @@ int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp)return ret; } +/** +* get_int_mask - get 64 bit int mask +* @dd - the devdata +* @i - the csr (relative to CCE_INT_MASK) +* +* Returns the mask with the urgent interrupt mask +* bit clear for kernel receive contexts. +*/ +static u64 get_int_mask (struct hfi1_devdata *dd,u32 i) +{ +u64 mask =U64_MAX ;/* default to no change */ + +if (i >= (IS_RCVURGENT_START /64 ) && i < (IS_RCVURGENT_END /64 )) { +int j = (i - (IS_RCVURGENT_START /64 )) *64 ; +int k = !j ?IS_RCVURGENT_START %64 :0 ; + +if (j) + j -=IS_RCVURGENT_START %64 ; +/* j = 0..dd->first_dyn_alloc_ctxt - 1,k = 0..63 */ +for (; j < dd->first_dyn_alloc_ctxt && k <64 ; j++, k++) +/* convert to bit in mask and clear */ + mask &= ~BIT_ULL (k); + } +return mask; +} +/* ========================================================================= */ /*
@@ -12951,9 +12977,12 @@ void set_intr_state(struct hfi1_devdata *dd, u32 enable)* In HFI, the mask needs to be 1 to allow interrupts. */ if (enable) { -/* enable all interrupts */ -for (i =0 ; i <CCE_NUM_INT_CSRS ; i++) -write_csr (dd,CCE_INT_MASK + (8 * i), ~(u64 )0 ); +/* enable all interrupts but urgent on kernel contexts */ +for (i =0 ; i <CCE_NUM_INT_CSRS ; i++) { +u64 mask =get_int_mask (dd, i); + +write_csr (dd,CCE_INT_MASK + (8 * i), mask); + }init_qsfp_int (dd); }else {
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 6/7 IB/hfi1: Don't modify num_user_contexts module parameter
In reply to a message not in the archiveFrom: Kamenee Arumugam <[email protected]>
The driver parameter num_user_contexts controls global behavior and should not be modified by the driver.
This patch eliminates modification of num_user_contexts by using a local variable to keep track of the value.
--- drivers/infiniband/hw/hfi1/chip.c | 29 +++++++++++++++-------------- 1 files changed, 15 insertions(+), 14 deletions(-)
drivers/infiniband/hw/hfi1/chip.c +15 -14
@@ -13409,6 +13409,7 @@ static int set_up_context_variables(struct hfi1_devdata *dd)unsigned ngroups;int qos_rmt_count;int user_rmt_reduced; +u32 n_usr_ctxts;/* * Kernel receive contexts:
@@ -13451,42 +13452,42 @@ static int set_up_context_variables(struct hfi1_devdata *dd)* num_user_contexts is negative */ if (num_user_contexts <0 ) - num_user_contexts = -cpumask_weight (&node_affinity.real_cpu_mask); - + n_usr_ctxts =cpumask_weight (&node_affinity.real_cpu_mask); +else + n_usr_ctxts = num_user_contexts;/* * Adjust the counts given a global max. */ -if (total_contexts + num_user_contexts > dd->chip_rcv_contexts) { +if (total_contexts + n_usr_ctxts > dd->chip_rcv_contexts) {dd_dev_err (dd, -"Reducing # user receive contexts to: %d, from %d\n" , +"Reducing # user receive contexts to: %d, from %u\n" , (int )(dd->chip_rcv_contexts - total_contexts), - (int )num_user_contexts); + n_usr_ctxts);/* recalculate */ - num_user_contexts = dd->chip_rcv_contexts - total_contexts; + n_usr_ctxts = dd->chip_rcv_contexts - total_contexts; }/* each user context requires an entry in the RMT */ qos_rmt_count =qos_rmt_entries (dd,NULL ,NULL ); -if (qos_rmt_count + num_user_contexts >NUM_MAP_ENTRIES ) { +if (qos_rmt_count + n_usr_ctxts >NUM_MAP_ENTRIES ) { user_rmt_reduced =NUM_MAP_ENTRIES - qos_rmt_count;dd_dev_err (dd, -"RMT size is reducing the number of user receive contexts from %d to %d\n" , - (int )num_user_contexts, +"RMT size is reducing the number of user receive contexts from %u to %d\n" , + n_usr_ctxts, user_rmt_reduced);/* recalculate */ - num_user_contexts = user_rmt_reduced; + n_usr_ctxts = user_rmt_reduced; } - total_contexts += num_user_contexts; + total_contexts += n_usr_ctxts;/* the first N are kernel contexts, the rest are user/vnic contexts */ dd->num_rcv_contexts = total_contexts; dd->n_krcv_queues = num_kernel_contexts; dd->first_dyn_alloc_ctxt = num_kernel_contexts; dd->num_vnic_contexts = num_vnic_contexts; - dd->num_user_contexts = num_user_contexts; - dd->freectxts = num_user_contexts; + dd->num_user_contexts = n_usr_ctxts; + dd->freectxts = n_usr_ctxts; dd_dev_info(dd, "rcv contexts: chip %d, used %d (kernel %d, vnic %u, user %u)\n", (int)dd->chip_rcv_contexts,
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH 7/7 IB/hfi1: Take advantage of kvzalloc_node in sdma initialization
In reply to a message not in the archiveFrom: Mike Marciniszyn <[email protected]>
The code that allocates the tx ring in the sdma code fails to take advantage of kvzalloc variations.
Fix by converting to use kvzalloc_node.
--- drivers/infiniband/hw/hfi1/sdma.c | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-)
drivers/infiniband/hw/hfi1/sdma.c +2 -7
@@ -1465,13 +1465,8 @@ int sdma_init(struct hfi1_devdata *dd, u8 port)if (!sde->descq) goto bail; sde->tx_ring = -kcalloc (descq_cnt,sizeof (struct sdma_txreq *), -GFP_KERNEL ); -if (!sde->tx_ring) - sde->tx_ring = -vzalloc ( -sizeof (struct sdma_txreq *) * - descq_cnt); +kvzalloc_node (sizeof (struct sdma_txreq *) * descq_cnt, +GFP_KERNEL , dd->node);if (!sde->tx_ring) goto bail; }
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH for-next 3/7] IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveQuoted 44 lines · Dennis Dalessandro
From: Sebastian Sanchez <[email protected]>
These are the use-cases where the pkey needs to be tested to see if a packet needs to be dropped.
a) Check if pkey is not
FULL_MGMT_P_KEYorLIM_MGMT_P_KEY, drop the packet as it's not part of the management partition. Self-originated packets are an exception.b) If pkey index points to
FULL_MGMT_P_KEYandLIM_MGMT_P_KEYis in the table, the packet is coming from a management node, and the receiving node is also a management node, so it is safe for the packet to go through.c) If pkey index points to
FULL_MGMT_P_KEYandLIM_MGMT_P_KEYis NOT in the table, drop the packet asLIM_MGMT_P_KEYshould always be in the pkey table. It could be a misconfiguration.d) If pkey index points to
LIM_MGMT_P_KEYandFULL_MGMT_P_KEYis NOT in the table, it is safe for the packet to go through since a non-management node is talking to another non-managment node.e) If pkey index points to
LIM_MGMT_P_KEYandFULL_MGMT_P_KEYis in the table, drop the packet because a non-management node is talking to a management node, and it could be an attack.For the implementation, these rules can be simplied to only checking for (a) and (e). There's no need to check for rule (b) as the packet doesn't need to be dropped. Rule (c) is not possible in the driver as
LIM_MGMT_P_KEYis always in the pkey table.Reviewed-by Michael J. Ruhl <[email protected]>Signed-off-by Sebastian Sanchez <[email protected]>Signed-off-by Dennis Dalessandro <[email protected]>--- drivers/infiniband/hw/hfi1/mad.c | 86 +++++++++++++++++++++++++++++++++++++- 1 files changed, 84 insertions(+), 2 deletions(-)drivers/infiniband/hw/hfi1/mad.c +0
-0
drivers/infiniband/hw/hfi1/mad.c static inline void clear_opa_smp_data(struct opa_smp *smp) · line 101
Quoted 4 lines
drivers/infiniband/hw/hfi1/mad.c +1
-0@@ -98,6 +98,16 @@ static inline void clear_opa_smp_data(struct opa_smp *smp)memset (data,0 , size); } +static inline u16 hfi1_lookup_pkey_value (struct hfi1_ibport *ibp,int pkey_idx)
Please, no "inline-function" in *.c files as it is written in CodingStyle.
Thanks
Leon Romanovsky
Reviewed-byRe: [PATCH for-next 7/7] IB/hfi1: Take advantage of kvzalloc_node in sdma initialization
In reply to a message not in the archiveQuoted 7 lines · Dennis Dalessandro
From: Mike Marciniszyn <[email protected]>
The code that allocates the tx ring in the sdma code fails to take advantage of kvzalloc variations.
Fix by converting to use
kvzalloc_node.
Quoted 8 lines
Reported-by Leon Romanovsky <[email protected]>Reviewed-by Dennis Dalessandro <[email protected]>Signed-off-by Mike Marciniszyn <[email protected]>Signed-off-by Dennis Dalessandro <[email protected]>--- drivers/infiniband/hw/hfi1/sdma.c | 9 ++------- 1 files changed, 2 insertions(+), 7 deletions(-)
Thanks, it fixed missed GFP_NOWARN flag too.
Re: [PATCH for-next 3/7] IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveQuoted 1 line · Leon Romanovsky
Review
Quoted 1 line
(quoted diff) +0
-0
b/drivers/infiniband/hw/hfi1/mad.c
Quoted 10 lines
drivers/infiniband/hw/hfi1/mad.c static inline void clear_opa_smp_data(struct opa_smp *smp) · line 101
Quoted 8 lines
index 07b80fa..dfe6224 100644
drivers/infiniband/hw/hfi1/mad.c +1
-0@@ -98,6 +98,16 @@ static inline void clear_opa_smp_data(struct opa_smp *smp)memset (data,0 , size); } +static inline u16 hfi1_lookup_pkey_value (struct hfi1_ibport *ibp,int pkey_idx)Please, no "inline-function" in *.c files as it is written in CodingStyle.
Sure about that?
As a rule of thumb I agree we should not put inlines in C files though. However as you can see from the context, this file already has static inlines so I wasn't too bothered by it.
-Denny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH for-next 3/7] IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveQuoted 10 lines · Dennis Dalessandro
Quoted 1 line · Leon Romanovsky
Review
Quoted 1 line
(quoted diff) +0
-0
b/drivers/infiniband/hw/hfi1/mad.cQuoted 7 lines
Quoted 7 lines
index 07b80fa..dfe6224 100644
drivers/infiniband/hw/hfi1/mad.c +0
-0@@ -98,6 +98,16 @@ static inline void clear_opa_smp_data(struct opa_smp *smp)memset (data,0 , size); }
Quoted 5 lines
Quoted 3 lines
drivers/infiniband/hw/hfi1/mad.c static inline void clear_opa_smp_data(struct opa_smp *smp) · line 101
Quoted 1 line
drivers/infiniband/hw/hfi1/mad.c +1
-0+static inline u16 hfi1_lookup_pkey_value (struct hfi1_ibport *ibp,int pkey_idx)Please, no "inline-function" in *.c files as it is written in CodingStyle.
Sure about that?
I'm not style-nazi, so won't insist, but this is in CodingStyle.
Documentation/process/coding-style.rst
841 15) The inline disease
842 ----------------------
843
844 There appears to be a common misperception that gcc has a magic "make me 845 faster" speedup option called ``inline``. While the use of inlines can be 846 appropriate (for example as a means of replacing macros, see Chapter 12), it 847 very often is not. Abundant use of the inline keyword leads to a much bigger 848 kernel, which in turn slows the system as a whole down, due to a bigger 849 icache footprint for the CPU and simply because there is less memory 850 available for the pagecache. Just think about it; a pagecache miss causes a 851 disk seek, which easily takes 5 milliseconds. There are a LOT of cpu cycles 852 that can go into these 5 milliseconds.
Quoted 7 lines
As a rule of thumb I agree we should not put inlines in C files though. However as you can see from the context, this file already has static inlines so I wasn't too bothered by it.
-Denny
Re: [PATCH for-next 3/7] IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveQuoted 6 lines · Leon Romanovsky
Quoted 3 lines
Quoted 1 line
Please, no "inline-function" in *.c files as it is written in CodingStyle.
Sure about that?
I'm not style-nazi, so won't insist, but this is in CodingStyle.
Quoted 13 lines
Documentation/process/coding-style.rst
841 15) The inline disease
842 ----------------------
843
844 There appears to be a common misperception that gcc has a magic "make me 845 faster" speedup option called ``inline``. While the use of inlines can be 846 appropriate (for example as a means of replacing macros, see Chapter 12), it 847 very often is not. Abundant use of the inline keyword leads to a much bigger 848 kernel, which in turn slows the system as a whole down, due to a bigger 849 icache footprint for the CPU and simply because there is less memory 850 available for the pagecache. Just think about it; a pagecache miss causes a 851 disk seek, which easily takes 5 milliseconds. There are a LOT of cpu cycles 852 that can go into these 5 milliseconds.
Right, which doesn't say not to put it in a .c file. So I think the location is OK for this. However, whether it even needs to be an inline function at all in the first place is certainly a valid question. Let me chat with Sebastian and double check, if it doesn't really need to be inlined we'll send a v2.
-Denny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH for-next 3/7] IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveQuoted 5 lines · Dennis Dalessandro
Right, which doesn't say not to put it in a .c file. So I think the location is OK for this. However, whether it even needs to be an inline function at all in the first place is certainly a valid question. Let me chat with Sebastian and double check, if it doesn't really need to be inlined we'll send a v2.
The general thinking is that the compiler is much better at deciding if something should be inlined or not. For the case of 'static' the compiler will always inline it if it makes sense. 'static inline' disables that heuristic and forces the compiler to always inline..
Jason
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH for-next 3/7] IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveQuoted 8 lines · Jason Gunthorpe
Quoted 5 lines · Dennis Dalessandro
Right, which doesn't say not to put it in a .c file. So I think the location is OK for this. However, whether it even needs to be an inline function at all in the first place is certainly a valid question. Let me chat with Sebastian and double check, if it doesn't really need to be inlined we'll send a v2.
Quoted 5 lines
The general thinking is that the compiler is much better at deciding if something should be inlined or not. For the case of 'static' the compiler will always inline it if it makes sense. 'static inline' disables that heuristic and forces the compiler to always inline..
After talking with Sebastian, this doesn't need to be inlined after all. I will send a v2.
-Denny
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
PATCH v2 3/7 IB/hfi1: Validate PKEY for incoming GSI MAD packets
In reply to a message not in the archiveFrom: Sebastian Sanchez <[email protected]>
These are the use-cases where the pkey needs to be tested to see if a packet needs to be dropped.
a) Check if pkey is not FULL_MGMT_P_KEY or LIM_MGMT_P_KEY, drop the packet as it's not part of the management partition. Self-originated packets are an exception.
b) If pkey index points to FULL_MGMT_P_KEY and LIM_MGMT_P_KEY is in the table, the packet is coming from a management node, and the receiving node is also a management node, so it is safe for the packet to go through.
c) If pkey index points to FULL_MGMT_P_KEY and LIM_MGMT_P_KEY is NOT in the table, drop the packet as LIM_MGMT_P_KEY should always be in the pkey table. It could be a misconfiguration.
d) If pkey index points to LIM_MGMT_P_KEY and FULL_MGMT_P_KEY is NOT in the table, it is safe for the packet to go through since a non-management node is talking to another non-managment node.
e) If pkey index points to LIM_MGMT_P_KEY and FULL_MGMT_P_KEY is in the table, drop the packet because a non-management node is talking to a management node, and it could be an attack.
For the implementation, these rules can be simplied to only checking for (a) and (e). There's no need to check for rule (b) as the packet doesn't need to be dropped. Rule (c) is not possible in the driver as LIM_MGMT_P_KEY is always in the pkey table.
--- drivers/infiniband/hw/hfi1/mad.c | 86 +++++++++++++++++++++++++++++++++++++- 1 files changed, 84 insertions(+), 2 deletions(-)
drivers/infiniband/hw/hfi1/mad.c +84 -2
@@ -98,6 +98,16 @@ static inline void clear_opa_smp_data(struct opa_smp *smp)memset (data,0 , size); } +static u16 hfi1_lookup_pkey_value (struct hfi1_ibport *ibp,int pkey_idx) +{ +struct hfi1_pportdata *ppd =ppd_from_ibp (ibp); + +if (pkey_idx <ARRAY_SIZE (ppd->pkeys)) +return ppd->pkeys[pkey_idx]; + +return 0 ; +} +void hfi1_event_pkey_change (struct hfi1_devdata *dd,u8 port) {struct ib_event event;
@@ -4260,6 +4270,18 @@ void clear_linkup_counters(struct hfi1_devdata *dd) dd->err_info_xmit_constraint.status &= ~OPA_EI_STATUS_SMASK ; } +static int is_full_mgmt_pkey_in_table (struct hfi1_ibport *ibp) +{ +unsigned int i; +struct hfi1_pportdata *ppd =ppd_from_ibp (ibp); + +for (i =0 ; i <ARRAY_SIZE (ppd->pkeys); ++i) +if (ppd->pkeys[i] ==FULL_MGMT_P_KEY ) +return 1 ; + +return 0 ; +} +/* * is_local_mad() returns 1 if 'mad' is sent from, and destined to the * local node, 0 otherwise.
@@ -4327,6 +4349,63 @@ static int opa_local_smp_check(struct hfi1_ibport *ibp,return 1 ; } +/** +* hfi1_pkey_validation_pma - It validates PKEYs for incoming PMA MAD packets. +* @ibp: IB port data +* @in_mad: MAD packet with header and data +* @in_wc: Work completion data such as source LID, port number, etc. +* +* These are all the possible logic rules for validating a pkey: +* +* a) If pkey neither FULL_MGMT_P_KEY nor LIM_MGMT_P_KEY, +* and NOT self-originated packet: +* Drop MAD packet as it should always be part of the +* management partition unless it's a self-originated packet. +* +* b) If pkey_index -> FULL_MGMT_P_KEY, and LIM_MGMT_P_KEY in pkey table: +* The packet is coming from a management node and the receiving node +* is also a management node, so it is safe for the packet to go through. +* +* c) If pkey_index -> FULL_MGMT_P_KEY, and LIM_MGMT_P_KEY is NOT in pkey table: +* Drop the packet as LIM_MGMT_P_KEY should always be in the pkey table. +* It could be an FM misconfiguration. +* +* d) If pkey_index -> LIM_MGMT_P_KEY and FULL_MGMT_P_KEY is NOT in pkey table: +* It is safe for the packet to go through since a non-management node is +* talking to another non-management node. +* +* e) If pkey_index -> LIM_MGMT_P_KEY and FULL_MGMT_P_KEY in pkey table: +* Drop the packet because a non-management node is talking to a +* management node, and it could be an attack. +* +* For the implementation, these rules can be simplied to only checking +* for (a) and (e). There's no need to check for rule (b) as +* the packet doesn't need to be dropped. Rule (c) is not possible in +* the driver as LIM_MGMT_P_KEY is always in the pkey table. +* +* Return: +* 0 - pkey is okay, -EINVAL it's a bad pkey +*/ +static int hfi1_pkey_validation_pma (struct hfi1_ibport *ibp, +const struct opa_mad *in_mad, +const struct ib_wc *in_wc) +{ +u16 pkey_value =hfi1_lookup_pkey_value (ibp, in_wc->pkey_index); + +/* Rule (a) from above */ +if (!is_local_mad (ibp, in_mad, in_wc) && + pkey_value !=LIM_MGMT_P_KEY && + pkey_value !=FULL_MGMT_P_KEY ) +return -EINVAL ; + +/* Rule (e) from above */ +if (pkey_value ==LIM_MGMT_P_KEY && +is_full_mgmt_pkey_in_table (ibp)) +return -EINVAL ; + +return 0 ; +} + static int process_subn_opa(struct ib_device *ibdev, int mad_flags, u8 port, const struct opa_mad *in_mad, struct opa_mad *out_mad,
@@ -4666,8 +4745,11 @@ static int hfi1_process_opa_mad(struct ib_device *ibdev, int mad_flags, out_mad, &resp_len); goto bail;case IB_MGMT_CLASS_PERF_MGMT : - ret =process_perf_opa (ibdev, port, in_mad, out_mad, - &resp_len); + ret =hfi1_pkey_validation_pma (ibp, in_mad, in_wc); +if (ret) +return IB_MAD_RESULT_FAILURE ; + + ret =process_perf_opa (ibdev, port, in_mad, out_mad, &resp_len); goto bail;default :
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
Quoted 9 lines · Dennis Dalessandro
Hi Doug,
Another round of fixes. These are all pretty minor. Mike M added some more
tx opcode stats to debugfs, similar to what we did for the rx side.Patches can can also be found in my GitHub repo at:
https://github.com/ddalessa/kernel/tree/for-4.15
Thanks Denny, series with updated patch 3/7 applied.
Signature · 8 lines
Doug Ledford <[email protected]> GPG KeyID: B826A3330E572FDD Key fingerprint = AE6B 1BDA 122B 23B4 265B 1274 B826 A333 0E57 2FDD -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html