hinic: add some error messages for debug
Subsystems (3)
v2 cover hinic: add some error messages for debug
patch #1: support to handle hw abnormal event
patch #2: improve the error messages when functions return failure and
dump relevant registers in some exception handling processes
Luo bin (2):
hinic: add support to handle hw abnormal event
hinic: add log in exception handling processes
drivers/net/ethernet/huawei/hinic/hinic_dev.h | 2 + .../net/ethernet/huawei/hinic/hinic_ethtool.c | 20 ++ .../ethernet/huawei/hinic/hinic_hw_api_cmd.c | 27 +- .../ethernet/huawei/hinic/hinic_hw_api_cmd.h | 4 + .../net/ethernet/huawei/hinic/hinic_hw_cmdq.c | 2 + .../net/ethernet/huawei/hinic/hinic_hw_cmdq.h | 2 + .../net/ethernet/huawei/hinic/hinic_hw_dev.c | 261 +++++++++++++++++- .../net/ethernet/huawei/hinic/hinic_hw_dev.h | 144 +++++++++- .../net/ethernet/huawei/hinic/hinic_hw_eqs.c | 39 +++ .../net/ethernet/huawei/hinic/hinic_hw_eqs.h | 6 +- .../net/ethernet/huawei/hinic/hinic_hw_if.c | 23 ++ .../net/ethernet/huawei/hinic/hinic_hw_if.h | 10 +- .../net/ethernet/huawei/hinic/hinic_hw_mbox.c | 2 + .../net/ethernet/huawei/hinic/hinic_hw_mgmt.c | 1 + .../net/ethernet/huawei/hinic/hinic_hw_mgmt.h | 4 + .../net/ethernet/huawei/hinic/hinic_main.c | 54 +++- .../net/ethernet/huawei/hinic/hinic_port.c | 62 ++--- .../net/ethernet/huawei/hinic/hinic_port.h | 25 ++ .../net/ethernet/huawei/hinic/hinic_sriov.c | 6 +- 19 files changed, 642 insertions(+), 52 deletions(-)
Signature · 1 line
2.17.1
PATCH v2 1/2 hinic: add support to handle hw abnormal event
add support to handle hw abnormal event such as hardware failure, cable unplugged,link error
---
V1~V2: add link extended state
V0~V1: fix auto build test WARNING
drivers/net/ethernet/huawei/hinic/hinic_dev.h | 2 + .../net/ethernet/huawei/hinic/hinic_ethtool.c | 20 ++ .../net/ethernet/huawei/hinic/hinic_hw_dev.c | 249 +++++++++++++++++- .../net/ethernet/huawei/hinic/hinic_hw_dev.h | 144 +++++++++- .../net/ethernet/huawei/hinic/hinic_hw_mgmt.h | 4 + .../net/ethernet/huawei/hinic/hinic_main.c | 54 +++- .../net/ethernet/huawei/hinic/hinic_port.h | 25 ++ 7 files changed, 491 insertions(+), 7 deletions(-)
drivers/net/ethernet/huawei/hinic/hinic_dev.h +2 -0
@@ -98,6 +98,8 @@ struct hinic_dev {int lb_pkt_len;u8 *lb_test_rx_buf;struct devlink *devlink; +bool cable_unplugged; +bool module_unrecognized; };struct hinic_devlink_priv {
drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +20 -0
@@ -1766,6 +1766,25 @@ static int hinic_get_module_eeprom(struct net_device *netdev,return 0 ; } +static int +hinic_get_link_ext_state (struct net_device *netdev, +struct ethtool_link_ext_state_info *link_ext_state_info) +{ +struct hinic_dev *nic_dev =netdev_priv (netdev); + +if (netif_carrier_ok (netdev)) +return -ENODATA ; + +if (nic_dev->cable_unplugged) + link_ext_state_info->link_ext_state = +ETHTOOL_LINK_EXT_STATE_NO_CABLE ; +else if (nic_dev->module_unrecognized) + link_ext_state_info->link_ext_state = +ETHTOOL_LINK_EXT_STATE_LINK_LOGICAL_MISMATCH ; + +return 0 ; +} +static const struct ethtool_ops hinic_ethtool_ops = { .supported_coalesce_params =ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_RX_MAX_FRAMES |
@@ -1776,6 +1795,7 @@ static const struct ethtool_ops hinic_ethtool_ops = { .set_link_ksettings = hinic_set_link_ksettings, .get_drvinfo = hinic_get_drvinfo, .get_link = ethtool_op_get_link, + .get_link_ext_state = hinic_get_link_ext_state, .get_ringparam = hinic_get_ringparam, .set_ringparam = hinic_set_ringparam, .get_coalesce = hinic_get_coalesce,
drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c +245 -4
275 lines not on this page · raw message
drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h +142 -2
169 lines not on this page · raw message
drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.h +4 -0
@@ -81,6 +81,8 @@ enum hinic_comm_cmd {HINIC_COMM_CMD_MSI_CTRL_REG_WR_BY_UP ,HINIC_COMM_CMD_MSI_CTRL_REG_RD_BY_UP , +HINIC_COMM_CMD_FAULT_REPORT =0x37 , +HINIC_COMM_CMD_SET_LED_STATUS =0x4a , HINIC_COMM_CMD_L2NIC_RESET = 0x4b,
@@ -89,6 +91,8 @@ enum hinic_comm_cmd {HINIC_COMM_CMD_GET_BOARD_INFO =0x52 , +HINIC_COMM_CMD_WATCHDOG_INFO =0x56 , + HINIC_COMM_CMD_MAX, };
drivers/net/ethernet/huawei/hinic/hinic_main.c +53 -1
91 lines not on this page · raw message
drivers/net/ethernet/huawei/hinic/hinic_port.h +25 -0
@@ -189,6 +189,31 @@ struct hinic_port_link_status {u8 port_id; }; +struct hinic_cable_plug_event { +u8 status; +u8 version; +u8 rsvd0[6 ]; + +u16 func_id; +u8 plugged;/* 0: unplugged, 1: plugged */ +u8 port_id; +}; + +enum link_err_type { +LINK_ERR_MODULE_UNRECOGENIZED , +LINK_ERR_NUM , +}; + +struct hinic_link_err_event { +u8 status; +u8 version; +u8 rsvd0[6 ]; + +u16 func_id; +u8 err_type; +u8 port_id; +}; +struct hinic_port_func_state_cmd {u8 status;u8 version;
Signature · 1 line
2.17.1
Jakub Kicinski
maintainerRe: [PATCH net-next v2 1/2] hinic: add support to handle hw abnormal event
Quoted 7 lines · On Sat, 18 Jul 2020 16:58:29 +0800 Luo bin wrote:
add support to handle hw abnormal event such as hardware failure, cable unplugged,link error
Signed-off-by Luo bin <[email protected]>---
V1~V2: add link extended state
V0~V1: fix auto build test WARNING
I don't understand what you think devlink health is missing.
If it is really missing some functionality you require you have to work on extending it.
Dumping error event state into kernel logs when we have specific infrastructure built to address this sort of needs won't fly.
Re: [PATCH net-next v2 1/2] hinic: add support to handle hw abnormal event
Quoted 9 lines · Jakub Kicinski
Quoted 7 lines · On Sat, 18 Jul 2020 16:58:29 +0800 Luo bin wrote:
add support to handle hw abnormal event such as hardware failure, cable unplugged,link error
Signed-off-by Luo bin <[email protected]>---
V1~V2: add link extended state
V0~V1: fix auto build test WARNING
Quoted 9 lines
I don't understand what you think devlink health is missing.
If it is really missing some functionality you require you have to work on extending it.
Dumping error event state into kernel logs when we have specific infrastructure built to address this sort of needs won't fly. .
Okay, I'll try to implement these functionality based on devlink health.
PATCH v2 2/2 hinic: add log in exception handling processes
In reply to Luo binimprove the error message when functions return failure and dump relevant registers in some exception handling processes
---
V0~V1: fix some warning reported by sparse
.../ethernet/huawei/hinic/hinic_hw_api_cmd.c | 27 +++++++- .../ethernet/huawei/hinic/hinic_hw_api_cmd.h | 4 ++ .../net/ethernet/huawei/hinic/hinic_hw_cmdq.c | 2 + .../net/ethernet/huawei/hinic/hinic_hw_cmdq.h | 2 + .../net/ethernet/huawei/hinic/hinic_hw_dev.c | 12 ++-- .../net/ethernet/huawei/hinic/hinic_hw_eqs.c | 39 ++++++++++++ .../net/ethernet/huawei/hinic/hinic_hw_eqs.h | 6 +- .../net/ethernet/huawei/hinic/hinic_hw_if.c | 23 +++++++ .../net/ethernet/huawei/hinic/hinic_hw_if.h | 10 ++- .../net/ethernet/huawei/hinic/hinic_hw_mbox.c | 2 + .../net/ethernet/huawei/hinic/hinic_hw_mgmt.c | 1 + .../net/ethernet/huawei/hinic/hinic_port.c | 62 +++++++++---------- .../net/ethernet/huawei/hinic/hinic_sriov.c | 6 +- 13 files changed, 151 insertions(+), 45 deletions(-)
drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c +25 -2
@@ -112,6 +112,26 @@ static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain)return HINIC_API_CMD_STATUS_GET (val,CONS_IDX ); } +static void dump_api_chain_reg (struct hinic_api_cmd_chain *chain) +{ +u32 addr, val; + + addr =HINIC_CSR_API_CMD_STATUS_ADDR (chain->chain_type); + val =hinic_hwif_read_reg (chain->hwif, addr); + +dev_err (&chain->hwif->pdev->dev,"Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n" , + chain->chain_type,HINIC_API_CMD_STATUS_GET (val,CPLD_ERR ), +HINIC_API_CMD_STATUS_GET (val,CHKSUM_ERR ), +HINIC_API_CMD_STATUS_GET (val,FSM )); + +dev_err (&chain->hwif->pdev->dev,"Chain hw current ci: 0x%x\n" , +HINIC_API_CMD_STATUS_GET (val,CONS_IDX )); + + addr =HINIC_CSR_API_CMD_CHAIN_PI_ADDR (chain->chain_type); + val =hinic_hwif_read_reg (chain->hwif, addr); +dev_err (&chain->hwif->pdev->dev,"Chain hw current pi: 0x%x\n" , val); +} +/** * chain_busy - check if the chain is still processing last requests * @chain: chain to check
@@ -131,8 +151,10 @@ static int chain_busy(struct hinic_api_cmd_chain *chain)/* check for a space for a new command */ if (chain->cons_idx ==MASKED_IDX (chain, prod_idx +1 )) { -dev_err (&pdev->dev,"API CMD chain %d is busy\n" , - chain->chain_type); +dev_err (&pdev->dev,"API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n" , + chain->chain_type, chain->cons_idx, + chain->prod_idx); +dump_api_chain_reg (chain);return -EBUSY ; }break ;
@@ -332,6 +354,7 @@ static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain) err =wait_for_status_poll (chain);if (err) {dev_err (&pdev->dev,"API CMD Poll status timeout\n" ); +dump_api_chain_reg (chain);break ; }break ;
drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.h +4 -0
@@ -103,10 +103,14 @@ HINIC_API_CMD_STATUS_HEADER_##member##_MASK)#define HINIC_API_CMD_STATUS_CONS_IDX_SHIFT 0 +#define HINIC_API_CMD_STATUS_FSM_SHIFT 24#define HINIC_API_CMD_STATUS_CHKSUM_ERR_SHIFT 28 +#define HINIC_API_CMD_STATUS_CPLD_ERR_SHIFT 30#define HINIC_API_CMD_STATUS_CONS_IDX_MASK 0xFFFFFF +#define HINIC_API_CMD_STATUS_FSM_MASK 0xFU#define HINIC_API_CMD_STATUS_CHKSUM_ERR_MASK 0x3 +#define HINIC_API_CMD_STATUS_CPLD_ERR_MASK 0x1U#define HINIC_API_CMD_STATUS_GET (val, member) \ (((val) >> HINIC_API_CMD_STATUS_##member##_SHIFT) & \
drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c +2 -0
@@ -401,6 +401,7 @@ static int cmdq_sync_cmd_direct_resp(struct hinic_cmdq *cmdq,spin_unlock_bh (&cmdq->cmdq_lock); +hinic_dump_ceq_info (cmdq->hwdev);return -ETIMEDOUT ; }
@@ -807,6 +808,7 @@ static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev, cmdq_type =HINIC_CMDQ_SYNC ;for (; cmdq_type <HINIC_MAX_CMDQ_TYPES ; cmdq_type++) { + cmdqs->cmdq[cmdq_type].hwdev = hwdev; err =init_cmdq (&cmdqs->cmdq[cmdq_type], &cmdqs->saved_wqs[cmdq_type], cmdq_type, db_area[cmdq_type]);
drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h +2 -0
@@ -130,6 +130,8 @@ struct hinic_cmdq_ctxt { };struct hinic_cmdq { +struct hinic_hwdev *hwdev; +struct hinic_wq *wq;enum hinic_cmdq_type cmdq_type;
drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c +6 -6
@@ -255,9 +255,9 @@ static int init_fw_ctxt(struct hinic_hwdev *hwdev) &fw_ctxt,sizeof (fw_ctxt), &fw_ctxt, &out_size);if (err || (out_size !=sizeof (fw_ctxt)) || fw_ctxt.status) { -dev_err (&pdev->dev,"Failed to init FW ctxt, ret = %d\n" , - fw_ctxt.status); -return -EFAULT ; +dev_err (&pdev->dev,"Failed to init FW ctxt, err: %d, status: 0x%x, out size: 0x%x\n" , + err, fw_ctxt.status, out_size); +return -EIO ; }return 0 ;
@@ -422,9 +422,9 @@ static int get_base_qpn(struct hinic_hwdev *hwdev, u16 *base_qpn) &cmd_base_qpn,sizeof (cmd_base_qpn), &cmd_base_qpn, &out_size);if (err || (out_size !=sizeof (cmd_base_qpn)) || cmd_base_qpn.status) { -dev_err (&pdev->dev,"Failed to get base qpn, status = %d\n" , - cmd_base_qpn.status); -return -EFAULT ; +dev_err (&pdev->dev,"Failed to get base qpn, err: %d, status: 0x%x, out size: 0x%x\n" , + err, cmd_base_qpn.status, out_size); +return -EIO ; } *base_qpn = cmd_base_qpn.qpn;
drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c +39 -0
@@ -953,3 +953,42 @@ void hinic_ceqs_free(struct hinic_ceqs *ceqs)for (q_id =0 ; q_id < ceqs->num_ceqs; q_id++)remove_eq (&ceqs->ceq[q_id]); } + +void hinic_dump_ceq_info (struct hinic_hwdev *hwdev) +{ +struct hinic_eq *eq =NULL ; +u32 addr, ci, pi; +int q_id; + +for (q_id =0 ; q_id < hwdev->func_to_io.ceqs.num_ceqs; q_id++) { + eq = &hwdev->func_to_io.ceqs.ceq[q_id]; + addr =EQ_CONS_IDX_REG_ADDR (eq); + ci =hinic_hwif_read_reg (hwdev->hwif, addr); + addr =EQ_PROD_IDX_REG_ADDR (eq); + pi =hinic_hwif_read_reg (hwdev->hwif, addr); +dev_err (&hwdev->hwif->pdev->dev,"Ceq id: %d, ci: 0x%08x, sw_ci: 0x%08x, pi: 0x%x, tasklet_state: 0x%lx, wrap: %d, ceqe: 0x%x\n" , + q_id, ci, eq->cons_idx, pi, + eq->ceq_tasklet.state, + eq->wrapped,be32_to_cpu (*(__be32 *)(GET_CURR_CEQ_ELEM (eq)))); + } +} + +void hinic_dump_aeq_info (struct hinic_hwdev *hwdev) +{ +struct hinic_aeq_elem *aeqe_pos =NULL ; +struct hinic_eq *eq =NULL ; +u32 addr, ci, pi; +int q_id; + +for (q_id =0 ; q_id < hwdev->aeqs.num_aeqs; q_id++) { + eq = &hwdev->aeqs.aeq[q_id]; + addr =EQ_CONS_IDX_REG_ADDR (eq); + ci =hinic_hwif_read_reg (hwdev->hwif, addr); + addr =EQ_PROD_IDX_REG_ADDR (eq); + pi =hinic_hwif_read_reg (hwdev->hwif, addr); + aeqe_pos =GET_CURR_AEQ_ELEM (eq); +dev_err (&hwdev->hwif->pdev->dev,"Aeq id: %d, ci: 0x%08x, pi: 0x%x, work_state: 0x%x, wrap: %d, desc: 0x%x\n" , + q_id, ci, pi,work_busy (&eq->aeq_work.work), + eq->wrapped,be32_to_cpu (aeqe_pos->desc)); + } +}
drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.h +5 -1
@@ -162,7 +162,7 @@ enum hinic_eqe_state {struct hinic_aeq_elem {u8 data[HINIC_AEQE_DATA_SIZE ]; -u32 desc; +__be32 desc; };struct hinic_eq_work {
@@ -254,4 +254,8 @@ int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif,void hinic_ceqs_free (struct hinic_ceqs *ceqs); +void hinic_dump_ceq_info (struct hinic_hwdev *hwdev); + +void hinic_dump_aeq_info (struct hinic_hwdev *hwdev); +#endif
drivers/net/ethernet/huawei/hinic/hinic_hw_if.c +23 -0
@@ -21,6 +21,8 @@#define WAIT_HWIF_READY_TIMEOUT 10000 +#define HINIC_SELFTEST_RESULT 0x883C +/** * hinic_msix_attr_set - set message attribute for msix entry * @hwif: the HW interface of a pci function device
@@ -369,6 +371,26 @@ u16 hinic_pf_id_of_vf_hw(struct hinic_hwif *hwif)return HINIC_FA0_GET (attr0,PF_IDX ); } +static void __print_selftest_reg (struct hinic_hwif *hwif) +{ +u32 addr, attr0, attr1; + + addr =HINIC_CSR_FUNC_ATTR1_ADDR ; + attr1 =hinic_hwif_read_reg (hwif, addr); + +if (attr1 ==HINIC_PCIE_LINK_DOWN ) { +dev_err (&hwif->pdev->dev,"PCIE is link down\n" ); +return ; + } + + addr =HINIC_CSR_FUNC_ATTR0_ADDR ; + attr0 =hinic_hwif_read_reg (hwif, addr); +if (HINIC_FA0_GET (attr0,FUNC_TYPE ) !=HINIC_VF && + !HINIC_FA0_GET (attr0,PCI_INTF_IDX )) +dev_err (&hwif->pdev->dev,"Selftest reg: 0x%08x\n" , +hinic_hwif_read_reg (hwif,HINIC_SELFTEST_RESULT )); +} +/** * hinic_init_hwif - initialize the hw interface * @hwif: the HW interface of a pci function device
@@ -398,6 +420,7 @@ int hinic_init_hwif(struct hinic_hwif *hwif, struct pci_dev *pdev) err =wait_hwif_ready (hwif);if (err) {dev_err (&pdev->dev,"HW interface is not ready\n" ); +__print_selftest_reg (hwif); goto err_hwif_ready; }
drivers/net/ethernet/huawei/hinic/hinic_hw_if.h +8 -2
@@ -12,6 +12,8 @@#include <linux/types.h> #include <asm/byteorder.h> +#define HINIC_PCIE_LINK_DOWN 0xFFFFFFFF +#define HINIC_DMA_ATTR_ST_SHIFT 0#define HINIC_DMA_ATTR_AT_SHIFT 8#define HINIC_DMA_ATTR_PH_SHIFT 10
@@ -249,13 +251,17 @@ struct hinic_hwif {static inline u32 hinic_hwif_read_reg (struct hinic_hwif *hwif,u32 reg) { -return be32_to_cpu (readl (hwif->cfg_regs_bar + reg)); +u32 out =readl (hwif->cfg_regs_bar + reg); + +return be32_to_cpu (*(__be32 *)&out); }static inline void hinic_hwif_write_reg (struct hinic_hwif *hwif,u32 reg,u32 val) { -writel (cpu_to_be32 (val), hwif->cfg_regs_bar + reg); +__be32 in =cpu_to_be32 (val); + +writel (*(u32 *)&in, hwif->cfg_regs_bar + reg); } int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index,
drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c +2 -0
@@ -650,6 +650,7 @@ wait_for_mbox_seg_completion(struct hinic_mbox_func_to_func *func_to_func,if (!wait_for_completion_timeout (done, jif)) {dev_err (&hwdev->hwif->pdev->dev,"Send mailbox segment timeout\n" );dump_mox_reg (hwdev); +hinic_dump_aeq_info (hwdev);return -ETIMEDOUT ; }
@@ -897,6 +898,7 @@ int hinic_mbox_to_func(struct hinic_mbox_func_to_func *func_to_func,set_mbox_to_func_event (func_to_func,EVENT_TIMEOUT );dev_err (&func_to_func->hwif->pdev->dev,"Send mbox msg timeout, msg_id: %d\n" , msg_info.msg_id); +hinic_dump_aeq_info (func_to_func->hwdev); err = -ETIMEDOUT ; goto err_send_mbox; }
drivers/net/ethernet/huawei/hinic/hinic_hw_mgmt.c +1 -0
@@ -274,6 +274,7 @@ static int msg_to_mgmt_sync(struct hinic_pf_to_mgmt *pf_to_mgmt,if (!wait_for_completion_timeout (recv_done, timeo)) {dev_err (&pdev->dev,"MGMT timeout, MSG id = %d\n" , msg_id); +hinic_dump_aeq_info (pf_to_mgmt->hwdev); err = -ETIMEDOUT ; goto unlock_sync_msg; }
drivers/net/ethernet/huawei/hinic/hinic_port.c +31 -31
135 lines not on this page · raw message
drivers/net/ethernet/huawei/hinic/hinic_sriov.c +3 -3
@@ -40,9 +40,9 @@ static int hinic_set_mac(struct hinic_hwdev *hwdev, const u8 *mac_addr,if (err || out_size !=sizeof (mac_info) || (mac_info.status && mac_info.status !=HINIC_PF_SET_VF_ALREADY && mac_info.status !=HINIC_MGMT_STATUS_EXIST )) { -dev_err (&hwdev->func_to_io.hwif->pdev->dev,"Failed to change MAC, ret = %d\n" , - mac_info.status); -return -EFAULT ; +dev_err (&hwdev->func_to_io.hwif->pdev->dev,"Failed to set MAC, err: %d, status: 0x%x, out size: 0x%x\n" , + err, mac_info.status, out_size); +return -EIO ; }return 0 ;
Signature · 1 line
2.17.1