refactor irq allocation in ice
Subsystems (2)
cover refactor irq allocation in ice
The ice driver uses the old PCI irq reseveration API. Change the ice driver to use the current API.
Implement a fallback mechanism where, if the driver can't reserve the maximum number of interrupts, it will limit the number of queues or disable capabilities.
First two patches add ability to turn on and off eswitch offload. This is needed when driver can't reserve maximum number of interrupts. In this case driver turns off eswitch offload as driver can work without it. Additionally, the eswitch can be supported only if SRIOV is available, so set eswitch capabilities only if SRIOV is supported.
Michal Swiatkowski (3):
ice: add check for eswitch support
ice: change mode only if eswitch is supported
ice: use new alloc irqs API
drivers/net/ethernet/intel/ice/Makefile | 3 +- drivers/net/ethernet/intel/ice/ice.h | 4 +- drivers/net/ethernet/intel/ice/ice_arfs.c | 3 +- drivers/net/ethernet/intel/ice/ice_eswitch.c | 46 +++- drivers/net/ethernet/intel/ice/ice_eswitch.h | 12 + drivers/net/ethernet/intel/ice/ice_irq.c | 213 ++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_irq.h | 12 + drivers/net/ethernet/intel/ice/ice_lib.c | 5 +- drivers/net/ethernet/intel/ice/ice_main.c | 220 +------------------ drivers/net/ethernet/intel/ice/ice_xsk.c | 3 +- 10 files changed, 300 insertions(+), 221 deletions(-) create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.c create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.h
Signature · 1 line
2.31.1
PATCH 1/3 ice: add check for eswitch support
Driver support eswitch mode if there is SRIOV capabilities on hardware and CONFIG_ICE_SWITCHDEV is on.
Create function to check if it is supported. Use it in MSI-X reservation to not reserve additional vector when eswitch isn't supported.
Introduce new capability flags to allow driver to disable eswitch support when MSI-X reservation fails.
--- drivers/net/ethernet/intel/ice/ice.h | 1 + drivers/net/ethernet/intel/ice/ice_eswitch.c | 29 ++++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_eswitch.h | 12 ++++++++ drivers/net/ethernet/intel/ice/ice_main.c | 15 ++++++---- 4 files changed, 52 insertions(+), 5 deletions(-)
drivers/net/ethernet/intel/ice/ice.h +1 -0
@@ -488,6 +488,7 @@ enum ice_pf_flags {ICE_FLAG_MDD_AUTO_RESET_VF ,ICE_FLAG_VF_VLAN_PRUNING ,ICE_FLAG_LINK_LENIENT_MODE_ENA , +ICE_FLAG_ESWITCH_CAPABLE ,ICE_FLAG_GNSS ,/* GNSS successfully initialized */ ICE_PF_FLAGS_NBITS /* must be last */ };
drivers/net/ethernet/intel/ice/ice_eswitch.c +29 -0
@@ -678,3 +678,32 @@ int ice_eswitch_rebuild(struct ice_pf *pf)return 0 ; } + +/** +* ice_is_eswitch_supported - check if eswitch can be supported +* @pf: pointer to PF structure +*/ +bool ice_is_eswitch_supported (struct ice_pf *pf) +{ +return test_bit (ICE_FLAG_ESWITCH_CAPABLE , pf->flags); +} + +/** +* ice_eswitch_set_cap - set eswitch cap based on SRIOV cap +* @pf: pointer to PF structure +*/ +void ice_eswitch_set_cap (struct ice_pf *pf) +{ +clear_bit (ICE_FLAG_ESWITCH_CAPABLE , pf->flags); +if (test_bit (ICE_FLAG_SRIOV_CAPABLE , pf->flags)) +set_bit (ICE_FLAG_ESWITCH_CAPABLE , pf->flags); +} + +/** +* ice_eswitch_clear_cap - clear switchdev cap when driver can't support it +* @pf: pointer to PF structure +*/ +void ice_eswitch_clear_cap (struct ice_pf *pf) +{ +clear_bit (ICE_FLAG_ESWITCH_CAPABLE , pf->flags); +}
drivers/net/ethernet/intel/ice/ice_eswitch.h +12 -0
@@ -29,6 +29,9 @@ void ice_eswitch_set_target_vsi(struct sk_buff *skb, struct ice_tx_offload_params *off);netdev_tx_t ice_eswitch_port_start_xmit (struct sk_buff *skb,struct net_device *netdev); +bool ice_is_eswitch_supported (struct ice_pf *pf); +void ice_eswitch_set_cap (struct ice_pf *pf); +void ice_eswitch_clear_cap (struct ice_pf *pf);#else /* CONFIG_ICE_SWITCHDEV */ static inline void ice_eswitch_release (struct ice_pf *pf) { }
@@ -36,6 +39,9 @@ static inline void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf) { }static inline void ice_eswitch_replay_vf_mac_rule (struct ice_vf *vf) { }static inline void ice_eswitch_del_vf_mac_rule (struct ice_vf *vf) { } +static inline void ice_eswitch_set_cap (struct ice_pf *pf) { } +static inline void ice_eswitch_clear_cap (struct ice_pf *pf) { } +static inline int ice_eswitch_add_vf_mac_rule (struct ice_pf *pf,struct ice_vf *vf,const u8 *mac)
@@ -81,5 +87,11 @@ ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev) {return NETDEV_TX_BUSY ; } + +static inline bool +ice_is_eswitch_supported (struct ice_pf *pf) +{ +return false; +}#endif /* CONFIG_ICE_SWITCHDEV */ #endif /* _ICE_ESWITCH_H_ */
drivers/net/ethernet/intel/ice/ice_main.c +10 -5
@@ -3735,6 +3735,8 @@ static void ice_set_pf_caps(struct ice_pf *pf)if (func_caps->common_cap.ieee_1588)set_bit (ICE_FLAG_PTP_SUPPORTED , pf->flags); +ice_eswitch_set_cap (pf); + pf->max_pf_txqs = func_caps->common_cap.num_txq; pf->max_pf_rxqs = func_caps->common_cap.num_rxq; }
@@ -3810,11 +3812,13 @@ static int ice_ena_msix_range(struct ice_pf *pf) }/* reserve for switchdev */ - needed =ICE_ESWITCH_MSIX ; -if (v_left < needed) - goto no_hw_vecs_left_err; - v_budget += needed; - v_left -= needed; +if (ice_is_eswitch_supported (pf)) { + needed =ICE_ESWITCH_MSIX ; +if (v_left < needed) + goto no_hw_vecs_left_err; + v_budget += needed; + v_left -= needed; + }/* total used for non-traffic vectors */ v_other = v_budget;
@@ -3920,6 +3924,7 @@ static int ice_ena_msix_range(struct ice_pf *pf) needed, v_left); err = -ERANGE ; exit_err: +ice_eswitch_clear_cap (pf); pf->num_rdma_msix =0 ; pf->num_lan_msix =0 ;return err;
Signature · 1 line
2.31.1
PATCH 2/3 ice: change mode only if eswitch is supported
In reply to Michal SwiatkowskiSupport for eswitch can be turned off when reservation for msix fails. Cover this situation and return error in changing eswitch mode function.
Add new lines in each dev_info call.
--- drivers/net/ethernet/intel/ice/ice_eswitch.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)
drivers/net/ethernet/intel/ice/ice_eswitch.c +13 -4
@@ -518,25 +518,34 @@ ice_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) {struct ice_pf *pf =devlink_priv (devlink); +struct device *dev =ice_pf_to_dev (pf);if (pf->eswitch_mode == mode)return 0 ;if (pf->num_alloc_vfs) { -dev_info (ice_pf_to_dev (pf),"Changing eswitch mode is allowed only if there is no VFs created" ); -NL_SET_ERR_MSG_MOD (extack,"Changing eswitch mode is allowed only if there is no VFs created" ); +dev_info (dev,"Changing eswitch mode is allowed only if there is no VFs created\n" ); +NL_SET_ERR_MSG_MOD (extack, +"Changing eswitch mode is allowed only if there is no VFs created" ); +return -EOPNOTSUPP ; + } + +if (!ice_is_eswitch_supported (pf)) { +dev_info (dev,"There is no eswitch support or eswitch resource allocation failed\n" ); +NL_SET_ERR_MSG_MOD (extack, +"There is no eswitch support or eswitch resource allocation failed" );return -EOPNOTSUPP ; }switch (mode) {case DEVLINK_ESWITCH_MODE_LEGACY : -dev_info (ice_pf_to_dev (pf),"PF %d changed eswitch mode to legacy" , +dev_info (ice_pf_to_dev (pf),"PF %d changed eswitch mode to legacy\n" , pf->hw.pf_id);NL_SET_ERR_MSG_MOD (extack,"Changed eswitch mode to legacy" );break ;case DEVLINK_ESWITCH_MODE_SWITCHDEV : { -dev_info (ice_pf_to_dev (pf),"PF %d changed eswitch mode to switchdev" , +dev_info (ice_pf_to_dev (pf),"PF %d changed eswitch mode to switchdev\n" , pf->hw.pf_id);NL_SET_ERR_MSG_MOD (extack,"Changed eswitch mode to switchdev" );break ;
Signature · 1 line
2.31.1
PATCH 3/3 ice: use new alloc irqs API
In reply to Michal SwiatkowskiMove code related to allocating and managing irq to new file to separate it from ice_main.
In new API system vectors number are tracked in kernel and driver can get it by calling pci_irq_vector. There is no need to track these values on driver site.
As there is no function to get exact number of irqs rewrite enabling irqs function to adjust number of irqs based on value returned from kernel.
--- drivers/net/ethernet/intel/ice/Makefile | 3 +- drivers/net/ethernet/intel/ice/ice.h | 3 +- drivers/net/ethernet/intel/ice/ice_arfs.c | 3 +- drivers/net/ethernet/intel/ice/ice_irq.c | 213 +++++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_irq.h | 12 ++ drivers/net/ethernet/intel/ice/ice_lib.c | 5 +- drivers/net/ethernet/intel/ice/ice_main.c | 221 +--------------------- drivers/net/ethernet/intel/ice/ice_xsk.c | 3 +- 8 files changed, 243 insertions(+), 220 deletions(-) create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.c create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.h
drivers/net/ethernet/intel/ice/Makefile +2 -1
@@ -32,7 +32,8 @@ ice-y := ice_main.o \ ice_lag.o \ ice_ethtool.o \ ice_repr.o \ - ice_tc_lib.o + ice_tc_lib.o \ + ice_irq.o ice-$(CONFIG_PCI_IOV) += ice_virtchnl_allowlist.o ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o ice_virtchnl_fdir.o ice_vf_vsi_vlan_ops.o ice-$(CONFIG_PTP_1588_CLOCK) += ice_ptp.o ice_ptp_hw.o
drivers/net/ethernet/intel/ice/ice.h +2 -1
@@ -92,6 +92,8 @@#define ICE_MIN_LAN_OICR_MSIX 1#define ICE_MIN_MSIX (ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)#define ICE_FDIR_MSIX 2 +#define ICE_MIN_LAN_MSIX 1 +#define ICE_OICR_MSIX 1#define ICE_RDMA_NUM_AEQ_MSIX 4#define ICE_MIN_RDMA_MSIX 2#define ICE_ESWITCH_MSIX 1
@@ -517,7 +519,6 @@ struct ice_pf {struct devlink_port devlink_port;/* OS reserved IRQ details */ -struct msix_entry *msix_entries;struct ice_res_tracker *irq_tracker;/* First MSIX vector used by SR-IOV VFs. Calculated by subtracting the * number of MSIX vectors needed for all SR-IOV VFs from the number of
drivers/net/ethernet/intel/ice/ice_arfs.c +2 -1
@@ -2,6 +2,7 @@/* Copyright (C) 2018-2020, Intel Corporation. */ #include "ice.h" +#include "ice_irq.h" /** * ice_is_arfs_active - helper to check is aRFS is active
@@ -616,7 +617,7 @@ int ice_set_cpu_rx_rmap(struct ice_vsi *vsi) base_idx = vsi->base_vector;ice_for_each_q_vector (vsi, i)if (irq_cpu_rmap_add (netdev->rx_cpu_rmap, - pf->msix_entries[base_idx + i].vector)) { +ice_get_irq_num (pf, base_idx +1 ))) {ice_free_cpu_rx_rmap (vsi);return -EINVAL ; }
drivers/net/ethernet/intel/ice/ice_irq.c new file +213 -0
213 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_irq.h new file +12 -0
@@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2021, Intel Corporation. */ + +#ifndef _ICE_IRQ_H_ +#define _ICE_IRQ_H_ + +int ice_init_interrupt_scheme (struct ice_pf *pf); +void ice_clear_interrupt_scheme (struct ice_pf *pf); + +int ice_get_irq_num (struct ice_pf *pf,int idx); + +#endif
drivers/net/ethernet/intel/ice/ice_lib.c +3 -2
@@ -9,6 +9,7 @@#include "ice_dcb_lib.h" #include "ice_devlink.h" #include "ice_vsi_vlan_ops.h" +#include "ice_irq.h" /** * ice_vsi_type_str - maps VSI type enum to string equivalents
@@ -2659,7 +2660,7 @@ void ice_vsi_free_irq(struct ice_vsi *vsi)u16 vector = i + base;int irq_num; - irq_num = pf->msix_entries[vector].vector; + irq_num =ice_get_irq_num (pf, vector);/* free only the irqs that were actually requested */ if (!vsi->q_vectors[i] ||
@@ -2837,7 +2838,7 @@ void ice_vsi_dis_irq(struct ice_vsi *vsi)return ;ice_for_each_q_vector (vsi, i) -synchronize_irq (pf->msix_entries[i + base].vector); +synchronize_irq (ice_get_irq_num (pf, i + base)); }/**
drivers/net/ethernet/intel/ice/ice_main.c +7 -214
263 lines not on this page · raw message
drivers/net/ethernet/intel/ice/ice_xsk.c +2 -1
@@ -11,6 +11,7 @@#include "ice_txrx.h" #include "ice_txrx_lib.h" #include "ice_lib.h" +#include "ice_irq.h" static struct xdp_buff **ice_xdp_buf (struct ice_rx_ring *rx_ring,u32 idx) {
@@ -94,7 +95,7 @@ ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring,wr32 (hw,GLINT_DYN_CTL (q_vector->reg_idx),0 );ice_flush (hw); -synchronize_irq (pf->msix_entries[v_idx + base].vector); +synchronize_irq (ice_get_irq_num (pf, v_idx + base)); } }
Signature · 1 line
2.31.1
cover refactor irq allocation in ice
In reply to Michal SwiatkowskiQuoted 29 lines · Michal Swiatkowski
The ice driver uses the old PCI irq reseveration API. Change the ice driver to use the current API.
Implement a fallback mechanism where, if the driver can't reserve the maximum number of interrupts, it will limit the number of queues or disable capabilities.
First two patches add ability to turn on and off eswitch offload. This is needed when driver can't reserve maximum number of interrupts. In this case driver turns off eswitch offload as driver can work without it. Additionally, the eswitch can be supported only if SRIOV is available, so set eswitch capabilities only if SRIOV is supported.
Michal Swiatkowski (3):
ice: add check for eswitch support
ice: change mode only if eswitch is supported
ice: use new alloc irqs APIdrivers/net/ethernet/intel/ice/Makefile | 3 +- drivers/net/ethernet/intel/ice/ice.h | 4 +- drivers/net/ethernet/intel/ice/ice_arfs.c | 3 +- drivers/net/ethernet/intel/ice/ice_eswitch.c | 46 +++- drivers/net/ethernet/intel/ice/ice_eswitch.h | 12 + drivers/net/ethernet/intel/ice/ice_irq.c | 213 ++++++++++++++++++ drivers/net/ethernet/intel/ice/ice_irq.h | 12 + drivers/net/ethernet/intel/ice/ice_lib.c | 5 +- drivers/net/ethernet/intel/ice/ice_main.c | 220 +------------------ drivers/net/ethernet/intel/ice/ice_xsk.c | 3 +- 10 files changed, 300 insertions(+), 221 deletions(-)
Quoted 3 lines
create mode 100644
drivers/net/ethernet/intel/ice/ice_irq.ccreate mode 100644drivers/net/ethernet/intel/ice/ice_irq.h
Tony Nguyen
maintainercover refactor irq allocation in ice
In reply to Michal SwiatkowskiQuoted 6 lines · Michal Swiatkowski
The ice driver uses the old PCI irq reseveration API. Change the ice driver to use the current API.
Implement a fallback mechanism where, if the driver can't reserve the maximum number of interrupts, it will limit the number of queues or disable capabilities.
A very similar implementation was already rejected by netdev [1]. I believe there's kernel work being done for dynamic MSIX allocations which is the route we should look to take on this (when it's completed).
Thanks,
Tony
[1]
https://lore.kernel.org/netdev/20210113234226.3638426-1-anthony.l.nguyen at intel.com/