Skip to contentSkip to conversation

refactor irq allocation in ice

PATCH New Michal Swiatkowski 3 participants 6 messages Dec 22, 2021 – Apr 27, 2022

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover refactor irq allocation in ice

The ice driver uses the old PCI irq reseveration API. Change the ice driver to use the current API.

Implement a fallback mechanism where, if the driver can't reserve the maximum number of interrupts, it will limit the number of queues or disable capabilities.

First two patches add ability to turn on and off eswitch offload. This is needed when driver can't reserve maximum number of interrupts. In this case driver turns off eswitch offload as driver can work without it. Additionally, the eswitch can be supported only if SRIOV is available, so set eswitch capabilities only if SRIOV is supported.

Michal Swiatkowski (3):
ice: add check for eswitch support
ice: change mode only if eswitch is supported
ice: use new alloc irqs API

 drivers/net/ethernet/intel/ice/Makefile      |   3 +-
 drivers/net/ethernet/intel/ice/ice.h         |   4 +-
 drivers/net/ethernet/intel/ice/ice_arfs.c    |   3 +-
 drivers/net/ethernet/intel/ice/ice_eswitch.c |  46 +++-
 drivers/net/ethernet/intel/ice/ice_eswitch.h |  12 +
 drivers/net/ethernet/intel/ice/ice_irq.c     | 213 ++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_irq.h     |  12 +
 drivers/net/ethernet/intel/ice/ice_lib.c     |   5 +-
 drivers/net/ethernet/intel/ice/ice_main.c    | 220 +------------------
 drivers/net/ethernet/intel/ice/ice_xsk.c     |   3 +-
 10 files changed, 300 insertions(+), 221 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.h
Signature · 1 line
2.31.1
#1 lore raw

PATCH 1/3 ice: add check for eswitch support

Driver support eswitch mode if there is SRIOV capabilities on hardware and CONFIG_ICE_SWITCHDEV is on.

Create function to check if it is supported. Use it in MSI-X reservation to not reserve additional vector when eswitch isn't supported.

Introduce new capability flags to allow driver to disable eswitch support when MSI-X reservation fails.

---
 drivers/net/ethernet/intel/ice/ice.h         |  1 +
 drivers/net/ethernet/intel/ice/ice_eswitch.c | 29 ++++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_eswitch.h | 12 ++++++++
 drivers/net/ethernet/intel/ice/ice_main.c    | 15 ++++++----
 4 files changed, 52 insertions(+), 5 deletions(-)
drivers/net/ethernet/intel/ice/ice.h +1 -0
index 0b9b5b9c24b6..1ca309feabbf 100644
@@ -488,6 +488,7 @@ enum ice_pf_flags {
 	ICE_FLAG_MDD_AUTO_RESET_VF,
 	ICE_FLAG_VF_VLAN_PRUNING,
 	ICE_FLAG_LINK_LENIENT_MODE_ENA,
+	ICE_FLAG_ESWITCH_CAPABLE,
 	ICE_FLAG_GNSS,			/* GNSS successfully initialized */
 	ICE_PF_FLAGS_NBITS		/* must be last */
 };
drivers/net/ethernet/intel/ice/ice_eswitch.c +29 -0
index 30a00fe59c52..fbe640d501c6 100644
@@ -678,3 +678,32 @@ int ice_eswitch_rebuild(struct ice_pf *pf)
 
 	return 0;
 }
+
+/**
+ * ice_is_eswitch_supported - check if eswitch can be supported
+ * @pf: pointer to PF structure
+ */
+bool ice_is_eswitch_supported(struct ice_pf *pf)
+{
+	return test_bit(ICE_FLAG_ESWITCH_CAPABLE, pf->flags);
+}
+
+/**
+ * ice_eswitch_set_cap - set eswitch cap based on SRIOV cap
+ * @pf: pointer to PF structure
+ */
+void ice_eswitch_set_cap(struct ice_pf *pf)
+{
+	clear_bit(ICE_FLAG_ESWITCH_CAPABLE, pf->flags);
+	if (test_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags))
+		set_bit(ICE_FLAG_ESWITCH_CAPABLE, pf->flags);
+}
+
+/**
+ * ice_eswitch_clear_cap - clear switchdev cap when driver can't support it
+ * @pf: pointer to PF structure
+ */
+void ice_eswitch_clear_cap(struct ice_pf *pf)
+{
+	clear_bit(ICE_FLAG_ESWITCH_CAPABLE, pf->flags);
+}
drivers/net/ethernet/intel/ice/ice_eswitch.h +12 -0
index 0d0fadaf2ba5..b405e1c9c2bc 100644
@@ -29,6 +29,9 @@ void ice_eswitch_set_target_vsi(struct sk_buff *skb,
 				struct ice_tx_offload_params *off);
 netdev_tx_t
 ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev);
+bool ice_is_eswitch_supported(struct ice_pf *pf);
+void ice_eswitch_set_cap(struct ice_pf *pf);
+void ice_eswitch_clear_cap(struct ice_pf *pf);
 #else /* CONFIG_ICE_SWITCHDEV */
 static inline void ice_eswitch_release(struct ice_pf *pf) { }
 
@@ -36,6 +39,9 @@ static inline void ice_eswitch_stop_all_tx_queues(struct ice_pf *pf) { }
 static inline void ice_eswitch_replay_vf_mac_rule(struct ice_vf *vf) { }
 static inline void ice_eswitch_del_vf_mac_rule(struct ice_vf *vf) { }
 
+static inline void ice_eswitch_set_cap(struct ice_pf *pf) { }
+static inline void ice_eswitch_clear_cap(struct ice_pf *pf) { }
+
 static inline int
 ice_eswitch_add_vf_mac_rule(struct ice_pf *pf, struct ice_vf *vf,
 			    const u8 *mac)
@@ -81,5 +87,11 @@ ice_eswitch_port_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 {
 	return NETDEV_TX_BUSY;
 }
+
+static inline bool
+ice_is_eswitch_supported(struct ice_pf *pf)
+{
+	return false;
+}
 #endif /* CONFIG_ICE_SWITCHDEV */
 #endif /* _ICE_ESWITCH_H_ */
drivers/net/ethernet/intel/ice/ice_main.c +10 -5
index 296c4dd90e26..e31c01673d3a 100644
@@ -3735,6 +3735,8 @@ static void ice_set_pf_caps(struct ice_pf *pf)
 	if (func_caps->common_cap.ieee_1588)
 		set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
 
+	ice_eswitch_set_cap(pf);
+
 	pf->max_pf_txqs = func_caps->common_cap.num_txq;
 	pf->max_pf_rxqs = func_caps->common_cap.num_rxq;
 }
@@ -3810,11 +3812,13 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 	}
 
 	/* reserve for switchdev */
-	needed = ICE_ESWITCH_MSIX;
-	if (v_left < needed)
-		goto no_hw_vecs_left_err;
-	v_budget += needed;
-	v_left -= needed;
+	if (ice_is_eswitch_supported(pf)) {
+		needed = ICE_ESWITCH_MSIX;
+		if (v_left < needed)
+			goto no_hw_vecs_left_err;
+		v_budget += needed;
+		v_left -= needed;
+	}
 
 	/* total used for non-traffic vectors */
 	v_other = v_budget;
@@ -3920,6 +3924,7 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 		needed, v_left);
 	err = -ERANGE;
 exit_err:
+	ice_eswitch_clear_cap(pf);
 	pf->num_rdma_msix = 0;
 	pf->num_lan_msix = 0;
 	return err;
Signature · 1 line
2.31.1
#2 lore raw

PATCH 2/3 ice: change mode only if eswitch is supported

In reply to Michal Swiatkowski

Support for eswitch can be turned off when reservation for msix fails. Cover this situation and return error in changing eswitch mode function.

Add new lines in each dev_info call.

---
 drivers/net/ethernet/intel/ice/ice_eswitch.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
drivers/net/ethernet/intel/ice/ice_eswitch.c +13 -4
index fbe640d501c6..2b6c7d27ac96 100644
@@ -518,25 +518,34 @@ ice_eswitch_mode_set(struct devlink *devlink, u16 mode,
 		     struct netlink_ext_ack *extack)
 {
 	struct ice_pf *pf = devlink_priv(devlink);
+	struct device *dev = ice_pf_to_dev(pf);
 
 	if (pf->eswitch_mode == mode)
 		return 0;
 
 	if (pf->num_alloc_vfs) {
-		dev_info(ice_pf_to_dev(pf), "Changing eswitch mode is allowed only if there is no VFs created");
-		NL_SET_ERR_MSG_MOD(extack, "Changing eswitch mode is allowed only if there is no VFs created");
+		dev_info(dev, "Changing eswitch mode is allowed only if there is no VFs created\n");
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Changing eswitch mode is allowed only if there is no VFs created");
+		return -EOPNOTSUPP;
+	}
+
+	if (!ice_is_eswitch_supported(pf)) {
+		dev_info(dev, "There is no eswitch support or eswitch resource allocation failed\n");
+		NL_SET_ERR_MSG_MOD(extack,
+				   "There is no eswitch support or eswitch resource allocation failed");
 		return -EOPNOTSUPP;
 	}
 
 	switch (mode) {
 	case DEVLINK_ESWITCH_MODE_LEGACY:
-		dev_info(ice_pf_to_dev(pf), "PF %d changed eswitch mode to legacy",
+		dev_info(ice_pf_to_dev(pf), "PF %d changed eswitch mode to legacy\n",
 			 pf->hw.pf_id);
 		NL_SET_ERR_MSG_MOD(extack, "Changed eswitch mode to legacy");
 		break;
 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
 	{
-		dev_info(ice_pf_to_dev(pf), "PF %d changed eswitch mode to switchdev",
+		dev_info(ice_pf_to_dev(pf), "PF %d changed eswitch mode to switchdev\n",
 			 pf->hw.pf_id);
 		NL_SET_ERR_MSG_MOD(extack, "Changed eswitch mode to switchdev");
 		break;
Signature · 1 line
2.31.1
#3 lore raw

PATCH 3/3 ice: use new alloc irqs API

In reply to Michal Swiatkowski

Move code related to allocating and managing irq to new file to separate it from ice_main.

In new API system vectors number are tracked in kernel and driver can get it by calling pci_irq_vector. There is no need to track these values on driver site.

As there is no function to get exact number of irqs rewrite enabling irqs function to adjust number of irqs based on value returned from kernel.

---
 drivers/net/ethernet/intel/ice/Makefile   |   3 +-
 drivers/net/ethernet/intel/ice/ice.h      |   3 +-
 drivers/net/ethernet/intel/ice/ice_arfs.c |   3 +-
 drivers/net/ethernet/intel/ice/ice_irq.c  | 213 +++++++++++++++++++++
 drivers/net/ethernet/intel/ice/ice_irq.h  |  12 ++
 drivers/net/ethernet/intel/ice/ice_lib.c  |   5 +-
 drivers/net/ethernet/intel/ice/ice_main.c | 221 +---------------------
 drivers/net/ethernet/intel/ice/ice_xsk.c  |   3 +-
 8 files changed, 243 insertions(+), 220 deletions(-)
 create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.c
 create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.h
drivers/net/ethernet/intel/ice/Makefile +2 -1
index 8771ac8460a7..c69da65d6f31 100644
@@ -32,7 +32,8 @@ ice-y := ice_main.o	\
 	 ice_lag.o	\
 	 ice_ethtool.o  \
 	 ice_repr.o	\
-	 ice_tc_lib.o
+	 ice_tc_lib.o	\
+	 ice_irq.o
 ice-$(CONFIG_PCI_IOV) += ice_virtchnl_allowlist.o
 ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o ice_sriov.o ice_virtchnl_fdir.o ice_vf_vsi_vlan_ops.o
 ice-$(CONFIG_PTP_1588_CLOCK) += ice_ptp.o ice_ptp_hw.o
drivers/net/ethernet/intel/ice/ice.h +2 -1
index 1ca309feabbf..1c6437b4dea2 100644
@@ -92,6 +92,8 @@
 #define ICE_MIN_LAN_OICR_MSIX	1
 #define ICE_MIN_MSIX		(ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_LAN_OICR_MSIX)
 #define ICE_FDIR_MSIX		2
+#define ICE_MIN_LAN_MSIX        1
+#define ICE_OICR_MSIX           1
 #define ICE_RDMA_NUM_AEQ_MSIX	4
 #define ICE_MIN_RDMA_MSIX	2
 #define ICE_ESWITCH_MSIX	1
@@ -517,7 +519,6 @@ struct ice_pf {
 	struct devlink_port devlink_port;
 
 	/* OS reserved IRQ details */
-	struct msix_entry *msix_entries;
 	struct ice_res_tracker *irq_tracker;
 	/* First MSIX vector used by SR-IOV VFs. Calculated by subtracting the
 	 * number of MSIX vectors needed for all SR-IOV VFs from the number of
drivers/net/ethernet/intel/ice/ice_arfs.c +2 -1
index 5daade32ea62..5bdfabd14c83 100644
@@ -2,6 +2,7 @@
 /* Copyright (C) 2018-2020, Intel Corporation. */
 
 #include "ice.h"
+#include "ice_irq.h"
 
 /**
  * ice_is_arfs_active - helper to check is aRFS is active
@@ -616,7 +617,7 @@ int ice_set_cpu_rx_rmap(struct ice_vsi *vsi)
 	base_idx = vsi->base_vector;
 	ice_for_each_q_vector(vsi, i)
 		if (irq_cpu_rmap_add(netdev->rx_cpu_rmap,
-				     pf->msix_entries[base_idx + i].vector)) {
+				     ice_get_irq_num(pf, base_idx + 1))) {
 			ice_free_cpu_rx_rmap(vsi);
 			return -EINVAL;
 		}
drivers/net/ethernet/intel/ice/ice_irq.c new file +213 -0

213 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_irq.h new file +12 -0
new file mode 100644
index 000000000000..f4db6964f9c7
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2021, Intel Corporation. */
+
+#ifndef _ICE_IRQ_H_
+#define _ICE_IRQ_H_
+
+int ice_init_interrupt_scheme(struct ice_pf *pf);
+void ice_clear_interrupt_scheme(struct ice_pf *pf);
+
+int ice_get_irq_num(struct ice_pf *pf, int idx);
+
+#endif
drivers/net/ethernet/intel/ice/ice_lib.c +3 -2
index 4e9efd49c149..6e7d121b6746 100644
@@ -9,6 +9,7 @@
 #include "ice_dcb_lib.h"
 #include "ice_devlink.h"
 #include "ice_vsi_vlan_ops.h"
+#include "ice_irq.h"
 
 /**
  * ice_vsi_type_str - maps VSI type enum to string equivalents
@@ -2659,7 +2660,7 @@ void ice_vsi_free_irq(struct ice_vsi *vsi)
 		u16 vector = i + base;
 		int irq_num;
 
-		irq_num = pf->msix_entries[vector].vector;
+		irq_num = ice_get_irq_num(pf, vector);
 
 		/* free only the irqs that were actually requested */
 		if (!vsi->q_vectors[i] ||
@@ -2837,7 +2838,7 @@ void ice_vsi_dis_irq(struct ice_vsi *vsi)
 		return;
 
 	ice_for_each_q_vector(vsi, i)
-		synchronize_irq(pf->msix_entries[i + base].vector);
+		synchronize_irq(ice_get_irq_num(pf, i + base));
 }
 
 /**
drivers/net/ethernet/intel/ice/ice_main.c +7 -214

263 lines not on this page · raw message

drivers/net/ethernet/intel/ice/ice_xsk.c +2 -1
index 0749f0e7a11c..d32571e2abb4 100644
@@ -11,6 +11,7 @@
 #include "ice_txrx.h"
 #include "ice_txrx_lib.h"
 #include "ice_lib.h"
+#include "ice_irq.h"
 
 static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx)
 {
@@ -94,7 +95,7 @@ ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring,
 
 		wr32(hw, GLINT_DYN_CTL(q_vector->reg_idx), 0);
 		ice_flush(hw);
-		synchronize_irq(pf->msix_entries[v_idx + base].vector);
+		synchronize_irq(ice_get_irq_num(pf, v_idx + base));
 	}
 }
 
Signature · 1 line
2.31.1
#4 lore raw

cover refactor irq allocation in ice

In reply to Michal Swiatkowski
Quoted 29 lines · Michal Swiatkowski

The ice driver uses the old PCI irq reseveration API. Change the ice driver to use the current API.

Implement a fallback mechanism where, if the driver can't reserve the maximum number of interrupts, it will limit the number of queues or disable capabilities.

First two patches add ability to turn on and off eswitch offload. This is needed when driver can't reserve maximum number of interrupts. In this case driver turns off eswitch offload as driver can work without it. Additionally, the eswitch can be supported only if SRIOV is available, so set eswitch capabilities only if SRIOV is supported.

Michal Swiatkowski (3):
ice: add check for eswitch support
ice: change mode only if eswitch is supported
ice: use new alloc irqs API

  drivers/net/ethernet/intel/ice/Makefile      |   3 +-
  drivers/net/ethernet/intel/ice/ice.h         |   4 +-
  drivers/net/ethernet/intel/ice/ice_arfs.c    |   3 +-
  drivers/net/ethernet/intel/ice/ice_eswitch.c |  46 +++-
  drivers/net/ethernet/intel/ice/ice_eswitch.h |  12 +
  drivers/net/ethernet/intel/ice/ice_irq.c     | 213 ++++++++++++++++++
  drivers/net/ethernet/intel/ice/ice_irq.h     |  12 +
  drivers/net/ethernet/intel/ice/ice_lib.c     |   5 +-
  drivers/net/ethernet/intel/ice/ice_main.c    | 220 +------------------
  drivers/net/ethernet/intel/ice/ice_xsk.c     |   3 +-
  10 files changed, 300 insertions(+), 221 deletions(-)
Quoted 3 lines

create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.c create mode 100644 drivers/net/ethernet/intel/ice/ice_irq.h

maintainer
#5 lore raw

cover refactor irq allocation in ice

In reply to Michal Swiatkowski
Quoted 6 lines · Michal Swiatkowski

The ice driver uses the old PCI irq reseveration API. Change the ice driver to use the current API.

Implement a fallback mechanism where, if the driver can't reserve the maximum number of interrupts, it will limit the number of queues or disable capabilities.

A very similar implementation was already rejected by netdev [1]. I believe there's kernel work being done for dynamic MSIX allocations which is the route we should look to take on this (when it's completed).

Thanks,

Tony

[1]
https://lore.kernel.org/netdev/20210113234226.3638426-1-anthony.l.nguyen at intel.com/