Skip to contentSkip to conversation

i40e: Add macvlan support on i40e

PATCH Superseded Harshitha Ramamurthy 3 participants 3 messages Jan 9, 2019 – Jan 14, 2019

Subsystems (2)

v2 of 6 Superseded Latest is v6 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 i40e: Add macvlan support on i40e

This patch enables macvlan offloads on Fortville devices. The idea is to use channels as macvlan interfaces. The channels are VSIs of type VMDQ. When the first macvlan is created, the maximum no. of channels possible are created. From then on, as a macvlan interface is created, a mac filter is added to these already created channels (VSIs).

This patch builds on top of the recent changes which move away from the select_queue implementation of picking the tx queue.

Steps to configure the macvlan:
1. sudo ethtool -K ens261f1 l2-fwd-offload on
2. ip link add link ens261f1 name macvlan1 type macvlan
3. sudo ip link add link ens261f1 name macvlan1 type macvlan
4. sudo ip link set macvlan1 up

---
v2: Addressed Shannon's comments
Added a new function to remove all macvlan VSIs

 drivers/net/ethernet/intel/i40e/i40e.h      |  26 ++
 drivers/net/ethernet/intel/i40e/i40e_main.c | 475 +++++++++++++++++++-
 2 files changed, 499 insertions(+), 2 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +26 -0
index c06a4b5cdfae..107edcf2cf19 100644
@@ -27,6 +27,7 @@
 #include <net/ip6_checksum.h>
 #include <linux/ethtool.h>
 #include <linux/if_vlan.h>
+#include <linux/if_macvlan.h>
 #include <linux/if_bridge.h>
 #include <linux/clocksource.h>
 #include <linux/net_tstamp.h>
@@ -391,6 +392,11 @@ struct i40e_flex_pit {
 	u8 pit_index;
 };
 
+struct i40e_fwd_adapter {
+	struct net_device *netdev;
+	int bit_no;
+};
+
 struct i40e_channel {
 	struct list_head list;
 	bool initialized;
@@ -405,11 +411,25 @@ struct i40e_channel {
 	struct i40e_aqc_vsi_properties_data info;
 
 	u64 max_tx_rate;
+	struct i40e_fwd_adapter *fwd;
 
 	/* track this channel belongs to which VSI */
 	struct i40e_vsi *parent_vsi;
 };
 
+static inline bool i40e_is_channel_macvlan(struct i40e_channel *ch)
+{
+	return !!ch->fwd;
+}
+
+static inline u8 *i40e_channel_mac(struct i40e_channel *ch)
+{
+	if (i40e_is_channel_macvlan(ch))
+		return ch->fwd->netdev->dev_addr;
+	else
+		return NULL;
+}
+
 /* struct that defines the Ethernet device */
 struct i40e_pf {
 	struct pci_dev *pdev;
@@ -787,6 +807,12 @@ struct i40e_vsi {
 	struct list_head ch_list;
 	u16 tc_seid_map[I40E_MAX_TRAFFIC_CLASS];
 
+	/* macvlan fields */
+#define I40E_MAX_MACVLANS	128 /* Max HW capable vectors - 1 on FVL */
+	DECLARE_BITMAP(fwd_bitmask, I40E_MAX_MACVLANS);
+	struct list_head macvlan_list;
+	int macvlan_cnt;
+
 	void *priv;	/* client driver data reference. */
 
 	/* VSI specific handlers */
drivers/net/ethernet/intel/i40e/i40e_main.c +473 -2

505 lines not on this page · raw message

Signature · 1 line
2.17.1
#1 lore raw

PATCH v2 1/1 i40e: Add macvlan support on i40e

Quoted 17 lines · Harshitha Ramamurthy

This patch enables macvlan offloads on Fortville devices. The idea is to use channels as macvlan interfaces. The channels are VSIs of type VMDQ. When the first macvlan is created, the maximum no. of channels possible are created. From then on, as a macvlan interface is created, a mac filter is added to these already created channels (VSIs).

This patch builds on top of the recent changes which move away from the select_queue implementation of picking the tx queue.

Steps to configure the macvlan:
1. sudo ethtool -K ens261f1 l2-fwd-offload on
2. ip link add link ens261f1 name macvlan1 type macvlan
3. sudo ip link add link ens261f1 name macvlan1 type macvlan
4. sudo ip link set macvlan1 up

Quoted 4 lines

---
v2: Addressed Shannon's comments
Added a new function to remove all macvlan VSIs

A few more comments below.

Quoted 179 lines
 drivers/net/ethernet/intel/i40e/i40e.h      |  26 ++
 drivers/net/ethernet/intel/i40e/i40e_main.c | 475 +++++++++++++++++++-
 2 files changed, 499 insertions(+), 2 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +26 -0
index c06a4b5cdfae..107edcf2cf19 100644
@@ -27,6 +27,7 @@
 #include <net/ip6_checksum.h>
 #include <linux/ethtool.h>
 #include <linux/if_vlan.h>
+#include <linux/if_macvlan.h>
 #include <linux/if_bridge.h>
 #include <linux/clocksource.h>
 #include <linux/net_tstamp.h>
@@ -391,6 +392,11 @@ struct i40e_flex_pit {
        u8 pit_index;
 };
 
+struct i40e_fwd_adapter {
+       struct net_device *netdev;
+       int bit_no;
+};
+
 struct i40e_channel {
        struct list_head list;
        bool initialized;
@@ -405,11 +411,25 @@ struct i40e_channel {
        struct i40e_aqc_vsi_properties_data info;
 
        u64 max_tx_rate;
+       struct i40e_fwd_adapter *fwd;
 
        /* track this channel belongs to which VSI */
        struct i40e_vsi *parent_vsi;
 };
 
+static inline bool i40e_is_channel_macvlan(struct i40e_channel *ch)
+{
+       return !!ch->fwd;
+}
+
+static inline u8 *i40e_channel_mac(struct i40e_channel *ch)
+{
+       if (i40e_is_channel_macvlan(ch))
+               return ch->fwd->netdev->dev_addr;
+       else
+               return NULL;
+}
+
 /* struct that defines the Ethernet device */
 struct i40e_pf {
        struct pci_dev *pdev;
@@ -787,6 +807,12 @@ struct i40e_vsi {
        struct list_head ch_list;
        u16 tc_seid_map[I40E_MAX_TRAFFIC_CLASS];
 
+       /* macvlan fields */
+#define I40E_MAX_MACVLANS      128 /* Max HW capable vectors - 1 on FVL */
+       DECLARE_BITMAP(fwd_bitmask, I40E_MAX_MACVLANS);
+       struct list_head macvlan_list;
+       int macvlan_cnt;
+
        void *priv;     /* client driver data reference. */
 
        /* VSI specific handlers */
drivers/net/ethernet/intel/i40e/i40e_main.c +94 -2
index 4a129c7ab6f4..6f81db63d25b 100644
@@ -5818,8 +5818,10 @@ static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
                return -ENOENT;
        }
 
-       /* Success, update channel */
-       ch->enabled_tc = enabled_tc;
+       /* Success, update channel, set enabled_tc only if the channel
+        * is not a macvlan
+        */
+       ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
        ch->seid = ctxt.seid;
        ch->vsi_number = ctxt.vsi_number;
        ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx);
@@ -6811,6 +6813,467 @@ static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
        }
 }
 
+/**
+ * i40e_del_macvlan_filter
+ * @hw: pointer to the HW structure
+ * @seid: seid of the channel VSI
+ * @macaddr: the mac address to apply as a filter
+ * @aq_err: store the admin Q error
+ *
+ * This function deletes a mac filter on the channel VSI which serves as the
+ * macvlan. Returns 0 on success.
+ **/
+static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
+                                          const u8 *macaddr, int *aq_err)
+{
+       struct i40e_aqc_remove_macvlan_element_data element;
+       i40e_status status;
+
+       memset(&element, 0, sizeof(element));
+       ether_addr_copy(element.mac_addr, macaddr);
+       element.vlan_tag = 0;
+       element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
+       status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
+       *aq_err = hw->aq.asq_last_status;
+       return status;
+}
+
+/**
+ * i40e_add_macvlan_filter
+ * @hw: pointer to the HW structure
+ * @seid: seid of the channel VSI
+ * @macaddr: the mac address to apply as a filter
+ * @aq_err: store the admin Q error
+ *
+ * This function adds a mac filter on the channel VSI which serves as the
+ * macvlan. Returns 0 on success.
+ **/
+static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
+                                          const u8 *macaddr, int *aq_err)
+{
+       struct i40e_aqc_add_macvlan_element_data element;
+       i40e_status status;
+       u16 cmd_flags = 0;
+
+       ether_addr_copy(element.mac_addr, macaddr);
+       element.vlan_tag = 0;
+       element.queue_number = 0;
+       element.match_method = I40E_AQC_MM_ERR_NO_RES;
+       cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
+       element.flags = cpu_to_le16(cmd_flags);
+       status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
+       *aq_err = hw->aq.asq_last_status;
+       return status;
+}
+
+/**
+ * i40e_reset_ch_rings - Reset the queue contexts in a channel
+ * @vsi: the VSI we want to access
+ * @ch: the channel we want to access
+ */
+static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
+{
+       struct i40e_ring *tx_ring, *rx_ring;
+       u16 pf_q;
+       int i;
+
+       for (i = 0; i < ch->num_queue_pairs; i++) {
+               pf_q = ch->base_queue + i;
+               tx_ring = vsi->tx_rings[pf_q];
+               tx_ring->ch = NULL;
+               rx_ring = vsi->rx_rings[pf_q];
+               rx_ring->ch = NULL;
+       }
+}
+
+/**
+ * i40e_remove_macvlan_channels
+ * @vsi: the VSI we want to access
+ *
+ * This functions removes all the channel VSIs which serve as a macvlan.
+ */
+static void i40e_remove_macvlan_channels(struct i40e_vsi *vsi)
+{
+       struct i40e_channel *ch, *ch_tmp;
+       int ret;
+
+       if (list_empty(&vsi->macvlan_list))
+               return;
+
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
+               struct i40e_vsi *parent_vsi;
+

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 6915

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+               if (i40e_is_channel_macvlan(ch)) {
+                       i40e_reset_ch_rings(vsi, ch);
+                       clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
+                       netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
+                       netdev_set_sb_channel(ch->fwd->netdev, 0);
+                       kfree(ch->fwd);
+                       ch->fwd = NULL;
+               }
+
+               list_del(&ch->list);

It looks like this removes more than just macvlan channels, as the name suggests. It removes all channels, and does a little extra cleaning if the channel is a macvlan. Maybe change the name to i40e_remove_channels()?

Quoted 158 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +158 -0
+               parent_vsi = ch->parent_vsi;
+               if (!parent_vsi || !ch->initialized) {
+                       kfree(ch);
+                       continue;
+               }
+
+               /* remove the VSI */
+               ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
+                                            NULL);
+               if (ret)
+                       dev_err(&vsi->back->pdev->dev,
+                               "unable to remove channel (%d) for parent VSI(%d)\n",
+                               ch->seid, parent_vsi->seid);
+               kfree(ch);
+       }
+       vsi->macvlan_cnt = 0;
+}
+
+/**
+ * i40e_fwd_ring_up - bring the macvlan device up
+ * @vsi: the VSI we want to access
+ * @vdev: macvlan netdevice
+ * @fwd: the private fwd structure
+ */
+static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
+                           struct i40e_fwd_adapter *fwd)
+{
+       int ret = 0, num_tc = 1,  i, aq_err;
+       struct i40e_channel *ch, *ch_tmp;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+
+       if (list_empty(&vsi->macvlan_list))
+               return -EINVAL;
+
+       /* Go through the list and find an available channel */
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
+               if (!i40e_is_channel_macvlan(ch)) {
+                       ch->fwd = fwd;
+                       /* record configuration for macvlan interface in vdev */
+                       for (i = 0; i < num_tc; i++)
+                               netdev_bind_sb_channel_queue(vsi->netdev, vdev,
+                                                            i,
+                                                            ch->num_queue_pairs,
+                                                            ch->base_queue);
+                       for (i = 0; i < ch->num_queue_pairs; i++) {
+                               struct i40e_ring *tx_ring, *rx_ring;
+                               u16 pf_q;
+
+                               pf_q = ch->base_queue + i;
+
+                               /* Get to TX ring ptr */
+                               tx_ring = vsi->tx_rings[pf_q];
+                               tx_ring->ch = ch;
+
+                               /* Get the RX ring ptr */
+                               rx_ring = vsi->rx_rings[pf_q];
+                               rx_ring->ch = ch;
+                       }
+                       break;
+               }
+       }
+
+       /* Guarantee all rings are updated before we update the
+        * MAC address filter.
+        */
+       wmb();
+
+       /* Add a mac filter */
+       ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
+       if (ret) {
+               /* if we cannot add the MAC rule then disable the offload */
+               macvlan_release_l2fw_offload(vdev);
+               for (i = 0; i < ch->num_queue_pairs; i++) {
+                       struct i40e_ring *rx_ring;
+                       u16 pf_q;
+
+                       pf_q = ch->base_queue + i;
+                       rx_ring = vsi->rx_rings[pf_q];
+                       rx_ring->netdev = NULL;
+               }
+               dev_info(&pf->pdev->dev,
+                        "Error adding mac filter on macvlan err %s, aq_err %s\n",
+                         i40e_stat_str(hw, ret),
+                         i40e_aq_str(hw, aq_err));
+               netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
+       }
+       return ret;
+}
+
+/**
+ * i40e_setup_macvlans - create the channels which will be macvlans
+ * @vsi: the VSI we want to access
+ * @macvlan_cnt: no. of macvlans to be setup
+ * @qcnt: no. of Qs per macvlan
+ * @vdev: macvlan netdevice
+ */
+static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
+                              struct net_device *vdev)
+{
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+       struct i40e_vsi_context ctxt;
+       u16 sections, qmap, num_qps;
+       struct i40e_channel *ch;
+       int i, pow, ret = 0;
+       u8 offset = 0;
+
+       if (vsi->type != I40E_VSI_MAIN)
+               return -EINVAL;
+       if (!macvlan_cnt)
+               return -EINVAL;
+
+       num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
+
+       /* find the next higher power-of-2 of num queue pairs */
+       pow = fls(roundup_pow_of_two(num_qps) - 1);
+
+       qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
+               (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
+
+       /* Setup context bits for the main VSI */
+       sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
+       sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
+       memset(&ctxt, 0, sizeof(ctxt));
+       ctxt.seid = vsi->seid;
+       ctxt.pf_num = vsi->back->hw.pf_id;
+       ctxt.vf_num = 0;
+       ctxt.uplink_seid = vsi->uplink_seid;
+       ctxt.info = vsi->info;
+       ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
+       ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
+       ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
+       ctxt.info.valid_sections |= cpu_to_le16(sections);
+
+       /* Reconfigure RSS for main VSI with new max queue count */
+       vsi->rss_size = max_t(u16, num_qps, qcnt);
+       ret = i40e_vsi_config_rss(vsi);
+       if (ret) {
+               dev_info(&pf->pdev->dev,
+                        "Failed to reconfig RSS for num_queues (%u)\n",
+                        vsi->rss_size);
+               return ret;
+       }
+       vsi->reconfig_rss = true;
+       dev_dbg(&vsi->back->pdev->dev,
+               "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
+       vsi->next_base_queue = num_qps;
+       vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
+
+       /* Update the VSI after updating the VSI queue-mapping
+        * information
+        */
+       ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
+       if (ret) {
+               dev_info(&pf->pdev->dev,
+                        "Update vsi tc config failed, err %s aq_err %s\n",
+                        i40e_stat_str(hw, ret),

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7083

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+                        i40e_aq_str(hw, hw->aq.asq_last_status));
+               return ret;
+       }
+       /* update the local VSI info with updated queue map */
+       i40e_vsi_update_queue_map(vsi, &ctxt);
+       vsi->info.valid_sections = 0;
+
+       /* Create channels for macvlans */
+       INIT_LIST_HEAD(&vsi->macvlan_list);
+       vsi->macvlan_cnt = macvlan_cnt;

You might wait until after this for-loop succeeds before setting the vsi->macvlan_cnt; or ig the i40e_remove_macvlan_channels() needs it, increment it each time through the loop at teh same time that you decrement cnt_q_avail.

Quoted 53 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +53 -0
+       for (i = 0; i < macvlan_cnt; i++) {
+               ch = kzalloc(sizeof(*ch), GFP_KERNEL);
+               if (!ch) {
+                       ret = -ENOMEM;
+                       goto err_free;
+               }
+               INIT_LIST_HEAD(&ch->list);
+               ch->num_queue_pairs = qcnt;
+               if (!i40e_setup_channel(pf, vsi, ch)) {
+                       ret = -EINVAL;
+                       goto err_free;
+               }
+               ch->parent_vsi = vsi;
+               vsi->cnt_q_avail -= ch->num_queue_pairs;
+               list_add_tail(&ch->list, &vsi->macvlan_list);
+       }
+       return ret;
+
+err_free:
+       dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
+       i40e_remove_macvlan_channels(vsi);
+       return ret;
+}
+
+/**
+ * i40e_fwd_add - configure macvlans
+ * @netdev: net device to configure
+ * @vdev: macvlan netdevice
+ **/
+static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
+{
+       struct i40e_netdev_priv *np = netdev_priv(netdev);
+       u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
+       struct i40e_vsi *vsi = np->vsi;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_fwd_adapter *fwd;
+       int avail_macvlan, ret;
+
+       if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
+               netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
+               return ERR_PTR(-EINVAL);
+       }
+       if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
+               netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
+               return ERR_PTR(-EINVAL);
+       }
+
+       /* The macvlan device can't be a multiqueue device */
+       if (netif_is_multiqueue(vdev))
+               return ERR_PTR(-ERANGE);
+
+       if (!vsi->macvlan_cnt) {
+               /* reserve bit 0 for the pf device */

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7146

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+               set_bit(0, vsi->fwd_bitmask);
+
+               /* Try to reserve as many queues for macvlans. First reserve
+                *  3/4th of max vectors, then half, then quarter and calculate
+                *  Qs per macvlan as you go
+                */
+               vectors = pf->num_lan_msix;
+               if (vectors <= I40E_MAX_MACVLANS && vectors > 96) {
+                       /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
+                       q_per_macvlan = 4;

If above you disallow multiqueue netdev, why do you allocate more than 1 queue per macvlan?

Quoted 21 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +21 -0
+                       macvlan_cnt = (vectors - 32) / 4;
+               } else if (vectors <= 96 && vectors > 64) {
+                       /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
+                       q_per_macvlan = 4;
+                       macvlan_cnt = (vectors - 32) / 4;
+               } else if (vectors <= 64 && vectors > 32) {
+                       /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
+                       q_per_macvlan = 2;
+                       macvlan_cnt = (vectors - 16) / 2;
+               } else {
+                       /* allocate 1 Q per macvlan 16 Qs to the PF*/
+                       q_per_macvlan = 1;
+                       macvlan_cnt = (vectors - 16);
+               }
+               if (macvlan_cnt == 0)
+                       return ERR_PTR(-EBUSY);
+
+               /* Quiesce VSI queues */
+               i40e_quiesce_vsi(vsi);
+
+               /* sets up the macvlans but does not "enable" them */

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7177

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+               ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
+                                         vdev);
+               if (ret)
+                       return ERR_PTR(ret);
+
+               /* Unquiesce VSI */
+               i40e_unquiesce_vsi(vsi);
+       }
+       avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
+                                           vsi->macvlan_cnt);

Is it possible there are none left?

Quoted 13 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +13 -0
+
+       /* create the fwd struct */
+       fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
+       if (!fwd)
+               return ERR_PTR(-ENOMEM);
+
+       set_bit(avail_macvlan, vsi->fwd_bitmask);
+       fwd->bit_no = avail_macvlan;
+       netdev_set_sb_channel(vdev, avail_macvlan);
+       fwd->netdev = vdev;
+
+       if (!netif_running(netdev))
+               return fwd;

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7200

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+
+       /* Set fwd ring up */
+       ret = i40e_fwd_ring_up(vsi, vdev, fwd);
+       if (ret) {
+               /* unbind the queues and drop the subordinate channel config */
+               netdev_unbind_sb_channel(netdev, vdev);
+               netdev_set_sb_channel(vdev, 0);
+
+               kfree(fwd);
+               return ERR_PTR(-EINVAL);

Do you need to call something like i40e_del_macvlan_filter() here?

Quoted 56 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +56 -0
+       }
+       return fwd;
+}
+
+/**
+ * i40e_del_all_macvlans - Delete all the mac filters on the channels
+ * @vsi: the VSI we want to access
+ */
+static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
+{
+       struct i40e_channel *ch, *ch_tmp;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+       int aq_err, ret = 0;
+
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
+               if (i40e_is_channel_macvlan(ch)) {
+                       ret = i40e_del_macvlan_filter(hw, ch->seid,
+                                                     i40e_channel_mac(ch),
+                                                     &aq_err);
+                       if (!ret) {
+                               /* Reset queue contexts */
+                               i40e_reset_ch_rings(vsi, ch);
+                               clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
+                               netdev_unbind_sb_channel(vsi->netdev,
+                                                        ch->fwd->netdev);
+                               netdev_set_sb_channel(ch->fwd->netdev, 0);
+                               kfree(ch->fwd);
+                               ch->fwd = NULL;
+                       }
+               }
+       }
+}
+
+/**
+ * i40e_fwd_del - delete macvlan interfaces
+ * @netdev: net device to configure
+ * @vdev: macvlan netdevice
+ */
+static void i40e_fwd_del(struct net_device *netdev, void *vdev)
+{
+       struct i40e_netdev_priv *np = netdev_priv(netdev);
+       struct i40e_fwd_adapter *fwd = vdev;
+       struct i40e_channel *ch, *ch_tmp;
+       struct i40e_vsi *vsi = np->vsi;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+       int aq_err, ret = 0;
+
+       /* Find the channel associated with the macvlan and del mac filter */
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
+               if (i40e_is_channel_macvlan(ch) &&
+                   ether_addr_equal(i40e_channel_mac(ch),
+                                    fwd->netdev->dev_addr)) {
+                       ret = i40e_del_macvlan_filter(hw, ch->seid,
+                                                     i40e_channel_mac(ch),

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7266

Quoted 10 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+                                                     &aq_err);
+                       if (!ret) {
+                               /* Reset queue contexts */
+                               i40e_reset_ch_rings(vsi, ch);
+                               clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
+                               netdev_unbind_sb_channel(netdev, fwd->netdev);
+                               netdev_set_sb_channel(fwd->netdev, 0);
+                               kfree(ch->fwd);
+                               ch->fwd = NULL;
+                               } else {

Indent issue?

Quoted 45 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +18 -1
+                                       dev_info(&pf->pdev->dev,
+                                                "Error deleting mac filter on macvlan err %s, aq_err %s\n",
+                                                i40e_stat_str(hw, ret),
+                                                i40e_aq_str(hw, aq_err));
+                               }
+                       break;
+               }
+       }
+}
+
 /**
  * i40e_setup_tc - configure multiple traffic classes
  * @netdev: net device to configure
@@ -11581,6 +12044,9 @@ static int i40e_set_features(struct net_device *netdev,
                return -EINVAL;
        }
 
+       if (!(features & NETIF_F_HW_L2FW_DOFFLOAD))
+               i40e_del_all_macvlans(vsi);
+
        need_reset = i40e_set_ntuple(pf, features);
 
        if (need_reset)
@@ -12314,6 +12780,8 @@ static const struct net_device_ops i40e_netdev_ops = {
        .ndo_bpf                = i40e_xdp,
        .ndo_xdp_xmit           = i40e_xdp_xmit,
        .ndo_xsk_async_xmit     = i40e_xsk_async_xmit,
+       .ndo_dfwd_add_station   = i40e_fwd_add,
+       .ndo_dfwd_del_station   = i40e_fwd_del,
 };
 
 /**
@@ -12373,6 +12841,9 @@ static int i40e_config_netdev(struct i40e_vsi *vsi)
        /* record features VLANs can make use of */
        netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
 
+       /* enable macvlan offloads */
+       netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
+
        hw_features = hw_enc_features           |
                      NETIF_F_HW_VLAN_CTAG_TX   |
                      NETIF_F_HW_VLAN_CTAG_RX;
--

2.17.1

Signature · 2 lines
==============================================
Mr. Shannon Nelson         Parents can't afford to be squeamish.
#2 lore raw

PATCH v2 1/1 i40e: Add macvlan support on i40e

Quoted 239 lines · Shannon Nelson
Quoted 17 lines · Harshitha Ramamurthy

This patch enables macvlan offloads on Fortville devices. The idea is to use channels as macvlan interfaces. The channels are VSIs of type VMDQ. When the first macvlan is created, the maximum no. of channels possible are created. From then on, as a macvlan interface is created, a mac filter is added to these already created channels (VSIs).

This patch builds on top of the recent changes which move away from the select_queue implementation of picking the tx queue.

Steps to configure the macvlan:
1. sudo ethtool -K ens261f1 l2-fwd-offload on
2. ip link add link ens261f1 name macvlan1 type macvlan
3. sudo ip link add link ens261f1 name macvlan1 type macvlan
4. sudo ip link set macvlan1 up

Quoted 5 lines
Quoted 1 line

---
v2: Addressed Shannon's comments
Added a new function to remove all macvlan VSIs

A few more comments below.

Quoted 205 lines
 drivers/net/ethernet/intel/i40e/i40e.h      |  26 ++
 drivers/net/ethernet/intel/i40e/i40e_main.c | 475
+++++++++++++++++++-
 2 files changed, 499 insertions(+), 2 deletions(-)
drivers/net/ethernet/intel/i40e/i40e.h +20 -0
index c06a4b5cdfae..107edcf2cf19 100644
@@ -27,6 +27,7 @@
 #include <net/ip6_checksum.h>
 #include <linux/ethtool.h>
 #include <linux/if_vlan.h>
+#include <linux/if_macvlan.h>
 #include <linux/if_bridge.h>
 #include <linux/clocksource.h>
 #include <linux/net_tstamp.h>
@@ -391,6 +392,11 @@ struct i40e_flex_pit {
        u8 pit_index;
 };
 
+struct i40e_fwd_adapter {
+       struct net_device *netdev;
+       int bit_no;
+};
+
 struct i40e_channel {
        struct list_head list;
        bool initialized;
@@ -405,11 +411,25 @@ struct i40e_channel {
        struct i40e_aqc_vsi_properties_data info;
 
        u64 max_tx_rate;
+       struct i40e_fwd_adapter *fwd;
 
        /* track this channel belongs to which VSI */
        struct i40e_vsi *parent_vsi;
 };
 
+static inline bool i40e_is_channel_macvlan(struct i40e_channel
 *ch)
+{
+       return !!ch->fwd;
+}
+
+static inline u8 *i40e_channel_mac(struct i40e_channel *ch)
+{
+       if (i40e_is_channel_macvlan(ch))
+               return ch->fwd->netdev->dev_addr;
+       else
+               return NULL;
+}
+
 /* struct that defines the Ethernet device */
 struct i40e_pf {
        struct pci_dev *pdev;
drivers/net/ethernet/intel/i40e/i40e.h +2 -0
@@ -787,6 +807,12 @@ struct i40e_vsi {
        struct list_head ch_list;
        u16 tc_seid_map[I40E_MAX_TRAFFIC_CLASS];
 
+       /* macvlan fields */
+#define I40E_MAX_MACVLANS      128 /* Max HW capable vectors - 1

on FVL */

drivers/net/ethernet/intel/i40e/i40e.h +4 -0
+       DECLARE_BITMAP(fwd_bitmask, I40E_MAX_MACVLANS);
+       struct list_head macvlan_list;
+       int macvlan_cnt;
+
        void *priv;     /* client driver data reference. */
 
        /* VSI specific handlers */
drivers/net/ethernet/intel/i40e/i40e_main.c +1 -2
index 4a129c7ab6f4..6f81db63d25b 100644
@@ -5818,8 +5818,10 @@ static int i40e_add_channel(struct i40e_pf
 *pf, u16 uplink_seid,
                return -ENOENT;
        }
 
-       /* Success, update channel */
-       ch->enabled_tc = enabled_tc;
+       /* Success, update channel, set enabled_tc only if the

channel

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+        * is not a macvlan
+        */
+       ch->enabled_tc = !i40e_is_channel_macvlan(ch) &&
enabled_tc;
        ch->seid = ctxt.seid;
        ch->vsi_number = ctxt.vsi_number;
        ch->stat_counter_idx =

cpu_to_le16(ctxt.info.stat_counter_idx);

drivers/net/ethernet/intel/i40e/i40e_main.c +8 -0
@@ -6811,6 +6813,467 @@ static void
 i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
        }
 }
 
+/**
+ * i40e_del_macvlan_filter
+ * @hw: pointer to the HW structure
+ * @seid: seid of the channel VSI
+ * @macaddr: the mac address to apply as a filter
+ * @aq_err: store the admin Q error
+ *
+ * This function deletes a mac filter on the channel VSI which

serves as the
+ * macvlan. Returns 0 on success.
+ **/
+static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
+ const u8 *macaddr, int
*aq_err)

drivers/net/ethernet/intel/i40e/i40e_main.c +9 -0
+{
+       struct i40e_aqc_remove_macvlan_element_data element;
+       i40e_status status;
+
+       memset(&element, 0, sizeof(element));
+       ether_addr_copy(element.mac_addr, macaddr);
+       element.vlan_tag = 0;
+       element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
+       status = i40e_aq_remove_macvlan(hw, seid, &element, 1,

NULL);

drivers/net/ethernet/intel/i40e/i40e_main.c +12 -0
+       *aq_err = hw->aq.asq_last_status;
+       return status;
+}
+
+/**
+ * i40e_add_macvlan_filter
+ * @hw: pointer to the HW structure
+ * @seid: seid of the channel VSI
+ * @macaddr: the mac address to apply as a filter
+ * @aq_err: store the admin Q error
+ *
+ * This function adds a mac filter on the channel VSI which serves

as the
+ * macvlan. Returns 0 on success.
+ **/
+static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
+ const u8 *macaddr, int
*aq_err)

drivers/net/ethernet/intel/i40e/i40e_main.c +22 -0
+{
+       struct i40e_aqc_add_macvlan_element_data element;
+       i40e_status status;
+       u16 cmd_flags = 0;
+
+       ether_addr_copy(element.mac_addr, macaddr);
+       element.vlan_tag = 0;
+       element.queue_number = 0;
+       element.match_method = I40E_AQC_MM_ERR_NO_RES;
+       cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
+       element.flags = cpu_to_le16(cmd_flags);
+       status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
+       *aq_err = hw->aq.asq_last_status;
+       return status;
+}
+
+/**
+ * i40e_reset_ch_rings - Reset the queue contexts in a channel
+ * @vsi: the VSI we want to access
+ * @ch: the channel we want to access
+ */
+static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct

i40e_channel *ch)

drivers/net/ethernet/intel/i40e/i40e_main.c +19 -0
+{
+       struct i40e_ring *tx_ring, *rx_ring;
+       u16 pf_q;
+       int i;
+
+       for (i = 0; i < ch->num_queue_pairs; i++) {
+               pf_q = ch->base_queue + i;
+               tx_ring = vsi->tx_rings[pf_q];
+               tx_ring->ch = NULL;
+               rx_ring = vsi->rx_rings[pf_q];
+               rx_ring->ch = NULL;
+       }
+}
+
+/**
+ * i40e_remove_macvlan_channels
+ * @vsi: the VSI we want to access
+ *
+ * This functions removes all the channel VSIs which serve as a

macvlan.
+ */

drivers/net/ethernet/intel/i40e/i40e_main.c +9 -0
+static void i40e_remove_macvlan_channels(struct i40e_vsi *vsi)
+{
+       struct i40e_channel *ch, *ch_tmp;
+       int ret;
+
+       if (list_empty(&vsi->macvlan_list))
+               return;
+
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list,

list) {

drivers/net/ethernet/intel/i40e/i40e_main.c +5 -0
+               struct i40e_vsi *parent_vsi;
+
+               if (i40e_is_channel_macvlan(ch)) {
+                       i40e_reset_ch_rings(vsi, ch);
+                       clear_bit(ch->fwd->bit_no, vsi-
Quoted 1 line

fwd_bitmask);

+ netdev_unbind_sb_channel(vsi->netdev, ch-

Quoted 1 line

fwd->netdev);

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 6915

Quoted 6 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +6 -0
+                       netdev_set_sb_channel(ch->fwd->netdev, 0);
+                       kfree(ch->fwd);
+                       ch->fwd = NULL;
+               }
+
+               list_del(&ch->list);
Quoted 4 lines

It looks like this removes more than just macvlan channels, as the name suggests. It removes all channels, and does a little extra cleaning if the channel is a macvlan. Maybe change the name to i40e_remove_channels()?

We iterate over the vsi->macvlan_list and delete only those channel VSIs which were set up to be MACVLANs. The vsi->macvlan_list only stores those channels which are macvlans. The extra cleaning that is being done here if the VSI is handed off to the macvlan net device is taking care of unbinding the Qs from the stack. I added this as a precaution only and isn't needed right now since it is being called from i40e_setup_macvlan at which point the Qs are not bound to the stack.

Quoted 185 lines
Quoted 185 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +8 -0
+               parent_vsi = ch->parent_vsi;
+               if (!parent_vsi || !ch->initialized) {
+                       kfree(ch);
+                       continue;
+               }
+
+               /* remove the VSI */
+               ret = i40e_aq_delete_element(&vsi->back->hw, ch-
Quoted 1 line

seid,

drivers/net/ethernet/intel/i40e/i40e_main.c +4 -0
+                                            NULL);
+               if (ret)
+                       dev_err(&vsi->back->pdev->dev,
+                               "unable to remove channel (%d) for

parent VSI(%d)\n",

drivers/net/ethernet/intel/i40e/i40e_main.c +13 -0
+                               ch->seid, parent_vsi->seid);
+               kfree(ch);
+       }
+       vsi->macvlan_cnt = 0;
+}
+
+/**
+ * i40e_fwd_ring_up - bring the macvlan device up
+ * @vsi: the VSI we want to access
+ * @vdev: macvlan netdevice
+ * @fwd: the private fwd structure
+ */
+static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct

net_device *vdev,

drivers/net/ethernet/intel/i40e/i40e_main.c +12 -0
+                           struct i40e_fwd_adapter *fwd)
+{
+       int ret = 0, num_tc = 1,  i, aq_err;
+       struct i40e_channel *ch, *ch_tmp;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+
+       if (list_empty(&vsi->macvlan_list))
+               return -EINVAL;
+
+       /* Go through the list and find an available channel */
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list,

list) {

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+               if (!i40e_is_channel_macvlan(ch)) {
+                       ch->fwd = fwd;
+                       /* record configuration for macvlan

interface in vdev */

drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
+                       for (i = 0; i < num_tc; i++)
+                               netdev_bind_sb_channel_queue(vsi-
Quoted 1 line

netdev, vdev,

drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
+                                                            i,
+                                                            ch-
Quoted 1 line

num_queue_pairs,

+ ch-

Quoted 1 line

base_queue);

drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
+                       for (i = 0; i < ch->num_queue_pairs; i++) {
+                               struct i40e_ring *tx_ring,

*rx_ring;

drivers/net/ethernet/intel/i40e/i40e_main.c +23 -0
+                               u16 pf_q;
+
+                               pf_q = ch->base_queue + i;
+
+                               /* Get to TX ring ptr */
+                               tx_ring = vsi->tx_rings[pf_q];
+                               tx_ring->ch = ch;
+
+                               /* Get the RX ring ptr */
+                               rx_ring = vsi->rx_rings[pf_q];
+                               rx_ring->ch = ch;
+                       }
+                       break;
+               }
+       }
+
+       /* Guarantee all rings are updated before we update the
+        * MAC address filter.
+        */
+       wmb();
+
+       /* Add a mac filter */
+       ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, 

&aq_err);

drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
+       if (ret) {
+               /* if we cannot add the MAC rule then disable the

offload */

drivers/net/ethernet/intel/i40e/i40e_main.c +11 -0
+               macvlan_release_l2fw_offload(vdev);
+               for (i = 0; i < ch->num_queue_pairs; i++) {
+                       struct i40e_ring *rx_ring;
+                       u16 pf_q;
+
+                       pf_q = ch->base_queue + i;
+                       rx_ring = vsi->rx_rings[pf_q];
+                       rx_ring->netdev = NULL;
+               }
+               dev_info(&pf->pdev->dev,
+                        "Error adding mac filter on macvlan err

%s, aq_err %s\n",

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+                         i40e_stat_str(hw, ret),
+                         i40e_aq_str(hw, aq_err));
+               netdev_err(vdev, "L2fwd offload disabled to L2

filter error\n");

drivers/net/ethernet/intel/i40e/i40e_main.c +6 -0
+       }
+       return ret;
+}
+
+/**
+ * i40e_setup_macvlans - create the channels which will be

macvlans
+ * @vsi: the VSI we want to access
+ * @macvlan_cnt: no. of macvlans to be setup
+ * @qcnt: no. of Qs per macvlan
+ * @vdev: macvlan netdevice
+ */
+static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,

drivers/net/ethernet/intel/i40e/i40e_main.c +34 -0
+                              struct net_device *vdev)
+{
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+       struct i40e_vsi_context ctxt;
+       u16 sections, qmap, num_qps;
+       struct i40e_channel *ch;
+       int i, pow, ret = 0;
+       u8 offset = 0;
+
+       if (vsi->type != I40E_VSI_MAIN)
+               return -EINVAL;
+       if (!macvlan_cnt)
+               return -EINVAL;
+
+       num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
+
+       /* find the next higher power-of-2 of num queue pairs */
+       pow = fls(roundup_pow_of_two(num_qps) - 1);
+
+       qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
+               (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
+
+       /* Setup context bits for the main VSI */
+       sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
+       sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
+       memset(&ctxt, 0, sizeof(ctxt));
+       ctxt.seid = vsi->seid;
+       ctxt.pf_num = vsi->back->hw.pf_id;
+       ctxt.vf_num = 0;
+       ctxt.uplink_seid = vsi->uplink_seid;
+       ctxt.info = vsi->info;
+       ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
+       ctxt.info.mapping_flags |=

cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);

drivers/net/ethernet/intel/i40e/i40e_main.c +9 -0
+       ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
+       ctxt.info.valid_sections |= cpu_to_le16(sections);
+
+       /* Reconfigure RSS for main VSI with new max queue count */
+       vsi->rss_size = max_t(u16, num_qps, qcnt);
+       ret = i40e_vsi_config_rss(vsi);
+       if (ret) {
+               dev_info(&pf->pdev->dev,
+                        "Failed to reconfig RSS for num_queues

(%u)\n",

drivers/net/ethernet/intel/i40e/i40e_main.c +6 -0
+                        vsi->rss_size);
+               return ret;
+       }
+       vsi->reconfig_rss = true;
+       dev_dbg(&vsi->back->pdev->dev,
+               "Reconfigured RSS with num_queues (%u)\n", vsi-
Quoted 1 line

rss_size);

drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+       vsi->next_base_queue = num_qps;
+       vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
+
+       /* Update the VSI after updating the VSI queue-mapping
+        * information
+        */
+       ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
+       if (ret) {
+               dev_info(&pf->pdev->dev,
+                        "Update vsi tc config failed, err %s

aq_err %s\n",

drivers/net/ethernet/intel/i40e/i40e_main.c +8 -0
+                        i40e_stat_str(hw, ret),
+                        i40e_aq_str(hw, hw->aq.asq_last_status));
+               return ret;
+       }
+       /* update the local VSI info with updated queue map */
+       i40e_vsi_update_queue_map(vsi, &ctxt);
+       vsi->info.valid_sections = 0;
+
Quoted 8 lines

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7083

Quoted 3 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+       /* Create channels for macvlans */
+       INIT_LIST_HEAD(&vsi->macvlan_list);
+       vsi->macvlan_cnt = macvlan_cnt;

You might wait until after this for-loop succeeds before setting the vsi->macvlan_cnt; or ig the i40e_remove_macvlan_channels() needs it, increment it each time through the loop at teh same time that you decrement cnt_q_avail.

That makes sense, will make the change.

Quoted 66 lines
Quoted 66 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +30 -0
+       for (i = 0; i < macvlan_cnt; i++) {
+               ch = kzalloc(sizeof(*ch), GFP_KERNEL);
+               if (!ch) {
+                       ret = -ENOMEM;
+                       goto err_free;
+               }
+               INIT_LIST_HEAD(&ch->list);
+               ch->num_queue_pairs = qcnt;
+               if (!i40e_setup_channel(pf, vsi, ch)) {
+                       ret = -EINVAL;
+                       goto err_free;
+               }
+               ch->parent_vsi = vsi;
+               vsi->cnt_q_avail -= ch->num_queue_pairs;
+               list_add_tail(&ch->list, &vsi->macvlan_list);
+       }
+       return ret;
+
+err_free:
+       dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
+       i40e_remove_macvlan_channels(vsi);
+       return ret;
+}
+
+/**
+ * i40e_fwd_add - configure macvlans
+ * @netdev: net device to configure
+ * @vdev: macvlan netdevice
+ **/
+static void *i40e_fwd_add(struct net_device *netdev, struct

net_device *vdev)

drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+{
+       struct i40e_netdev_priv *np = netdev_priv(netdev);
+       u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
+       struct i40e_vsi *vsi = np->vsi;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_fwd_adapter *fwd;
+       int avail_macvlan, ret;
+
+       if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
+               netdev_info(netdev, "Macvlans are not supported

when DCB is enabled\n");

drivers/net/ethernet/intel/i40e/i40e_main.c +4 -0
+               return ERR_PTR(-EINVAL);
+       }
+       if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
+               netdev_info(netdev, "Macvlans are not supported

when HW TC offload is on\n");

drivers/net/ethernet/intel/i40e/i40e_main.c +12 -0
+               return ERR_PTR(-EINVAL);
+       }
+
+       /* The macvlan device can't be a multiqueue device */
+       if (netif_is_multiqueue(vdev))
+               return ERR_PTR(-ERANGE);
+
+       if (!vsi->macvlan_cnt) {
+               /* reserve bit 0 for the pf device */
+               set_bit(0, vsi->fwd_bitmask);
+
+               /* Try to reserve as many queues for macvlans.

First reserve
+ * 3/4th of max vectors, then half, then quarter
and calculate

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+                *  Qs per macvlan as you go
+                */
+               vectors = pf->num_lan_msix;
Quoted 8 lines
Quoted 4 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
+               if (vectors <= I40E_MAX_MACVLANS && vectors > 96) {
+                       /* allocate 4 Qs per macvlan and 32 Qs to

the PF*/
+ q_per_macvlan = 4;

If above you disallow multiqueue netdev, why do you allocate more than
1 queue per macvlan?

With Alex's changes to move away from the ndo_select_queue implementation of picking the TX queue, we started using the tc_to_txq mapping to pick the queues for transmit. Which is why we "lock down" the macvlan device to be a single queue device so that the tc_to_txq mapping in the macvlan netdev can be used to pick the tx queue. I will add a comment in the next version explaining this where I disallow multiqueue netdev above.

Quoted 31 lines
Quoted 31 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+                       macvlan_cnt = (vectors - 32) / 4;
+               } else if (vectors <= 96 && vectors > 64) {
+                       /* allocate 4 Qs per macvlan and 32 Qs to

the PF*/

drivers/net/ethernet/intel/i40e/i40e_main.c +4 -0
+                       q_per_macvlan = 4;
+                       macvlan_cnt = (vectors - 32) / 4;
+               } else if (vectors <= 64 && vectors > 32) {
+                       /* allocate 2 Qs per macvlan and 16 Qs to

the PF*/

drivers/net/ethernet/intel/i40e/i40e_main.c +4 -0
+                       q_per_macvlan = 2;
+                       macvlan_cnt = (vectors - 16) / 2;
+               } else {
+                       /* allocate 1 Q per macvlan 16 Qs to the

PF*/

drivers/net/ethernet/intel/i40e/i40e_main.c +10 -0
+                       q_per_macvlan = 1;
+                       macvlan_cnt = (vectors - 16);
+               }
+               if (macvlan_cnt == 0)
+                       return ERR_PTR(-EBUSY);
+
+               /* Quiesce VSI queues */
+               i40e_quiesce_vsi(vsi);
+
+               /* sets up the macvlans but does not "enable" them

*/
+ ret = i40e_setup_macvlans(vsi, macvlan_cnt,
q_per_macvlan,

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+                                         vdev);
+               if (ret)
+                       return ERR_PTR(ret);
Quoted 8 lines

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7177

Quoted 6 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +6 -0
+
+               /* Unquiesce VSI */
+               i40e_unquiesce_vsi(vsi);
+       }
+       avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
+                                           vsi->macvlan_cnt);

Is it possible there are none left?

Yes, I will return error if none are left.

Quoted 19 lines
Quoted 19 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +18 -0
+
+       /* create the fwd struct */
+       fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
+       if (!fwd)
+               return ERR_PTR(-ENOMEM);
+
+       set_bit(avail_macvlan, vsi->fwd_bitmask);
+       fwd->bit_no = avail_macvlan;
+       netdev_set_sb_channel(vdev, avail_macvlan);
+       fwd->netdev = vdev;
+
+       if (!netif_running(netdev))
+               return fwd;
+
+       /* Set fwd ring up */
+       ret = i40e_fwd_ring_up(vsi, vdev, fwd);
+       if (ret) {
+               /* unbind the queues and drop the subordinate
Quoted 8 lines

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7200

Quoted 6 lines

channel config */

drivers/net/ethernet/intel/i40e/i40e_main.c +5 -0
+               netdev_unbind_sb_channel(netdev, vdev);
+               netdev_set_sb_channel(vdev, 0);
+
+               kfree(fwd);
+               return ERR_PTR(-EINVAL);

Do you need to call something like i40e_del_macvlan_filter() here?

No, because the macvlan filter is not added until the end of the i40e_fwd_ring_up function where if it fails, we return an error.

Quoted 81 lines
Quoted 77 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +6 -0
+       }
+       return fwd;
+}
+
+/**
+ * i40e_del_all_macvlans - Delete all the mac filters on the

channels
+ * @vsi: the VSI we want to access
+ */

drivers/net/ethernet/intel/i40e/i40e_main.c +8 -0
+static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
+{
+       struct i40e_channel *ch, *ch_tmp;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+       int aq_err, ret = 0;
+
+       list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list,

list) {

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+               if (i40e_is_channel_macvlan(ch)) {
+                       ret = i40e_del_macvlan_filter(hw, ch->seid,
+                                                     i40e_channel_

mac(ch),

drivers/net/ethernet/intel/i40e/i40e_main.c +5 -0
+                                                     &aq_err);
+                       if (!ret) {
+                               /* Reset queue contexts */
+                               i40e_reset_ch_rings(vsi, ch);
+                               clear_bit(ch->fwd->bit_no, vsi-
Quoted 1 line

fwd_bitmask);

+ netdev_unbind_sb_channel(vsi-

Quoted 1 line

netdev,

+ ch->fwd-

Quoted 1 line

netdev);

+ netdev_set_sb_channel(ch->fwd-

Quoted 1 line

netdev, 0);

drivers/net/ethernet/intel/i40e/i40e_main.c +23 -0
+                               kfree(ch->fwd);
+                               ch->fwd = NULL;
+                       }
+               }
+       }
+}
+
+/**
+ * i40e_fwd_del - delete macvlan interfaces
+ * @netdev: net device to configure
+ * @vdev: macvlan netdevice
+ */
+static void i40e_fwd_del(struct net_device *netdev, void *vdev)
+{
+       struct i40e_netdev_priv *np = netdev_priv(netdev);
+       struct i40e_fwd_adapter *fwd = vdev;
+       struct i40e_channel *ch, *ch_tmp;
+       struct i40e_vsi *vsi = np->vsi;
+       struct i40e_pf *pf = vsi->back;
+       struct i40e_hw *hw = &pf->hw;
+       int aq_err, ret = 0;
+
+       /* Find the channel associated with the macvlan and del mac

filter */
+ list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list,
list) {

drivers/net/ethernet/intel/i40e/i40e_main.c +5 -0
+               if (i40e_is_channel_macvlan(ch) &&
+                   ether_addr_equal(i40e_channel_mac(ch),
+                                    fwd->netdev->dev_addr)) {
+                       ret = i40e_del_macvlan_filter(hw, ch->seid,
+                                                     i40e_channel_

mac(ch),

drivers/net/ethernet/intel/i40e/i40e_main.c +5 -0
+                                                     &aq_err);
+                       if (!ret) {
+                               /* Reset queue contexts */
+                               i40e_reset_ch_rings(vsi, ch);
+                               clear_bit(ch->fwd->bit_no, vsi-
Quoted 1 line

fwd_bitmask);

+ netdev_unbind_sb_channel(netdev,
fwd->netdev);
+ netdev_set_sb_channel(fwd->netdev,
0);

drivers/net/ethernet/intel/i40e/i40e_main.c static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) · line 7266

Quoted 3 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
+                               kfree(ch->fwd);
+                               ch->fwd = NULL;
+                               } else {
Quoted 1 line

Indent issue?

Yes, will fix it in the next version.

Quoted 55 lines
Quoted 53 lines
drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
+                                       dev_info(&pf->pdev->dev,
+                                                "Error deleting

mac filter on macvlan err %s, aq_err %s\n",
+ i40e_stat_str(hw,
ret),
+ i40e_aq_str(hw,
aq_err));

drivers/net/ethernet/intel/i40e/i40e_main.c +9 -0
+                               }
+                       break;
+               }
+       }
+}
+
 /**
  * i40e_setup_tc - configure multiple traffic classes
  * @netdev: net device to configure
@@ -11581,6 +12044,9 @@ static int i40e_set_features(struct
 net_device *netdev,
                return -EINVAL;
        }
 
+       if (!(features & NETIF_F_HW_L2FW_DOFFLOAD))
+               i40e_del_all_macvlans(vsi);
+
        need_reset = i40e_set_ntuple(pf, features);
 

if (need_reset)

drivers/net/ethernet/intel/i40e/i40e_main.c +2 -0
@@ -12314,6 +12780,8 @@ static const struct net_device_ops
 i40e_netdev_ops = {
        .ndo_bpf                = i40e_xdp,
        .ndo_xdp_xmit           = i40e_xdp_xmit,
        .ndo_xsk_async_xmit     = i40e_xsk_async_xmit,
+       .ndo_dfwd_add_station   = i40e_fwd_add,
+       .ndo_dfwd_del_station   = i40e_fwd_del,
 };
 

/**

drivers/net/ethernet/intel/i40e/i40e_main.c +3 -0
@@ -12373,6 +12841,9 @@ static int i40e_config_netdev(struct
 i40e_vsi *vsi)
        /* record features VLANs can make use of */
        netdev->vlan_features |= hw_enc_features |
 NETIF_F_TSO_MANGLEID;
 
+       /* enable macvlan offloads */
+       netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
+
        hw_features = hw_enc_features           |
                      NETIF_F_HW_VLAN_CTAG_TX   |
                      NETIF_F_HW_VLAN_CTAG_RX;

--
2.17.1