Skip to contentSkip to conversation

net/ena: v2.8.0 driver release

PATCH Superseded [email protected] 2 participants 8 messages Oct 25, 2023 – Oct 26, 2023 Network drivers

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

v2 cover net/ena: v2.8.0 driver release

From: Shai Brandes <[email protected]>

Hi,

This patchset contains alignment of the driver to the latest HAL version which adds support for retrieving new metrics from the device and opens a path to use additional device features that are not yet supported by the driver.

The new driver features are mostly about adding additional metrics. We added support for an additional customer metric, ENA-express metrics and are now reporting also rx overrun errors. All new metrics have MP support.

* Testing:
The team performed directed tests for all the features and executed our proprietary performance benchamarking (BW, PPS, Latency) to verify there is no regression. It was executed on the entire Amazon EC2 instance types matrix that covers all device generations and CPU architectures.

---
v2:
* Fixed spelling issues from checkpatch in patch 0001

Shai Brandes (5):
net/ena: hal upgrade
net/ena: add support for connection tracking metric net/ena: report Rx overrun errors in xstats net/ena: add support for ena-express metrics net/ena: update ena version to 2.8.0

 doc/guides/rel_notes/release_23_11.rst        |   7 +
 drivers/net/ena/base/ena_com.c                | 499 +++++++++++++++---
 drivers/net/ena/base/ena_com.h                | 197 ++++++-
 .../net/ena/base/ena_defs/ena_admin_defs.h    | 198 ++++++-
 .../net/ena/base/ena_defs/ena_eth_io_defs.h   |  18 +-
 drivers/net/ena/base/ena_defs/ena_gen_info.h  |   4 +-
 drivers/net/ena/base/ena_defs/ena_regs_defs.h |  12 +
 drivers/net/ena/base/ena_eth_com.c            |  45 +-
 drivers/net/ena/base/ena_eth_com.h            |  30 +-
 drivers/net/ena/base/ena_plat.h               |   8 +-
 drivers/net/ena/base/ena_plat_dpdk.h          |  49 +-
 drivers/net/ena/ena_ethdev.c                  | 321 ++++++++---
 drivers/net/ena/ena_ethdev.h                  |  43 +-
 13 files changed, 1204 insertions(+), 227 deletions(-)
Signature · 1 line
2.17.1
#1 lore raw

PATCH v2 1/5 net/ena: hal upgrade

From: Shai Brandes <[email protected]>

ENA maintains a HAL that is shared by all supported host drivers. Main features introduced to the HAL:
[1] Reworked the mechanism that queries the performance metrics from the device.
[2] Added support for a new metric that allows monitoring the available tracked connections.
[3] Added support for a new statistic that counts RX drops due to insufficient buffers provided by host.
[4] Added support for Scalable Reliable Datagram (SRD) metrics from ENA Express.
[5] Added support for querying the LLQ entry size recommendation from the device.
[6] Added support for PTP hardware clock (PHC) feature that provides enhanced accuracy (Not supported by the driver).
[7] Added support for new reset reasons for a suspected CPU starvation and for completion descriptor inconsistency.
[8] Aligned all return error code to a common notation.
[9] Removed an obsolete queue tail pointer update API.

---
 doc/guides/rel_notes/release_23_11.rst        |   4 +
 drivers/net/ena/base/ena_com.c                | 499 +++++++++++++++---
 drivers/net/ena/base/ena_com.h                | 197 ++++++-
 .../net/ena/base/ena_defs/ena_admin_defs.h    | 198 ++++++-
 .../net/ena/base/ena_defs/ena_eth_io_defs.h   |  18 +-
 drivers/net/ena/base/ena_defs/ena_gen_info.h  |   4 +-
 drivers/net/ena/base/ena_defs/ena_regs_defs.h |  12 +
 drivers/net/ena/base/ena_eth_com.c            |  45 +-
 drivers/net/ena/base/ena_eth_com.h            |  30 +-
 drivers/net/ena/base/ena_plat.h               |   8 +-
 drivers/net/ena/base/ena_plat_dpdk.h          |  49 +-
 drivers/net/ena/ena_ethdev.c                  |  16 +-
 12 files changed, 915 insertions(+), 165 deletions(-)
doc/guides/rel_notes/release_23_11.rst +4 -0
index 0a6fc76a9d..e3b0ba58c9 100644
@@ -144,6 +144,10 @@ New Features
 
   * Added support for Network Service Header (NSH) flow matching.
 
+* **Updated Amazon Elastic Network Adapter ena net driver.**
+
+  * Upgraded ENA HAL to latest version.
+
 * **Updated Solarflare net driver.**
 
   * Added support for transfer flow action ``INDIRECT`` with subtype ``VXLAN_ENCAP``.
drivers/net/ena/base/ena_com.c +424 -75

700 lines not on this page · raw message

drivers/net/ena/base/ena_com.h +185 -12

329 lines not on this page · raw message

drivers/net/ena/base/ena_defs/ena_admin_defs.h +191 -7

327 lines not on this page · raw message

drivers/net/ena/base/ena_defs/ena_eth_io_defs.h +17 -1
index 2b934fff08..2107d17fdf 100644
@@ -261,7 +261,8 @@ struct ena_eth_io_intr_reg {
 	/* 14:0 : rx_intr_delay
 	 * 29:15 : tx_intr_delay
 	 * 30 : intr_unmask
-	 * 31 : reserved
+	 * 31 : no_moderation_update - 0 - moderation
+	 *    updated, 1 - moderation not updated
 	 */
 	uint32_t intr_control;
 };
@@ -381,6 +382,8 @@ struct ena_eth_io_numa_node_cfg_reg {
 #define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK              GENMASK(29, 15)
 #define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT               30
 #define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK                BIT(30)
+#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT      31
+#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK       BIT(31)
 
 /* numa_node_cfg_reg */
 #define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK              GENMASK(7, 0)
@@ -918,6 +921,19 @@ static inline void set_ena_eth_io_intr_reg_intr_unmask(struct ena_eth_io_intr_re
 	p->intr_control |= (val << ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT) & ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK;
 }
 
+static inline uint32_t get_ena_eth_io_intr_reg_no_mod_update(const struct ena_eth_io_intr_reg *p)
+{
+	return (p->intr_control & ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK) >>
+			ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT;
+}
+
+static inline void set_ena_eth_io_intr_reg_no_mod_update(struct ena_eth_io_intr_reg *p,
+									     uint32_t val)
+{
+	p->intr_control |= (val << ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT) &
+			ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK;
+}
+
 static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa(const struct ena_eth_io_numa_node_cfg_reg *p)
 {
 	return p->numa_cfg & ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK;
drivers/net/ena/base/ena_defs/ena_gen_info.h +2 -2
index 49895108d5..fc62201ccf 100644
@@ -2,5 +2,5 @@
  * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
  * All rights reserved.
  */
-#define	ENA_GEN_DATE	"Fri Sep 18 17:09:00 IDT 2020"
-#define	ENA_GEN_COMMIT	"0f80d82"
+#define	ENA_GEN_DATE	"Thu 14 Apr 2022 12:50:30 PM IDT"
+#define	ENA_GEN_COMMIT	"35388392"
drivers/net/ena/base/ena_defs/ena_regs_defs.h +12 -0
index 639cd98c0a..6a33f74812 100644
@@ -21,6 +21,8 @@ enum ena_regs_reset_reason_types {
 	ENA_REGS_RESET_USER_TRIGGER                 = 12,
 	ENA_REGS_RESET_GENERIC                      = 13,
 	ENA_REGS_RESET_MISS_INTERRUPT               = 14,
+	ENA_REGS_RESET_SUSPECTED_POLL_STARVATION    = 15,
+	ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED	    = 16,
 	ENA_REGS_RESET_LAST,
 };
 
@@ -52,6 +54,11 @@ enum ena_regs_reset_reason_types {
 #define ENA_REGS_MMIO_RESP_HI_OFF                           0x64
 #define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF                   0x68
 
+/* phc_registers offsets */
+
+/* 100 base */
+#define ENA_REGS_PHC_DB_OFF                                 0x100
+
 /* version register */
 #define ENA_REGS_VERSION_MINOR_VERSION_MASK                 0xff
 #define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT                8
@@ -98,6 +105,8 @@ enum ena_regs_reset_reason_types {
 #define ENA_REGS_DEV_CTL_QUIESCENT_MASK                     0x4
 #define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT                    3
 #define ENA_REGS_DEV_CTL_IO_RESUME_MASK                     0x8
+#define ENA_REGS_DEV_CTL_RESET_REASON_EXT_SHIFT             24
+#define ENA_REGS_DEV_CTL_RESET_REASON_EXT_MASK              0xf000000
 #define ENA_REGS_DEV_CTL_RESET_REASON_SHIFT                 28
 #define ENA_REGS_DEV_CTL_RESET_REASON_MASK                  0xf0000000
 
@@ -128,4 +137,7 @@ enum ena_regs_reset_reason_types {
 #define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT          16
 #define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK           0xffff0000
 
+/* phc_db_req_id register */
+#define ENA_REGS_PHC_DB_REQ_ID_MASK                         0xffff
+
 #endif /* _ENA_REGS_H_ */
drivers/net/ena/base/ena_eth_com.c +28 -17

112 lines not on this page · raw message

drivers/net/ena/base/ena_eth_com.h +5 -25
index 0cfd18882e..cee4f35124 100644
@@ -11,8 +11,10 @@ extern "C" {
 #endif
 #include "ena_com.h"
 
-/* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */
-#define ENA_COMP_HEAD_THRESH 4
+/* we allow 2 DMA descriptors per LLQ entry */
+#define ENA_LLQ_ENTRY_DESC_CHUNK_SIZE	(2 * sizeof(struct ena_eth_io_tx_desc))
+#define ENA_LLQ_HEADER		(128UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE)
+#define ENA_LLQ_LARGE_HEADER	(256UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE)
 
 struct ena_com_tx_ctx {
 	struct ena_com_tx_meta ena_meta;
@@ -48,7 +50,7 @@ struct ena_com_rx_ctx {
 	bool frag;
 	u32 hash;
 	u16 descs;
-	int max_bufs;
+	u16 max_bufs;
 	u8 pkt_offset;
 };
 
@@ -171,28 +173,6 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)
 	return 0;
 }
 
-static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq)
-{
-	u16 unreported_comp, head;
-	bool need_update;
-
-	if (unlikely(io_cq->cq_head_db_reg)) {
-		head = io_cq->head;
-		unreported_comp = head - io_cq->last_head_update;
-		need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH);
-
-		if (unlikely(need_update)) {
-			ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq),
-				    "Write completion queue doorbell for queue %d: head: %d\n",
-				    io_cq->qid, head);
-			ENA_REG_WRITE32(io_cq->bus, head, io_cq->cq_head_db_reg);
-			io_cq->last_head_update = head;
-		}
-	}
-
-	return 0;
-}
-
 static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq,
 					    u8 numa_node)
 {
drivers/net/ena/base/ena_plat.h +5 -3
index 2583823080..a3649e0cb6 100644
@@ -14,14 +14,16 @@
 #else
 #include <ena_plat_dpdk.h>
 #endif
+#elif defined(_WIN32)
+#include <ena_plat_windows.h>
 #elif defined(__FreeBSD__)
-#if defined(_KERNEL)
+#if defined(__KERNEL__)
 #include <ena_plat_fbsd.h>
 #else
 #include <ena_plat_dpdk.h>
 #endif
-#elif defined(_WIN32)
-#include <ena_plat_windows.h>
+#elif defined(__APPLE__)
+#include <ena_plat_macos.h>
 #else
 #error "Invalid platform"
 #endif
drivers/net/ena/base/ena_plat_dpdk.h +41 -8

102 lines not on this page · raw message

drivers/net/ena/ena_ethdev.c +1 -15
index 7345e480f8..b764442dbb 100644
@@ -1171,7 +1171,6 @@ static int ena_start(struct rte_eth_dev *dev)
 	struct ena_adapter *adapter = dev->data->dev_private;
 	uint64_t ticks;
 	int rc = 0;
-	uint16_t i;
 
 	/* Cannot allocate memory in secondary process */
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
@@ -1209,11 +1208,6 @@ static int ena_start(struct rte_eth_dev *dev)
 	++adapter->dev_stats.dev_start;
 	adapter->state = ENA_ADAPTER_STATE_RUNNING;
 
-	for (i = 0; i < dev->data->nb_rx_queues; i++)
-		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
-	for (i = 0; i < dev->data->nb_tx_queues; i++)
-		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
-
 	return 0;
 
 err_rss_init:
@@ -1229,7 +1223,6 @@ static int ena_stop(struct rte_eth_dev *dev)
 	struct ena_com_dev *ena_dev = &adapter->ena_dev;
 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 	struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
-	uint16_t i;
 	int rc;
 
 	/* Cannot free memory in secondary process */
@@ -1261,11 +1254,6 @@ static int ena_stop(struct rte_eth_dev *dev)
 	adapter->state = ENA_ADAPTER_STATE_STOPPED;
 	dev->data->dev_started = 0;
 
-	for (i = 0; i < dev->data->nb_rx_queues; i++)
-		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
-	for (i = 0; i < dev->data->nb_tx_queues; i++)
-		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
-
 	return 0;
 }
 
@@ -2687,7 +2675,6 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
 
 	/* Burst refill to save doorbells, memory barriers, const interval */
 	if (free_queue_entries >= rx_ring->rx_free_thresh) {
-		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
 		ena_populate_rx_queue(rx_ring, free_queue_entries);
 	}
 
@@ -3096,7 +3083,6 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
 		/* acknowledge completion of sent packets */
 		tx_ring->next_to_clean = next_to_clean;
 		ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs);
-		ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
 	}
 
 	if (mbuf_cnt != 0)
@@ -3629,7 +3615,7 @@ static void ena_rx_queue_intr_set(struct rte_eth_dev *dev,
 	struct ena_ring *rxq = &adapter->rx_ring[queue_id];
 	struct ena_eth_io_intr_reg intr_reg;
 
-	ena_com_update_intr_reg(&intr_reg, 0, 0, unmask);
+	ena_com_update_intr_reg(&intr_reg, 0, 0, unmask, 1);
 	ena_com_unmask_intr(rxq->ena_com_io_cq, &intr_reg);
 }
 
Signature · 1 line
2.17.1
#2 lore raw

Re: [PATCH v2 1/5] net/ena: hal upgrade

Quoted 43 lines · On 10/25/2023 2:36 PM, [email protected] wrote:

From: Shai Brandes <[email protected]>

ENA maintains a HAL that is shared by all supported host drivers. Main features introduced to the HAL:
[1] Reworked the mechanism that queries the performance metrics from the device.
[2] Added support for a new metric that allows monitoring the available tracked connections.
[3] Added support for a new statistic that counts RX drops due to insufficient buffers provided by host.
[4] Added support for Scalable Reliable Datagram (SRD) metrics from ENA Express.
[5] Added support for querying the LLQ entry size recommendation from the device.
[6] Added support for PTP hardware clock (PHC) feature that provides enhanced accuracy (Not supported by the driver).
[7] Added support for new reset reasons for a suspected CPU starvation and for completion descriptor inconsistency.
[8] Aligned all return error code to a common notation.
[9] Removed an obsolete queue tail pointer update API.

---
 doc/guides/rel_notes/release_23_11.rst        |   4 +
 drivers/net/ena/base/ena_com.c                | 499 +++++++++++++++---
 drivers/net/ena/base/ena_com.h                | 197 ++++++-
 .../net/ena/base/ena_defs/ena_admin_defs.h    | 198 ++++++-
 .../net/ena/base/ena_defs/ena_eth_io_defs.h   |  18 +-
 drivers/net/ena/base/ena_defs/ena_gen_info.h  |   4 +-
 drivers/net/ena/base/ena_defs/ena_regs_defs.h |  12 +
 drivers/net/ena/base/ena_eth_com.c            |  45 +-
 drivers/net/ena/base/ena_eth_com.h            |  30 +-
 drivers/net/ena/base/ena_plat.h               |   8 +-
 drivers/net/ena/base/ena_plat_dpdk.h          |  49 +-
 drivers/net/ena/ena_ethdev.c                  |  16 +-
 12 files changed, 915 insertions(+), 165 deletions(-)
doc/guides/rel_notes/release_23_11.rst +0 -0
index 0a6fc76a9d..e3b0ba58c9 100644

doc/guides/rel_notes/release_23_11.rst New Features · line 149

Quoted 6 lines
doc/guides/rel_notes/release_23_11.rst +2 -0
@@ -144,6 +144,10 @@ New Features
 
   * Added support for Network Service Header (NSH) flow matching.
 
+* **Updated Amazon Elastic Network Adapter ena net driver.**
+
 

What do you think clarify as:

Updated Amazon ena (Elastic Network Adapter) net driver.

doc/guides/rel_notes/release_23_11.rst New Features · line 152

Quoted 3 lines
doc/guides/rel_notes/release_23_11.rst +2 -0
+  * Upgraded ENA HAL to latest version.
+
 

Updates should be ordered in vendor in the driver group, so can you please sort it based on 'Amazon'? With current release notes, it makes before Intel driver updates.

Quoted 21 lines
doc/guides/rel_notes/release_23_11.rst +0 -0
 * **Updated Solarflare net driver.**

* Added support for transfer flow action ``INDIRECT`` with subtype ``VXLAN_ENCAP``.

drivers/net/ena/base/ena_com.c +6 -0
index 5ca36ab6d9..880d047956 100644
@@ -38,6 +38,12 @@
 
 #define ENA_MAX_ADMIN_POLL_US 5000
 
+/* PHC definitions */
+#define ENA_PHC_DEFAULT_EXPIRE_TIMEOUT_USEC 20
+#define ENA_PHC_DEFAULT_BLOCK_TIMEOUT_USEC 1000
+#define ENA_PHC_TIMESTAMP_ERROR 0xFFFFFFFFFFFFFFFF
+#define ENA_PHC_REQ_ID_OFFSET 0xDEAD
+
 /*****************************************************************************/
 /*****************************************************************************/
 /*****************************************************************************/

drivers/net/ena/base/ena_com.c static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev, · line 80

Quoted 6 lines
drivers/net/ena/base/ena_com.c +1 -1
@@ -70,7 +76,7 @@ static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,
 				       dma_addr_t addr)
 {
 	if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {
-		ena_trc_err(ena_dev, "DMA address has more bits than the device supports\n");
+		ena_trc_err(ena_dev, "DMA address has more bits that the device supports\n");
 

Original wording looks better to me, 'than' instead of 'that', but we can get another opinion from a native English speaker.

<...>

Quoted 15 lines
drivers/net/ena/base/ena_plat.h +3 -1
index 2583823080..a3649e0cb6 100644
@@ -14,14 +14,16 @@
 #else
 #include <ena_plat_dpdk.h>
 #endif
+#elif defined(_WIN32)
+#include <ena_plat_windows.h>
 #elif defined(__FreeBSD__)
-#if defined(_KERNEL)
+#if defined(__KERNEL__)
 #include <ena_plat_fbsd.h>

drivers/net/ena/base/ena_plat.h line 29

Quoted 10 lines
drivers/net/ena/base/ena_plat.h +2 -2
 #else
 #include <ena_plat_dpdk.h>
 #endif
-#elif defined(_WIN32)
-#include <ena_plat_windows.h>
+#elif defined(__APPLE__)
+#include <ena_plat_macos.h>
 #else
 #error "Invalid platform"
 #endif

As far as I can see only ena_plat_dpdk.h exists, other ena_plat_*.h files not exists at all in dpdk driver,
would it be possible to strip those lines for the dpdk version of the ena_plat.h to not confuse about the support.

If this creates additional maintanence burden, OK to continue as it is.

<...>

drivers/net/ena/base/ena_plat.h extern int ena_logtype_com; · line 114

Quoted 6 lines
drivers/net/ena/base/ena_plat.h +1 -0
@@ -107,6 +110,7 @@ extern int ena_logtype_com;
 #define BITS_PER_LONG_LONG (__SIZEOF_LONG_LONG__ * 8)
 #define U64_C(x) x ## ULL
 #define BIT(nr)         (1UL << (nr))
+#define BIT64(nr)	BIT(nr)
 

Can use existing RTE_BIT64 / RTE_BIT32 macros

<...>

Quoted 34 lines
drivers/net/ena/ena_ethdev.c +0 -7
index 7345e480f8..b764442dbb 100644
@@ -1171,7 +1171,6 @@ static int ena_start(struct rte_eth_dev *dev)
 	struct ena_adapter *adapter = dev->data->dev_private;
 	uint64_t ticks;
 	int rc = 0;
-	uint16_t i;
 
 	/* Cannot allocate memory in secondary process */
 	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
@@ -1209,11 +1208,6 @@ static int ena_start(struct rte_eth_dev *dev)
 	++adapter->dev_stats.dev_start;
 	adapter->state = ENA_ADAPTER_STATE_RUNNING;
 
-	for (i = 0; i < dev->data->nb_rx_queues; i++)
-		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
-	for (i = 0; i < dev->data->nb_tx_queues; i++)
-		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
-
 	return 0;
 
 err_rss_init:
@@ -1229,7 +1223,6 @@ static int ena_stop(struct rte_eth_dev *dev)
 	struct ena_com_dev *ena_dev = &adapter->ena_dev;
 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 	struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
-	uint16_t i;
 	int rc;
 
 	/* Cannot free memory in secondary process */
@@ -1261,11 +1254,6 @@ static int ena_stop(struct rte_eth_dev *dev)
 	adapter->state = ENA_ADAPTER_STATE_STOPPED;

drivers/net/ena/ena_ethdev.c static int ena_stop(struct rte_eth_dev *dev) · line 1259

Quoted 10 lines
drivers/net/ena/ena_ethdev.c +0 -5
 	dev->data->dev_started = 0;
 
-	for (i = 0; i < dev->data->nb_rx_queues; i++)
-		dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
-	for (i = 0; i < dev->data->nb_tx_queues; i++)
-		dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
-
 	return 0;
 }
 

Above queue state related changes are coming from other patch in this release, I guess removing them is a mistake, can you please double check?

#3 lore raw

RE: [PATCH v2 1/5] net/ena: hal upgrade

Quoted 64 lines

-----Original Message-----
From: Ferruh Yigit <[email protected]>
Sent: Thursday, October 26, 2023 12:30 AM
To: Brandes, Shai <[email protected]>
Cc: [email protected]; Beider, Ron <[email protected]>; Atrash, Wajeeh <[email protected]>; Bernstein, Amit <[email protected]>
Subject: RE: [EXTERNAL] [PATCH v2 1/5] net/ena: hal upgrade

CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.

Quoted 47 lines · On 10/25/2023 2:36 PM, [email protected] wrote:

From: Shai Brandes <[email protected]>

ENA maintains a HAL that is shared by all supported host drivers. Main features introduced to the HAL:
[1] Reworked the mechanism that queries the performance metrics from the device.
[2] Added support for a new metric that allows monitoring the available tracked connections.
[3] Added support for a new statistic that counts RX drops due to insufficient buffers provided by host.
[4] Added support for Scalable Reliable Datagram (SRD) metrics from ENA Express.
[5] Added support for querying the LLQ entry size recommendation from the device.
[6] Added support for PTP hardware clock (PHC) feature that provides enhanced accuracy (Not supported by the driver).
[7] Added support for new reset reasons for a suspected CPU starvation and for completion descriptor inconsistency.
[8] Aligned all return error code to a common notation.
[9] Removed an obsolete queue tail pointer update API.

---
 doc/guides/rel_notes/release_23_11.rst        |   4 +
 drivers/net/ena/base/ena_com.c                | 499 +++++++++++++++---
 drivers/net/ena/base/ena_com.h                | 197 ++++++-
 .../net/ena/base/ena_defs/ena_admin_defs.h    | 198 ++++++-
 .../net/ena/base/ena_defs/ena_eth_io_defs.h   |  18 +-
 drivers/net/ena/base/ena_defs/ena_gen_info.h  |   4 +-
 drivers/net/ena/base/ena_defs/ena_regs_defs.h |  12 +
 drivers/net/ena/base/ena_eth_com.c            |  45 +-
 drivers/net/ena/base/ena_eth_com.h            |  30 +-
 drivers/net/ena/base/ena_plat.h               |   8 +-
 drivers/net/ena/base/ena_plat_dpdk.h          |  49 +-
 drivers/net/ena/ena_ethdev.c                  |  16 +-
 12 files changed, 915 insertions(+), 165 deletions(-)
doc/guides/rel_notes/release_23_11.rst +0 -0
index 0a6fc76a9d..e3b0ba58c9 100644
@@ -144,6 +144,10 @@ New Features
 
   * Added support for Network Service Header (NSH) flow matching.
 
Quoted 7 lines

doc/guides/rel_notes/release_23_11.rst New Features · line 152

Quoted 3 lines
doc/guides/rel_notes/release_23_11.rst +2 -0
+* **Updated Amazon Elastic Network Adapter ena net driver.**
+
 

What do you think clarify as:

Updated Amazon ena (Elastic Network Adapter) net driver.

[Brandes, Shai] agree

Quoted 9 lines

doc/guides/rel_notes/release_23_11.rst New Features · line 152

Quoted 5 lines
doc/guides/rel_notes/release_23_11.rst +2 -0
+  * Upgraded ENA HAL to latest version.
+
 

Updates should be ordered in vendor in the driver group, so can you please sort it based on 'Amazon'? With current release notes, it makes before Intel driver updates.

[Brandes, Shai] fixed

Quoted 39 lines
Quoted 6 lines
doc/guides/rel_notes/release_23_11.rst +0 -0
 * **Updated Solarflare net driver.**

* Added support for transfer flow action ``INDIRECT`` with subtype

``VXLAN_ENCAP``.

Quoted 6 lines
(quoted diff) +0 -0

b/drivers/net/ena/base/ena_com.c index 5ca36ab6d9..880d047956 100644

drivers/net/ena/base/ena_com.c +0 -0
@@ -38,6 +38,12 @@
 

drivers/net/ena/base/ena_com.c line 48

Quoted 10 lines
drivers/net/ena/base/ena_com.c +6 -0
 #define ENA_MAX_ADMIN_POLL_US 5000
 
+/* PHC definitions */
+#define ENA_PHC_DEFAULT_EXPIRE_TIMEOUT_USEC 20 #define
+ENA_PHC_DEFAULT_BLOCK_TIMEOUT_USEC 1000 #define
+ENA_PHC_TIMESTAMP_ERROR 0xFFFFFFFFFFFFFFFF #define
+ENA_PHC_REQ_ID_OFFSET 0xDEAD
+
 
 

/**********************************************************
***********

drivers/net/ena/base/ena_com.c line 49

Quoted 2 lines
drivers/net/ena/base/ena_com.c +0 -0
 ********/

/**********************************************************
***********

Quoted 2 lines

********/

/**********************************************************
***********

Quoted 5 lines

********/ @@ -70,7 +76,7 @@ static int ena_com_mem_addr_set(struct ena_com_dev *ena_dev,

                                     dma_addr_t addr)  {
      if ((addr & GENMASK_ULL(ena_dev->dma_addr_bits - 1, 0)) != addr) {

- ena_trc_err(ena_dev, "DMA address has more bits than the device

supports\n");

Quoted 9 lines
Quoted 3 lines

+ ena_trc_err(ena_dev, "DMA address has more bits that the
+ device supports\n");

Original wording looks better to me, 'than' instead of 'that', but we can get another opinion from a native English speaker.

<...>

[Brandes, Shai] agree, fixed

Quoted 30 lines
Quoted 15 lines
(quoted diff) +0 -0

b/drivers/net/ena/base/ena_plat.h index 2583823080..a3649e0cb6 100644

drivers/net/ena/base/ena_plat.h +3 -1
@@ -14,14 +14,16 @@
 #else
 #include <ena_plat_dpdk.h>
 #endif
+#elif defined(_WIN32)
+#include <ena_plat_windows.h>
 #elif defined(__FreeBSD__)
-#if defined(_KERNEL)
+#if defined(__KERNEL__)
 #include <ena_plat_fbsd.h>

drivers/net/ena/base/ena_plat.h line 29

Quoted 10 lines
drivers/net/ena/base/ena_plat.h +2 -2
 #else
 #include <ena_plat_dpdk.h>
 #endif
-#elif defined(_WIN32)
-#include <ena_plat_windows.h>
+#elif defined(__APPLE__)
+#include <ena_plat_macos.h>
 #else
 #error "Invalid platform"
 #endif

As far as I can see only ena_plat_dpdk.h exists, other ena_plat_*.h files not exists at all in dpdk driver, would it be possible to strip those lines for the dpdk version of the ena_plat.h to not confuse about the support.

Quoted 2 lines

If this creates additional maintanence burden, OK to continue as it is.

[Brandes, Shai] removed all, no problem

Quoted 6 lines

<...>

drivers/net/ena/base/ena_plat.h extern int ena_logtype_com; #define · line 110

Quoted 2 lines
drivers/net/ena/base/ena_plat.h +0 -0
@@ -107,6 +110,7 @@ extern int ena_logtype_com;  #define
 BITS_PER_LONG_LONG (__SIZEOF_LONG_LONG__ * 8)  #define U64_C(x)

x ##

Quoted 8 lines
Quoted 4 lines

ULL
#define BIT(nr) (1UL << (nr))
+#define BIT64(nr) BIT(nr)

Can use existing RTE_BIT64 / RTE_BIT32 macros

<...>

[Brandes, Shai] changed, thanks for the notice

Quoted 44 lines
Quoted 9 lines
(quoted diff) +0 -0

b/drivers/net/ena/ena_ethdev.c index 7345e480f8..b764442dbb 100644

drivers/net/ena/ena_ethdev.c +0 -0
@@ -1171,7 +1171,6 @@ static int ena_start(struct rte_eth_dev *dev)
      struct ena_adapter *adapter = dev->data->dev_private;
      uint64_t ticks;
      int rc = 0;

drivers/net/ena/ena_ethdev.c static int ena_start(struct rte_eth_dev *dev)

Quoted 10 lines
drivers/net/ena/ena_ethdev.c +1 -3
-     uint16_t i;
 
      /* Cannot allocate memory in secondary process */
      if (rte_eal_process_type() != RTE_PROC_PRIMARY) { @@ -1209,11
+1208,6 @@ static int ena_start(struct rte_eth_dev *dev)
      ++adapter->dev_stats.dev_start;
      adapter->state = ENA_ADAPTER_STATE_RUNNING;
 
-     for (i = 0; i < dev->data->nb_rx_queues; i++)
-             dev->data->rx_queue_state[i] =

RTE_ETH_QUEUE_STATE_STARTED;

drivers/net/ena/ena_ethdev.c

Quoted 2 lines
drivers/net/ena/ena_ethdev.c +0 -2
-     for (i = 0; i < dev->data->nb_tx_queues; i++)
-             dev->data->tx_queue_state[i] =

RTE_ETH_QUEUE_STATE_STARTED;

Quoted 8 lines
drivers/net/ena/ena_ethdev.c +0 -1
-
      return 0;
 
 err_rss_init:
@@ -1229,7 +1223,6 @@ static int ena_stop(struct rte_eth_dev *dev)
      struct ena_com_dev *ena_dev = &adapter->ena_dev;
      struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
      struct rte_intr_handle *intr_handle = pci_dev->intr_handle;

drivers/net/ena/ena_ethdev.c static int ena_stop(struct rte_eth_dev *dev)

Quoted 10 lines
drivers/net/ena/ena_ethdev.c +1 -3
-     uint16_t i;
      int rc;
 
      /* Cannot free memory in secondary process */ @@ -1261,11
+1254,6 @@ static int ena_stop(struct rte_eth_dev *dev)
      adapter->state = ENA_ADAPTER_STATE_STOPPED;
      dev->data->dev_started = 0;
 
-     for (i = 0; i < dev->data->nb_rx_queues; i++)
-             dev->data->rx_queue_state[i] =

RTE_ETH_QUEUE_STATE_STOPPED;

drivers/net/ena/ena_ethdev.c

Quoted 2 lines
drivers/net/ena/ena_ethdev.c +0 -2
-     for (i = 0; i < dev->data->nb_tx_queues; i++)
-             dev->data->tx_queue_state[i] =
Quoted 8 lines

RTE_ETH_QUEUE_STATE_STOPPED;

Quoted 4 lines

-

      return 0;

}

Above queue state related changes are coming from other patch in this release, I guess removing them is a mistake, can you please double check?

[Brandes, Shai] good catch, indeed was removed accidently

#4 lore raw

PATCH v2 2/5 net/ena: add support for connection tracking metric

In reply to [email protected]

From: Shai Brandes <[email protected]>

The driver publishes network performance metrics that the application can use to troubleshoot performance issues, monitor the workload, and benchmark applications to determine whether they maximize the performance.

This patch adds support for the connection tracking allowance utilization metric (Conntrack_allowance_available) that allows monitoring the available tracked connections that can be established before the interface exceeds its allowance.

The driver uses the redesigned HAL mechanism that is backward compatible with the old method to query the metrics.

Probing the customer metrics is performed via an admin command. Thus, a multi-process proxy handler was added.

---
 doc/guides/rel_notes/release_23_11.rst |   1 +
 drivers/net/ena/ena_ethdev.c           | 198 +++++++++++++++++--------
 drivers/net/ena/ena_ethdev.h           |  24 ++-
 3 files changed, 161 insertions(+), 62 deletions(-)
doc/guides/rel_notes/release_23_11.rst +1 -0
index e3b0ba58c9..eefbcc08fe 100644
@@ -147,6 +147,7 @@ New Features
 * **Updated Amazon Elastic Network Adapter ena net driver.**
 
   * Upgraded ENA HAL to latest version.
+  * Added support for connection tracking allowance utilization metric.
 
 * **Updated Solarflare net driver.**
 
drivers/net/ena/ena_ethdev.c +140 -58

374 lines not on this page · raw message

drivers/net/ena/ena_ethdev.h +20 -4
index 4d27fba585..9268d44dde 100644
@@ -12,9 +12,9 @@
 #include <ethdev_pci.h>
 #include <rte_cycles.h>
 #include <rte_pci.h>
-#include <bus_pci_driver.h>
+#include <rte_bus_pci.h>
 #include <rte_timer.h>
-#include <dev_driver.h>
+#include <rte_dev.h>
 #include <rte_net.h>
 
 #include "ena_com.h"
@@ -54,6 +54,12 @@
 #define ENA_REFILL_THRESH_DIVIDER      8
 #define ENA_REFILL_THRESH_PACKET       256
 
+/*
+ * The max customer metrics is equal or bigger than the ENI metrics. That
+ * assumption simplifies the fallback to the legacy metrics mechanism.
+ */
+#define ENA_MAX_CUSTOMER_METRICS	6
+
 #define ENA_IDX_NEXT_MASKED(idx, mask) (((idx) + 1) & (mask))
 #define ENA_IDX_ADD_MASKED(idx, n, mask) (((idx) + (n)) & (mask))
 
@@ -215,7 +221,7 @@ struct ena_stats_dev {
 	u64 tx_drops;
 };
 
-struct ena_stats_eni {
+struct ena_stats_metrics {
 	/*
 	 * The number of packets shaped due to inbound aggregate BW
 	 * allowance being exceeded
@@ -239,6 +245,10 @@ struct ena_stats_eni {
 	 * allowance being exceeded
 	 */
 	uint64_t linklocal_allowance_exceeded;
+	 /*
+	  * The number of available connections
+	  */
+	uint64_t conntrack_allowance_available;
 };
 
 struct ena_offloads {
@@ -293,7 +303,6 @@ struct ena_adapter {
 	uint64_t keep_alive_timeout;
 
 	struct ena_stats_dev dev_stats;
-	struct ena_stats_eni eni_stats;
 	struct ena_admin_basic_stats basic_stats;
 
 	u32 indirect_table[ENA_RX_RSS_TABLE_SIZE];
@@ -312,6 +321,13 @@ struct ena_adapter {
 	uint64_t tx_cleanup_stall_delay;
 
 	uint64_t memzone_cnt;
+
+	/*
+	 * Helper variables for holding the information about the supported
+	 * metrics.
+	 */
+	uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;
+	uint16_t metrics_num;
 };
 
 int ena_mp_indirect_table_set(struct ena_adapter *adapter);
Signature · 1 line
2.17.1
#5 lore raw

PATCH v2 3/5 net/ena: report Rx overrun errors in xstats

In reply to [email protected]

From: Shai Brandes <[email protected]>

RX overrun error occur when a packet arrives but there are not enough free buffers in the RX ring to receive it. The driver publishes the extended statistics with the RX buffer overrun errors as reported by the device.

---
 doc/guides/rel_notes/release_23_11.rst | 1 +
 drivers/net/ena/ena_ethdev.c           | 4 ++++
 drivers/net/ena/ena_ethdev.h           | 1 +
 3 files changed, 6 insertions(+)
doc/guides/rel_notes/release_23_11.rst +1 -0
index eefbcc08fe..f622d93384 100644
@@ -148,6 +148,7 @@ New Features
 
   * Upgraded ENA HAL to latest version.
   * Added support for connection tracking allowance utilization metric.
+  * Added support for reporting rx overrun errors in xstats.
 
 * **Updated Solarflare net driver.**
 
drivers/net/ena/ena_ethdev.c +4 -0
index daec7f7d16..b3ebda6049 100644
@@ -90,6 +90,7 @@ static const struct ena_stats ena_stats_global_strings[] = {
 	ENA_STAT_GLOBAL_ENTRY(dev_start),
 	ENA_STAT_GLOBAL_ENTRY(dev_stop),
 	ENA_STAT_GLOBAL_ENTRY(tx_drops),
+	ENA_STAT_GLOBAL_ENTRY(rx_overruns),
 };
 
 /*
@@ -3894,15 +3895,18 @@ static void ena_keep_alive(void *adapter_data,
 	struct ena_admin_aenq_keep_alive_desc *desc;
 	uint64_t rx_drops;
 	uint64_t tx_drops;
+	uint64_t rx_overruns;
 
 	adapter->timestamp_wd = rte_get_timer_cycles();
 
 	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
 	rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
 	tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low;
+	rx_overruns = ((uint64_t)desc->rx_overruns_high << 32) | desc->rx_overruns_low;
 
 	adapter->drv_stats->rx_drops = rx_drops;
 	adapter->dev_stats.tx_drops = tx_drops;
+	adapter->dev_stats.rx_overruns = rx_overruns;
 }
 
 /**
drivers/net/ena/ena_ethdev.h +1 -0
index 9268d44dde..3f29764ca6 100644
@@ -219,6 +219,7 @@ struct ena_stats_dev {
 	 * As a workaround it is being published as an extended statistic.
 	 */
 	u64 tx_drops;
+	u64 rx_overruns;
 };
 
 struct ena_stats_metrics {
Signature · 1 line
2.17.1
#6 lore raw

PATCH v2 4/5 net/ena: add support for ena-express metrics

In reply to [email protected]

From: Shai Brandes <[email protected]>

ENA-express is powered by AWS scalable reliable datagram (SRD) technology. SRD is a high performance network transport protocol that uses dynamic routing to increase throughput and minimize tail latency.

The driver expose the following ENA-express metrics via xstats:
* ena_srd_mode – Describes which ENA-express features are enabled
* ena_srd_eligible_tx_pkts – The number of network packets sent within a given time period that meet SRD requirements for eligibility
* ena_srd_tx_pkts – The number of SRD packets transmitted within a given time period.
* ena_srd_rx_pkts – The number of SRD packets received within a given time period.
* ena_srd_resource_utilization – The percentage of the maximum allowed memory utilization for concurrent SRD connections that the instance has consumed.

Probing the ENA Express metrics is performed via an admin command. Thus, a multi-process proxy handler was added.

---
 doc/guides/rel_notes/release_23_11.rst |   1 +
 drivers/net/ena/ena_ethdev.c           | 105 ++++++++++++++++++++++++-
 drivers/net/ena/ena_ethdev.h           |  18 +++++
 3 files changed, 123 insertions(+), 1 deletion(-)
doc/guides/rel_notes/release_23_11.rst +1 -0
index f622d93384..7aaa78e6c9 100644
@@ -149,6 +149,7 @@ New Features
   * Upgraded ENA HAL to latest version.
   * Added support for connection tracking allowance utilization metric.
   * Added support for reporting rx overrun errors in xstats.
+  * Added support for ENA-express metrics.
 
 * **Updated Solarflare net driver.**
 
drivers/net/ena/ena_ethdev.c +104 -1

200 lines not on this page · raw message

drivers/net/ena/ena_ethdev.h +18 -0
index 3f29764ca6..4988fbffb5 100644
@@ -252,6 +252,23 @@ struct ena_stats_metrics {
 	uint64_t conntrack_allowance_available;
 };
 
+struct ena_stats_srd {
+	/* Describes which ENA Express features are enabled */
+	uint64_t ena_srd_mode;
+
+	/* Number of packets transmitted over ENA SRD */
+	uint64_t ena_srd_tx_pkts;
+
+	/* Number of packets transmitted or could have been transmitted over ENA SRD */
+	uint64_t ena_srd_eligible_tx_pkts;
+
+	/* Number of packets received over ENA SRD */
+	uint64_t ena_srd_rx_pkts;
+
+	/* Percentage of the ENA SRD resources that is in use */
+	uint64_t ena_srd_resource_utilization;
+};
+
 struct ena_offloads {
 	uint32_t tx_offloads;
 	uint32_t rx_offloads;
@@ -329,6 +346,7 @@ struct ena_adapter {
 	 */
 	uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;
 	uint16_t metrics_num;
+	struct ena_stats_srd srd_stats __rte_cache_aligned;
 };
 
 int ena_mp_indirect_table_set(struct ena_adapter *adapter);
Signature · 1 line
2.17.1
#7 lore raw

PATCH v2 5/5 net/ena: update ena version to 2.8.0

In reply to [email protected]

From: Shai Brandes <[email protected]>

This release introduces:
* Upgraded ENA HAL.
* Support for connection tracking allowance utilization metric.
* Support for reporting rx overrun errors in xstats.
* Support for ENA-express metrics.

---
 drivers/net/ena/ena_ethdev.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/net/ena/ena_ethdev.c +1 -1
index 59cb2792d4..591bdf864f 100644
@@ -21,7 +21,7 @@
 #include <ena_eth_io_defs.h>
 
 #define DRV_MODULE_VER_MAJOR	2
-#define DRV_MODULE_VER_MINOR	7
+#define DRV_MODULE_VER_MINOR	8
 #define DRV_MODULE_VER_SUBMINOR	0
 
 #define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l)
Signature · 1 line
2.17.1