net/ena: v2.8.0 driver release
cover net/ena: v2.8.0 driver release
From: Shai Brandes <[email protected]>
Hi,
This patchset contains alignment of the driver to the latest HAL version which adds support for retrieving new metrics from the device and opens a path to use additional device features that are not yet supported by the driver.
The new driver features are mostly about adding additional metrics. We added support for an additional customer metric, ENA-express metrics and are now reporting also rx overrun errors. All new metrics have MP support.
* Testing:
The team performed directed tests for all the features and executed our proprietary performance benchamarking (BW, PPS, Latency) to verify there is no regression. It was executed on the entire Amazon EC2 instance types matrix that covers all device generations and CPU architectures.
Shai Brandes (5):
net/ena: hal upgrade
net/ena: add support for connection tracking metric net/ena: report Rx overrun errors in xstats net/ena: add support for ena-express metrics net/ena: update ena version to 2.8.0
doc/guides/rel_notes/release_23_11.rst | 7 + drivers/net/ena/base/ena_com.c | 499 +++++++++++++++--- drivers/net/ena/base/ena_com.h | 197 ++++++- .../net/ena/base/ena_defs/ena_admin_defs.h | 198 ++++++- .../net/ena/base/ena_defs/ena_eth_io_defs.h | 18 +- drivers/net/ena/base/ena_defs/ena_gen_info.h | 4 +- drivers/net/ena/base/ena_defs/ena_regs_defs.h | 12 + drivers/net/ena/base/ena_eth_com.c | 45 +- drivers/net/ena/base/ena_eth_com.h | 30 +- drivers/net/ena/base/ena_plat.h | 8 +- drivers/net/ena/base/ena_plat_dpdk.h | 49 +- drivers/net/ena/ena_ethdev.c | 321 ++++++++--- drivers/net/ena/ena_ethdev.h | 43 +- 13 files changed, 1204 insertions(+), 227 deletions(-)
Signature · 1 line
2.17.1
PATCH 1/5 net/ena: hal upgrade
From: Shai Brandes <[email protected]>
ENA maintains a HAL that is shared by all supported host drivers. Main features introduced to the HAL:
[1] Reworked the mechanism that queries the performance metrics from the device.
[2] Added support for a new metric that allows monitoring the available tracked connections.
[3] Added support for a new statistic that counts RX drops due to insufficient buffers provided by host.
[4] Added support for Scalable Reliable Datagram (SRD) metrics from ENA Express.
[5] Added support for querying the LLQ entry size recommendation from the device.
[6] Added support for PTP hardware clock (PHC) feature that provides enhanced accuracy (Not supported by the driver).
[7] Added support for new reset reasons for a suspected CPU starvation and for completion descriptor inconsistency.
[8] Aligned all return error code to a common notation.
[9] Removed an obsolete queue tail pointer update API.
--- doc/guides/rel_notes/release_23_11.rst | 4 + drivers/net/ena/base/ena_com.c | 499 +++++++++++++++--- drivers/net/ena/base/ena_com.h | 197 ++++++- .../net/ena/base/ena_defs/ena_admin_defs.h | 198 ++++++- .../net/ena/base/ena_defs/ena_eth_io_defs.h | 18 +- drivers/net/ena/base/ena_defs/ena_gen_info.h | 4 +- drivers/net/ena/base/ena_defs/ena_regs_defs.h | 12 + drivers/net/ena/base/ena_eth_com.c | 45 +- drivers/net/ena/base/ena_eth_com.h | 30 +- drivers/net/ena/base/ena_plat.h | 8 +- drivers/net/ena/base/ena_plat_dpdk.h | 49 +- drivers/net/ena/ena_ethdev.c | 16 +- 12 files changed, 915 insertions(+), 165 deletions(-)
doc/guides/rel_notes/release_23_11.rst +4 -0
@@ -144,6 +144,10 @@ New Features * Added support for Network Service Header (NSH) flow matching. +* **Updated Amazon Elastic Network Adapter ena net driver.** + + * Upgraded ENA HAL to latest version. + * **Updated Solarflare net driver.** * Added support for transfer flow action ``INDIRECT`` with subtype ``VXLAN_ENCAP``.
drivers/net/ena/base/ena_com.c +424 -75
700 lines not on this page · raw message
drivers/net/ena/base/ena_com.h +185 -12
329 lines not on this page · raw message
drivers/net/ena/base/ena_defs/ena_admin_defs.h +191 -7
327 lines not on this page · raw message
drivers/net/ena/base/ena_defs/ena_eth_io_defs.h +17 -1
@@ -261,7 +261,8 @@ struct ena_eth_io_intr_reg {/* 14:0 : rx_intr_delay * 29:15 : tx_intr_delay * 30 : intr_unmask -* 31 : reserved +* 31 : no_moderation_update - 0 - moderation +* updated, 1 - moderation not updated */ uint32_t intr_control; };
@@ -381,6 +382,8 @@ struct ena_eth_io_numa_node_cfg_reg {#define ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK GENMASK(29, 15)#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT 30#define ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK BIT(30) +#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT 31 +#define ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK BIT(31)/* numa_node_cfg_reg */ #define ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK GENMASK(7, 0)
@@ -918,6 +921,19 @@ static inline void set_ena_eth_io_intr_reg_intr_unmask(struct ena_eth_io_intr_re p->intr_control |= (val <<ENA_ETH_IO_INTR_REG_INTR_UNMASK_SHIFT ) &ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK ; } +static inline uint32_t get_ena_eth_io_intr_reg_no_mod_update (const struct ena_eth_io_intr_reg *p) +{ +return (p->intr_control &ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK ) >> +ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT ; +} + +static inline void set_ena_eth_io_intr_reg_no_mod_update (struct ena_eth_io_intr_reg *p, +uint32_t val) +{ + p->intr_control |= (val <<ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_SHIFT ) & +ENA_ETH_IO_INTR_REG_NO_MODERATION_UPDATE_MASK ; +} +static inline uint32_t get_ena_eth_io_numa_node_cfg_reg_numa (const struct ena_eth_io_numa_node_cfg_reg *p) {return p->numa_cfg &ENA_ETH_IO_NUMA_NODE_CFG_REG_NUMA_MASK ;
drivers/net/ena/base/ena_defs/ena_gen_info.h +2 -2
@@ -2,5 +2,5 @@* Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. * All rights reserved. */ -#define ENA_GEN_DATE "Fri Sep 18 17:09:00 IDT 2020" -#define ENA_GEN_COMMIT "0f80d82" +#define ENA_GEN_DATE "Thu 14 Apr 2022 12:50:30 PM IDT" +#define ENA_GEN_COMMIT "35388392"
drivers/net/ena/base/ena_defs/ena_regs_defs.h +12 -0
@@ -21,6 +21,8 @@ enum ena_regs_reset_reason_types {ENA_REGS_RESET_USER_TRIGGER =12 ,ENA_REGS_RESET_GENERIC =13 ,ENA_REGS_RESET_MISS_INTERRUPT =14 , +ENA_REGS_RESET_SUSPECTED_POLL_STARVATION =15 , +ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED =16 , ENA_REGS_RESET_LAST, };
@@ -52,6 +54,11 @@ enum ena_regs_reset_reason_types {#define ENA_REGS_MMIO_RESP_HI_OFF 0x64#define ENA_REGS_RSS_IND_ENTRY_UPDATE_OFF 0x68 +/* phc_registers offsets */ + +/* 100 base */ +#define ENA_REGS_PHC_DB_OFF 0x100 +/* version register */ #define ENA_REGS_VERSION_MINOR_VERSION_MASK 0xff#define ENA_REGS_VERSION_MAJOR_VERSION_SHIFT 8
@@ -98,6 +105,8 @@ enum ena_regs_reset_reason_types {#define ENA_REGS_DEV_CTL_QUIESCENT_MASK 0x4#define ENA_REGS_DEV_CTL_IO_RESUME_SHIFT 3#define ENA_REGS_DEV_CTL_IO_RESUME_MASK 0x8 +#define ENA_REGS_DEV_CTL_RESET_REASON_EXT_SHIFT 24 +#define ENA_REGS_DEV_CTL_RESET_REASON_EXT_MASK 0xf000000#define ENA_REGS_DEV_CTL_RESET_REASON_SHIFT 28#define ENA_REGS_DEV_CTL_RESET_REASON_MASK 0xf0000000
@@ -128,4 +137,7 @@ enum ena_regs_reset_reason_types {#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_SHIFT 16#define ENA_REGS_RSS_IND_ENTRY_UPDATE_CQ_IDX_MASK 0xffff0000 +/* phc_db_req_id register */ +#define ENA_REGS_PHC_DB_REQ_ID_MASK 0xffff +#endif /* _ENA_REGS_H_ */
drivers/net/ena/base/ena_eth_com.c +28 -17
112 lines not on this page · raw message
drivers/net/ena/base/ena_eth_com.h +5 -25
@@ -11,8 +11,10 @@ extern "C" {#endif #include "ena_com.h" -/* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */ -#define ENA_COMP_HEAD_THRESH 4 +/* we allow 2 DMA descriptors per LLQ entry */ +#define ENA_LLQ_ENTRY_DESC_CHUNK_SIZE (2 * sizeof(struct ena_eth_io_tx_desc)) +#define ENA_LLQ_HEADER (128UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE) +#define ENA_LLQ_LARGE_HEADER (256UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE)struct ena_com_tx_ctx {struct ena_com_tx_meta ena_meta;
@@ -48,7 +50,7 @@ struct ena_com_rx_ctx {bool frag;u32 hash;u16 descs; -int max_bufs; +u16 max_bufs;u8 pkt_offset; };
@@ -171,28 +173,6 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq)return 0 ; } -static inline int ena_com_update_dev_comp_head (struct ena_com_io_cq *io_cq) -{ -u16 unreported_comp, head; -bool need_update; - -if (unlikely (io_cq->cq_head_db_reg)) { - head = io_cq->head; - unreported_comp = head - io_cq->last_head_update; - need_update = unreported_comp > (io_cq->q_depth /ENA_COMP_HEAD_THRESH ); - -if (unlikely (need_update)) { -ena_trc_dbg (ena_com_io_cq_to_ena_dev (io_cq), -"Write completion queue doorbell for queue %d: head: %d\n" , - io_cq->qid, head); -ENA_REG_WRITE32 (io_cq->bus, head, io_cq->cq_head_db_reg); - io_cq->last_head_update = head; - } - } - -return 0 ; -} -static inline void ena_com_update_numa_node (struct ena_com_io_cq *io_cq,u8 numa_node) {
drivers/net/ena/base/ena_plat.h +5 -3
@@ -14,14 +14,16 @@#else #include <ena_plat_dpdk.h> #endif +#elif defined(_WIN32) +#include <ena_plat_windows.h> #elif defined(__FreeBSD__) -#if defined(_KERNEL) +#if defined(__KERNEL__)#include <ena_plat_fbsd.h> #else #include <ena_plat_dpdk.h> #endif -#elif defined(_WIN32) -#include <ena_plat_windows.h> +#elif defined(__APPLE__) +#include <ena_plat_macos.h> #else #error "Invalid platform"#endif
drivers/net/ena/base/ena_plat_dpdk.h +41 -8
102 lines not on this page · raw message
drivers/net/ena/ena_ethdev.c +1 -15
@@ -1171,7 +1171,6 @@ static int ena_start(struct rte_eth_dev *dev)struct ena_adapter *adapter = dev->data->dev_private;uint64_t ticks;int rc =0 ; -uint16_t i;/* Cannot allocate memory in secondary process */ if (rte_eal_process_type () !=RTE_PROC_PRIMARY ) {
@@ -1209,11 +1208,6 @@ static int ena_start(struct rte_eth_dev *dev) ++adapter->dev_stats.dev_start; adapter->state =ENA_ADAPTER_STATE_RUNNING ; -for (i =0 ; i < dev->data->nb_rx_queues; i++) - dev->data->rx_queue_state[i] =RTE_ETH_QUEUE_STATE_STARTED ; -for (i =0 ; i < dev->data->nb_tx_queues; i++) - dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STARTED ; -return 0 ; err_rss_init:
@@ -1229,7 +1223,6 @@ static int ena_stop(struct rte_eth_dev *dev)struct ena_com_dev *ena_dev = &adapter->ena_dev;struct rte_pci_device *pci_dev =RTE_ETH_DEV_TO_PCI (dev);struct rte_intr_handle *intr_handle = pci_dev->intr_handle; -uint16_t i;int rc;/* Cannot free memory in secondary process */
@@ -1261,11 +1254,6 @@ static int ena_stop(struct rte_eth_dev *dev) adapter->state =ENA_ADAPTER_STATE_STOPPED ; dev->data->dev_started =0 ; -for (i =0 ; i < dev->data->nb_rx_queues; i++) - dev->data->rx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; -for (i =0 ; i < dev->data->nb_tx_queues; i++) - dev->data->tx_queue_state[i] =RTE_ETH_QUEUE_STATE_STOPPED ; -return 0 ; }
@@ -2687,7 +2675,6 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,/* Burst refill to save doorbells, memory barriers, const interval */ if (free_queue_entries >= rx_ring->rx_free_thresh) { -ena_com_update_dev_comp_head (rx_ring->ena_com_io_cq);ena_populate_rx_queue (rx_ring, free_queue_entries); }
@@ -3096,7 +3083,6 @@ static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)/* acknowledge completion of sent packets */ tx_ring->next_to_clean = next_to_clean;ena_com_comp_ack (tx_ring->ena_com_io_sq, total_tx_descs); -ena_com_update_dev_comp_head (tx_ring->ena_com_io_cq); }if (mbuf_cnt !=0 )
@@ -3629,7 +3615,7 @@ static void ena_rx_queue_intr_set(struct rte_eth_dev *dev,struct ena_ring *rxq = &adapter->rx_ring[queue_id];struct ena_eth_io_intr_reg intr_reg; -ena_com_update_intr_reg (&intr_reg,0 ,0 , unmask); +ena_com_update_intr_reg (&intr_reg,0 ,0 , unmask,1 );ena_com_unmask_intr (rxq->ena_com_io_cq, &intr_reg); }
Signature · 1 line
2.17.1
PATCH 2/5 net/ena: add support for connection tracking metric
In reply to [email protected]From: Shai Brandes <[email protected]>
The driver publishes network performance metrics that the application can use to troubleshoot performance issues, monitor the workload, and benchmark applications to determine whether they maximize the performance.
This patch adds support for the connection tracking allowance utilization metric (Conntrack_allowance_available) that allows monitoring the available tracked connections that can be established before the interface exceeds its allowance.
The driver uses the redesigned HAL mechanism that is backward compatible with the old method to query the metrics.
Probing the customer metrics is performed via an admin command. Thus, a multi-process proxy handler was added.
--- doc/guides/rel_notes/release_23_11.rst | 1 + drivers/net/ena/ena_ethdev.c | 198 +++++++++++++++++-------- drivers/net/ena/ena_ethdev.h | 24 ++- 3 files changed, 161 insertions(+), 62 deletions(-)
doc/guides/rel_notes/release_23_11.rst +1 -0
@@ -147,6 +147,7 @@ New Features * **Updated Amazon Elastic Network Adapter ena net driver.** * Upgraded ENA HAL to latest version. + * Added support for connection tracking allowance utilization metric. * **Updated Solarflare net driver.**
drivers/net/ena/ena_ethdev.c +140 -58
374 lines not on this page · raw message
drivers/net/ena/ena_ethdev.h +20 -4
@@ -12,9 +12,9 @@#include <ethdev_pci.h> #include <rte_cycles.h> #include <rte_pci.h> -#include <bus_pci_driver.h> +#include <rte_bus_pci.h> #include <rte_timer.h> -#include <dev_driver.h> +#include <rte_dev.h> #include <rte_net.h> #include "ena_com.h"
@@ -54,6 +54,12 @@#define ENA_REFILL_THRESH_DIVIDER 8#define ENA_REFILL_THRESH_PACKET 256 +/* +* The max customer metrics is equal or bigger than the ENI metrics. That +* assumption simplifies the fallback to the legacy metrics mechanism. +*/ +#define ENA_MAX_CUSTOMER_METRICS 6 +#define ENA_IDX_NEXT_MASKED (idx, mask) (((idx) + 1) & (mask))#define ENA_IDX_ADD_MASKED (idx, n, mask) (((idx) + (n)) & (mask))
@@ -215,7 +221,7 @@ struct ena_stats_dev {u64 tx_drops; }; -struct ena_stats_eni { +struct ena_stats_metrics {/* * The number of packets shaped due to inbound aggregate BW * allowance being exceeded
@@ -239,6 +245,10 @@ struct ena_stats_eni {* allowance being exceeded */ uint64_t linklocal_allowance_exceeded; +/* +* The number of available connections +*/ +uint64_t conntrack_allowance_available; };struct ena_offloads {
@@ -293,7 +303,6 @@ struct ena_adapter {uint64_t keep_alive_timeout;struct ena_stats_dev dev_stats; -struct ena_stats_eni eni_stats;struct ena_admin_basic_stats basic_stats;u32 indirect_table[ENA_RX_RSS_TABLE_SIZE ];
@@ -312,6 +321,13 @@ struct ena_adapter {uint64_t tx_cleanup_stall_delay;uint64_t memzone_cnt; + +/* +* Helper variables for holding the information about the supported +* metrics. +*/ + uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned; +uint16_t metrics_num; };int ena_mp_indirect_table_set (struct ena_adapter *adapter);
Signature · 1 line
2.17.1
PATCH 3/5 net/ena: report Rx overrun errors in xstats
In reply to [email protected]From: Shai Brandes <[email protected]>
RX overrun error occur when a packet arrives but there are not enough free buffers in the RX ring to receive it. The driver publishes the extended statistics with the RX buffer overrun errors as reported by the device.
--- doc/guides/rel_notes/release_23_11.rst | 1 + drivers/net/ena/ena_ethdev.c | 4 ++++ drivers/net/ena/ena_ethdev.h | 1 + 3 files changed, 6 insertions(+)
doc/guides/rel_notes/release_23_11.rst +1 -0
@@ -148,6 +148,7 @@ New Features * Upgraded ENA HAL to latest version. * Added support for connection tracking allowance utilization metric. + * Added support for reporting rx overrun errors in xstats. * **Updated Solarflare net driver.**
drivers/net/ena/ena_ethdev.c +4 -0
@@ -90,6 +90,7 @@ static const struct ena_stats ena_stats_global_strings[] = {ENA_STAT_GLOBAL_ENTRY (dev_start),ENA_STAT_GLOBAL_ENTRY (dev_stop),ENA_STAT_GLOBAL_ENTRY (tx_drops), + ENA_STAT_GLOBAL_ENTRY(rx_overruns), };/*
@@ -3894,15 +3895,18 @@ static void ena_keep_alive(void *adapter_data,struct ena_admin_aenq_keep_alive_desc *desc;uint64_t rx_drops;uint64_t tx_drops; +uint64_t rx_overruns; adapter->timestamp_wd =rte_get_timer_cycles (); desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; rx_drops = ((uint64_t )desc->rx_drops_high <<32 ) | desc->rx_drops_low; tx_drops = ((uint64_t )desc->tx_drops_high <<32 ) | desc->tx_drops_low; + rx_overruns = ((uint64_t )desc->rx_overruns_high <<32 ) | desc->rx_overruns_low; adapter->drv_stats->rx_drops = rx_drops; adapter->dev_stats.tx_drops = tx_drops; + adapter->dev_stats.rx_overruns = rx_overruns; }/**
drivers/net/ena/ena_ethdev.h +1 -0
@@ -219,6 +219,7 @@ struct ena_stats_dev {* As a workaround it is being published as an extended statistic. */ u64 tx_drops; +u64 rx_overruns; };struct ena_stats_metrics {
Signature · 1 line
2.17.1
PATCH 4/5 net/ena: add support for ena-express metrics
In reply to [email protected]From: Shai Brandes <[email protected]>
ENA-express is powered by AWS scalable reliable datagram (SRD) technology. SRD is a high performance network transport protocol that uses dynamic routing to increase throughput and minimize tail latency.
The driver expose the following ENA-express metrics via xstats:
* ena_srd_mode – Describes which ENA-express features are enabled
* ena_srd_eligible_tx_pkts – The number of network packets sent within a given time period that meet SRD requirements for eligibility
* ena_srd_tx_pkts – The number of SRD packets transmitted within a given time period.
* ena_srd_rx_pkts – The number of SRD packets received within a given time period.
* ena_srd_resource_utilization – The percentage of the maximum allowed memory utilization for concurrent SRD connections that the instance has consumed.
Probing the ENA Express metrics is performed via an admin command. Thus, a multi-process proxy handler was added.
--- doc/guides/rel_notes/release_23_11.rst | 1 + drivers/net/ena/ena_ethdev.c | 105 ++++++++++++++++++++++++- drivers/net/ena/ena_ethdev.h | 18 +++++ 3 files changed, 123 insertions(+), 1 deletion(-)
doc/guides/rel_notes/release_23_11.rst +1 -0
@@ -149,6 +149,7 @@ New Features * Upgraded ENA HAL to latest version. * Added support for connection tracking allowance utilization metric. * Added support for reporting rx overrun errors in xstats. + * Added support for ENA-express metrics. * **Updated Solarflare net driver.**
drivers/net/ena/ena_ethdev.c +104 -1
200 lines not on this page · raw message
drivers/net/ena/ena_ethdev.h +18 -0
@@ -252,6 +252,23 @@ struct ena_stats_metrics {uint64_t conntrack_allowance_available; }; +struct ena_stats_srd { +/* Describes which ENA Express features are enabled */ +uint64_t ena_srd_mode; + +/* Number of packets transmitted over ENA SRD */ +uint64_t ena_srd_tx_pkts; + +/* Number of packets transmitted or could have been transmitted over ENA SRD */ +uint64_t ena_srd_eligible_tx_pkts; + +/* Number of packets received over ENA SRD */ +uint64_t ena_srd_rx_pkts; + +/* Percentage of the ENA SRD resources that is in use */ +uint64_t ena_srd_resource_utilization; +}; +struct ena_offloads {uint32_t tx_offloads;uint32_t rx_offloads;
@@ -329,6 +346,7 @@ struct ena_adapter {*/ uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;uint16_t metrics_num; + struct ena_stats_srd srd_stats __rte_cache_aligned; };int ena_mp_indirect_table_set (struct ena_adapter *adapter);
Signature · 1 line
2.17.1
Re: [PATCH 4/5] net/ena: add support for ena-express metrics
drivers/net/ena/ena_ethdev.h struct ena_adapter { · line 350
Quoted 9 lines · On Tue, 24 Oct 2023 13:21:27 +0300 <[email protected]> wrote:
struct
ena_offloads{drivers/net/ena/ena_ethdev.h +1
-0uint32_t tx_offloads;uint32_t rx_offloads;@@ -329,6 +346,7 @@ struct ena_adapter {*/ uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;uint16_t metrics_num; + struct ena_stats_srd srd_stats __rte_cache_aligned; };
If metrics_num was before the metrics_stats[] you would save some space.
RE: [PATCH 4/5] net/ena: add support for ena-express metrics
Quoted 11 lines · On Tue, 24 Oct 2023 13:21:27 +0300 <[email protected]> wrote:
drivers/net/ena/ena_ethdev.h struct ena_adapter { · line 350
Quoted 9 lines
struct ena_offloads { uint32_t tx_offloads; uint32_t rx_offloads;drivers/net/ena/ena_ethdev.h +1
-0@@ -329,6 +346,7 @@ struct ena_adapter {*/ uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;uint16_t metrics_num; + struct ena_stats_srd srd_stats __rte_cache_aligned; };If
metrics_numwas before the metrics_stats[] you would save some space.
Thank you for the review! My reasoning was that the metrics_num should reside in the same cache line as the metrics_stats (metrics_stats can contain up to 6 metrics so its max length is 48B). In addition, the struct ena_stats_srd was added in a later patch, so I will run pahole and verify which order of these three fields gives the best results. BTW, we are starting a cache optimization task now that should address all these issues, this is indeed one of our main structures, so it will definitely get modified later.
RE: [PATCH 4/5] net/ena: add support for ena-express metrics
Quoted 11 lines · On Tue, 24 Oct 2023 13:21:27 +0300 <[email protected]> wrote:
drivers/net/ena/ena_ethdev.h struct ena_adapter { · line 350
Quoted 9 lines
struct ena_offloads { uint32_t tx_offloads; uint32_t rx_offloads;drivers/net/ena/ena_ethdev.h +1
-0@@ -329,6 +346,7 @@ struct ena_adapter {*/ uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS] __rte_cache_aligned;uint16_t metrics_num; + struct ena_stats_srd srd_stats __rte_cache_aligned; };If
metrics_numwas before the metrics_stats[] you would save some space.
Hi, I checked it with pahole and both ways provide same structure size and overall same padding size (14B):
uint64_t metrics_stats[6] __attribute__((__aligned__(64))); /* 181952 48 */
uint16_t metrics_num; /* 182000 2 */
/* XXX 14 bytes hole, try to pack */
/* --- cacheline 2844 boundary (182016 bytes) --- */
struct ena_stats_srd srd_stats __attribute__((__aligned__(64))); /* 182016 40 */
/* size: 182080, cachelines: 2845, members: 40 */
/* sum members: 181910, holes: 9, sum holes: 146 */Vs:
uint16_t metrics_num __attribute__((__aligned__(64))); /* 181952 2 */
/* XXX 6 bytes hole, try to pack */
uint64_t metrics_stats[6]; /* 181960 48 */
/* XXX 8 bytes hole, try to pack */
/* --- cacheline 2844 boundary (182016 bytes) --- */
struct ena_stats_srd srd_stats __attribute__((__aligned__(64))); /* 182016 40 */
/* size: 182080, cachelines: 2845, members: 40 */
/* sum members: 181910, holes: 10, sum holes: 146 */PATCH 5/5 net/ena: update ena version to 2.8.0
In reply to [email protected]From: Shai Brandes <[email protected]>
This release introduces:
* Upgraded ENA HAL.
* Support for connection tracking allowance utilization metric.
* Support for reporting rx overrun errors in xstats.
* Support for ENA-express metrics.
--- drivers/net/ena/ena_ethdev.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/net/ena/ena_ethdev.c +1 -1
@@ -21,7 +21,7 @@#include <ena_eth_io_defs.h> #define DRV_MODULE_VER_MAJOR 2 -#define DRV_MODULE_VER_MINOR 7 +#define DRV_MODULE_VER_MINOR 8#define DRV_MODULE_VER_SUBMINOR 0#define __MERGE_64B_H_L (h, l) (((uint64_t)h << 32) | l)
Signature · 1 line
2.17.1