net/ena: fix assigning NUMA node to IO queue
PATCH 1/1 net/ena: fix assigning NUMA node to IO queue
Previous solution was using memzones in invalid way in hope to assign IO queue to the appropriate NUMA zone.
The right way is to use socket_id from the rx/tx queue setup function and then pass it to the IO queue.
--- drivers/net/ena/ena_ethdev.c | 24 +++++------------------- drivers/net/ena/ena_ethdev.h | 2 ++ 2 files changed, 7 insertions(+), 19 deletions(-)
drivers/net/ena/ena_ethdev.c +5 -19
@@ -272,22 +272,6 @@ static const struct eth_dev_ops ena_dev_ops = { .reta_query = ena_rss_reta_query, }; -#define NUMA_NO_NODE SOCKET_ID_ANY - -static inline int ena_cpu_to_node (int cpu) -{ -struct rte_config *config =rte_eal_get_configuration (); -struct rte_fbarray *arr = &config->mem_config->memzones; -const struct rte_memzone *mz; - -if (unlikely (cpu >=RTE_MAX_MEMZONE )) -return NUMA_NO_NODE ; - - mz =rte_fbarray_get (arr, cpu); - -return mz->socket_id; -} -static inline void ena_rx_mbuf_prepare (struct rte_mbuf *mbuf,struct ena_com_rx_ctx *ena_rx_ctx) {
@@ -1126,7 +1110,7 @@ static int ena_create_io_queue(struct ena_ring *ring) } ctx.qid = ena_qid; ctx.msix_vector =-1 ;/* interrupts not used */ - ctx.numa_node =ena_cpu_to_node (ring->id); + ctx.numa_node = ring->numa_socket_id; rc =ena_com_create_io_queue (ena_dev, &ctx);if (rc) {
@@ -1224,7 +1208,7 @@ static int ena_queue_start(struct ena_ring *ring)static int ena_tx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id,const struct rte_eth_txconf *tx_conf) {struct ena_ring *txq =NULL ;
@@ -1262,6 +1246,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, txq->next_to_clean =0 ; txq->next_to_use =0 ; txq->ring_size = nb_desc; + txq->numa_socket_id = socket_id; txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", sizeof(struct ena_tx_buffer) *
@@ -1309,7 +1294,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,static int ena_rx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id, __rte_unused const struct rte_eth_rxconf *rx_conf,struct rte_mempool *mp) {
@@ -1347,6 +1332,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, rxq->next_to_clean =0 ; rxq->next_to_use =0 ; rxq->ring_size = nb_desc; + rxq->numa_socket_id = socket_id; rxq->mb_pool = mp; rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info",
drivers/net/ena/ena_ethdev.h +2 -0
@@ -143,6 +143,8 @@ struct ena_ring {struct ena_stats_rx rx_stats;struct ena_stats_tx tx_stats; }; + +unsigned int numa_socket_id; } __rte_cache_aligned;enum ena_adapter_state {
Signature · 1 line
2.7.4
PATCH v2 1/1 net/ena: fix assigning NUMA node to IO queue
Previous solution was using memzones in invalid way in hope to assign IO queue to the appropriate NUMA zone.
The right way is to use socket_id from the rx/tx queue setup function and then pass it to the IO queue.
---
v2:
* Remove gerrit change id
* Add dpdk-stable to cc
drivers/net/ena/ena_ethdev.c | 24 +++++------------------- drivers/net/ena/ena_ethdev.h | 2 ++ 2 files changed, 7 insertions(+), 19 deletions(-)
drivers/net/ena/ena_ethdev.c +5 -19
@@ -272,22 +272,6 @@ static const struct eth_dev_ops ena_dev_ops = { .reta_query = ena_rss_reta_query, }; -#define NUMA_NO_NODE SOCKET_ID_ANY - -static inline int ena_cpu_to_node (int cpu) -{ -struct rte_config *config =rte_eal_get_configuration (); -struct rte_fbarray *arr = &config->mem_config->memzones; -const struct rte_memzone *mz; - -if (unlikely (cpu >=RTE_MAX_MEMZONE )) -return NUMA_NO_NODE ; - - mz =rte_fbarray_get (arr, cpu); - -return mz->socket_id; -} -static inline void ena_rx_mbuf_prepare (struct rte_mbuf *mbuf,struct ena_com_rx_ctx *ena_rx_ctx) {
@@ -1126,7 +1110,7 @@ static int ena_create_io_queue(struct ena_ring *ring) } ctx.qid = ena_qid; ctx.msix_vector =-1 ;/* interrupts not used */ - ctx.numa_node =ena_cpu_to_node (ring->id); + ctx.numa_node = ring->numa_socket_id; rc =ena_com_create_io_queue (ena_dev, &ctx);if (rc) {
@@ -1224,7 +1208,7 @@ static int ena_queue_start(struct ena_ring *ring)static int ena_tx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id,const struct rte_eth_txconf *tx_conf) {struct ena_ring *txq =NULL ;
@@ -1262,6 +1246,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, txq->next_to_clean =0 ; txq->next_to_use =0 ; txq->ring_size = nb_desc; + txq->numa_socket_id = socket_id; txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", sizeof(struct ena_tx_buffer) *
@@ -1309,7 +1294,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,static int ena_rx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id, __rte_unused const struct rte_eth_rxconf *rx_conf,struct rte_mempool *mp) {
@@ -1347,6 +1332,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, rxq->next_to_clean =0 ; rxq->next_to_use =0 ; rxq->ring_size = nb_desc; + rxq->numa_socket_id = socket_id; rxq->mb_pool = mp; rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info",
drivers/net/ena/ena_ethdev.h +2 -0
@@ -143,6 +143,8 @@ struct ena_ring {struct ena_stats_rx rx_stats;struct ena_stats_tx tx_stats; }; + +unsigned int numa_socket_id; } __rte_cache_aligned;enum ena_adapter_state {
Signature · 1 line
2.7.4
Quoted 34 lines · Michal Krawczyk
Previous solution was using memzones in invalid way in hope to assign IO queue to the appropriate NUMA zone.
The right way is to use
socket_idfrom the rx/tx queue setup function and then pass it to the IO queue.Fixes 3d3edc265fc8 ("net/ena: make coherent memory allocation NUMA-aware")Signed-off-by Michal Krawczyk <[email protected]>---
v2:
* Remove gerrit change id
* Add dpdk-stable to ccdrivers/net/ena/ena_ethdev.c | 24 +++++------------------- drivers/net/ena/ena_ethdev.h | 2 ++ 2 files changed, 7 insertions(+), 19 deletions(-)drivers/net/ena/ena_ethdev.c +0
-7@@ -272,22 +272,6 @@ static const struct eth_dev_ops ena_dev_ops = { .reta_query = ena_rss_reta_query, }; -#define NUMA_NO_NODE SOCKET_ID_ANY - -static inline int ena_cpu_to_node (int cpu) -{ -struct rte_config *config =rte_eal_get_configuration (); -struct rte_fbarray *arr = &config->mem_config->memzones; -const struct rte_memzone *mz;
drivers/net/ena/ena_ethdev.c static const struct eth_dev_ops ena_dev_ops = { · line 275
Quoted 10 lines
drivers/net/ena/ena_ethdev.c +0
-9- -if (unlikely (cpu >=RTE_MAX_MEMZONE )) -return NUMA_NO_NODE ; - - mz =rte_fbarray_get (arr, cpu); - -return mz->socket_id; -} -
I suppose you can remove the rte_eal_memconfig.h header inclusion now.
static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf,
Quoted 63 lines
drivers/net/ena/ena_ethdev.c +3
-2struct ena_com_rx_ctx *ena_rx_ctx) {@@ -1126,7 +1110,7 @@ static int ena_create_io_queue(struct ena_ring *ring) } ctx.qid = ena_qid; ctx.msix_vector =-1 ;/* interrupts not used */ - ctx.numa_node =ena_cpu_to_node (ring->id); + ctx.numa_node = ring->numa_socket_id; rc =ena_com_create_io_queue (ena_dev, &ctx);if (rc) {@@ -1224,7 +1208,7 @@ static int ena_queue_start(struct ena_ring *ring)static int ena_tx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id,const struct rte_eth_txconf *tx_conf) {struct ena_ring *txq =NULL ;@@ -1262,6 +1246,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, txq->next_to_clean =0 ; txq->next_to_use =0 ; txq->ring_size = nb_desc; + txq->numa_socket_id = socket_id; txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info",
sizeof(struct ena_tx_buffer)*drivers/net/ena/ena_ethdev.c +1
-1@@ -1309,7 +1294,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, static int ena_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx, uint16_t nb_desc, - __rte_unused unsigned int socket_id, + unsigned int socket_id, __rte_unused const struct rte_eth_rxconf *rx_conf,struct
rte_mempool*mp)
{drivers/net/ena/ena_ethdev.c +1
-0@@ -1347,6 +1332,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, rxq->next_to_clean =0 ; rxq->next_to_use =0 ; rxq->ring_size = nb_desc; + rxq->numa_socket_id = socket_id; rxq->mb_pool = mp;rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info",drivers/net/ena/ena_ethdev.h +2
-1@@ -143,6 +143,8 @@ struct ena_ring {struct ena_stats_rx rx_stats;struct ena_stats_tx tx_stats; }; + +unsigned int numa_socket_id; } __rte_cache_aligned;enum ena_adapter_state { --
Quoted 3 lines
2.7.4
Signature · 1 line
David Marchand
PATCH v3 1/1 net/ena: fix assigning NUMA node to IO queue
Previous solution was using memzones in invalid way in hope to assign IO queue to the appropriate NUMA zone.
The right way is to use socket_id from the rx/tx queue setup function and then pass it to the IO queue.
---
v3:
* Remove rte_eal_memconfig.h header inclusion
v2:
* Remove gerrit change id
* Add dpdk-stable to cc
drivers/net/ena/ena_ethdev.c | 25 +++++--------------------
drivers/net/ena/ena_ethdev.h | 2 ++ 2 files changed, 7 insertions(+), 20 deletions(-)
drivers/net/ena/ena_ethdev.c +5 -20
@@ -40,7 +40,6 @@#include <rte_dev.h> #include <rte_errno.h> #include <rte_version.h> -#include <rte_eal_memconfig.h> #include <rte_net.h> #include "ena_ethdev.h"
@@ -272,22 +271,6 @@ static const struct eth_dev_ops ena_dev_ops = { .reta_query = ena_rss_reta_query, }; -#define NUMA_NO_NODE SOCKET_ID_ANY - -static inline int ena_cpu_to_node (int cpu) -{ -struct rte_config *config =rte_eal_get_configuration (); -struct rte_fbarray *arr = &config->mem_config->memzones; -const struct rte_memzone *mz; - -if (unlikely (cpu >=RTE_MAX_MEMZONE )) -return NUMA_NO_NODE ; - - mz =rte_fbarray_get (arr, cpu); - -return mz->socket_id; -} -static inline void ena_rx_mbuf_prepare (struct rte_mbuf *mbuf,struct ena_com_rx_ctx *ena_rx_ctx) {
@@ -1126,7 +1109,7 @@ static int ena_create_io_queue(struct ena_ring *ring) } ctx.qid = ena_qid; ctx.msix_vector =-1 ;/* interrupts not used */ - ctx.numa_node =ena_cpu_to_node (ring->id); + ctx.numa_node = ring->numa_socket_id; rc =ena_com_create_io_queue (ena_dev, &ctx);if (rc) {
@@ -1224,7 +1207,7 @@ static int ena_queue_start(struct ena_ring *ring)static int ena_tx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id,const struct rte_eth_txconf *tx_conf) {struct ena_ring *txq =NULL ;
@@ -1262,6 +1245,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, txq->next_to_clean =0 ; txq->next_to_use =0 ; txq->ring_size = nb_desc; + txq->numa_socket_id = socket_id; txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", sizeof(struct ena_tx_buffer) *
@@ -1309,7 +1293,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev,static int ena_rx_queue_setup (struct rte_eth_dev *dev,uint16_t queue_idx,uint16_t nb_desc, - __rte_unused unsigned int socket_id, +unsigned int socket_id, __rte_unused const struct rte_eth_rxconf *rx_conf,struct rte_mempool *mp) {
@@ -1347,6 +1331,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, rxq->next_to_clean =0 ; rxq->next_to_use =0 ; rxq->ring_size = nb_desc; + rxq->numa_socket_id = socket_id; rxq->mb_pool = mp; rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info",
drivers/net/ena/ena_ethdev.h +2 -0
@@ -143,6 +143,8 @@ struct ena_ring {struct ena_stats_rx rx_stats;struct ena_stats_tx tx_stats; }; + +unsigned int numa_socket_id; } __rte_cache_aligned;enum ena_adapter_state {
Signature · 1 line
2.7.4
Ferruh Yigit
AppliedQuoted 11 lines · Michal Krawczyk
Previous solution was using memzones in invalid way in hope to assign IO queue to the appropriate NUMA zone.
The right way is to use
socket_idfrom the rx/tx queue setup function and then pass it to the IO queue.Fixes 3d3edc265fc8 ("net/ena: make coherent memory allocation NUMA-aware")Signed-off-by Michal Krawczyk <[email protected]>Reviewed-by David Marchand <[email protected]>
Applied to dpdk-next-net/master, thanks.