Skip to contentSkip to conversation

net/mlx5: add indexed pool local cache

PATCH New Suanming Mou 2 participants 135 messages May 27, 2021 – Jul 13, 2021 Network drivers Page 1 of 2

lore
Share & tools
lore
#0 lore raw

cover net/mlx5: add indexed pool local cache

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

Suanming Mou (4):
net/mlx5: add index allocate with up limit
net/mlx5: add indexed pool local cache
net/mlx5: add index pool cache flush
net/mlx5: replace flow list with index pool

 drivers/net/mlx5/linux/mlx5_os.c |  11 +
 drivers/net/mlx5/mlx5.c          |   3 +-
 drivers/net/mlx5/mlx5.h          |   4 +-
 drivers/net/mlx5/mlx5_flow.c     | 111 ++++------
 drivers/net/mlx5/mlx5_flow.h     |   1 +
 drivers/net/mlx5/mlx5_flow_dv.c  |   5 +
 drivers/net/mlx5/mlx5_trigger.c  |   8 +-
 drivers/net/mlx5/mlx5_utils.c    | 337 ++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_utils.h    |  69 ++++++-
 9 files changed, 463 insertions(+), 86 deletions(-)
Signature · 1 line
2.25.1
#1 lore raw

PATCH 1/4 net/mlx5: add index allocate with up limit

The index pool can be used as ID allocator. In the ID allocator case, to support the maximum ID can be allocated is very useful since some registers only want limited bits of ID.

This patch add the maximum ID configurable for the index pool.

---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +12 -2
index 18fe23e4fb..bf2b2ebc72 100644
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
drivers/net/mlx5/mlx5_utils.h +1 -0
index b54517c6df..15870e14c2 100644
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
Signature · 1 line
2.25.1
#2 lore raw

PATCH 2/4 net/mlx5: add indexed pool local cache

In reply to Suanming Mou

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

This commit adds a new local ring cache mechanism for indexed pool.

---
 drivers/net/mlx5/mlx5_utils.c | 253 +++++++++++++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_utils.h |  55 +++++++-
 2 files changed, 306 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +252 -1
index bf2b2ebc72..900de2a831 100644
@@ -273,6 +273,22 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (cfg->per_core_cache) {
+		char cache_name[64] = { 0 };
+
+		for (i = 0; i < MLX5_IPOOL_MAX_CORES; i++) {
+			snprintf(cache_name, RTE_DIM(cache_name),
+				 "Ipool_cache-%p-%u", (void *)pool, i);
+			pool->l_idx_c[i] = rte_ring_create(cache_name,
+				cfg->per_core_cache, SOCKET_ID_ANY,
+				RING_F_SC_DEQ | RING_F_SP_ENQ);
+			if (!pool->l_idx_c[i]) {
+				printf("Ipool allocate ring failed:%d\n", i);
+				mlx5_free(pool);
+				return NULL;
+			}
+		}
+	}
 	return pool;
 }
 
@@ -355,6 +371,232 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static struct mlx5_indexed_trunk *
+mlx5_ipool_grow_cache(struct mlx5_indexed_pool *pool)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *pi;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx;
+	int n_grow;
+	int cidx = 0;
+	char cache_name[64] = { 0 };
+
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return NULL;
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1 || cidx > (MLX5_IPOOL_MAX_CORES - 1))
+		cidx = 0;
+	trunk_idx = __atomic_fetch_add(&pool->n_trunk_valid, 1,
+				__ATOMIC_ACQUIRE);
+	/* Have enough space in trunk array. Allocate the trunk directly. */
+	if (trunk_idx < __atomic_load_n(&pool->n_trunk, __ATOMIC_ACQUIRE))
+		goto allocate_trunk;
+	mlx5_ipool_lock(pool);
+	/* Double check if trunks array has been resized. */
+	if (trunk_idx < __atomic_load_n(&pool->n_trunk, __ATOMIC_ACQUIRE)) {
+		mlx5_ipool_unlock(pool);
+		goto allocate_trunk;
+	}
+	n_grow = trunk_idx ? pool->n_trunk :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk + n_grow);
+	/* Resize the trunk array. */
+	p = pool->cfg.malloc(MLX5_MEM_ZERO, ((trunk_idx + n_grow) *
+			     sizeof(struct mlx5_indexed_trunk *)) + sizeof(*p),
+			     RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!p) {
+		mlx5_ipool_unlock(pool);
+		return NULL;
+	}
+	p->trunks = (struct mlx5_indexed_trunk **)(p + 1);
+	if (pool->trunks_g)
+		memcpy(p->trunks, pool->trunks_g->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+	memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+	       n_grow * sizeof(void *));
+	/* Resize the global index cache ring. */
+	pi = pool->cfg.malloc(MLX5_MEM_ZERO, sizeof(*pi), 0, rte_socket_id());
+	if (!pi) {
+		mlx5_free(p);
+		mlx5_ipool_unlock(pool);
+		return NULL;
+	}
+	snprintf(cache_name, RTE_DIM(cache_name), "Idxc-%p-%u",
+		 (void *)pool, trunk_idx);
+	pi->ring = rte_ring_create(cache_name, rte_align32pow2(cur_max_idx),
+		SOCKET_ID_ANY, RING_F_SC_DEQ | RING_F_SP_ENQ);
+	if (!pi->ring) {
+		mlx5_free(p);
+		mlx5_free(pi);
+		mlx5_ipool_unlock(pool);
+		return NULL;
+	}
+	p->ref_cnt = 1;
+	pool->trunks_g = p;
+	pi->ref_cnt = 1;
+	pool->idx_g = pi;
+	/* Check if trunks array is not used any more. */
+	if (pool->trunks_c[cidx] && (!(--pool->trunks_c[cidx]->ref_cnt)))
+		mlx5_free(pool->trunks_c[cidx]);
+	/* Check if index cache is not used any more. */
+	if (pool->idx_c[cidx] &&
+	    (!(--pool->idx_c[cidx]->ref_cnt))) {
+		rte_ring_free(pool->idx_c[cidx]->ring);
+		mlx5_free(pool->idx_c[cidx]);
+	}
+	pool->trunks_c[cidx] = p;
+	pool->idx_c[cidx] = pi;
+	__atomic_fetch_add(&pool->n_trunk, n_grow, __ATOMIC_ACQUIRE);
+	mlx5_ipool_unlock(pool);
+allocate_trunk:
+	/* Initialize the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!trunk)
+		return NULL;
+	pool->trunks_g->trunks[trunk_idx] = trunk;
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+#ifdef POOL_DEBUG
+	pool->trunk_new++;
+#endif
+	return trunk;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx = 0;
+
+	if (!idx)
+		return NULL;
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1)
+		cidx = 0;
+	if (pool->trunks_c[cidx] != pool->trunks_g) {
+		mlx5_ipool_lock(pool);
+		/* Rlease the old ring if we are the last thread cache it. */
+		if (pool->trunks_c[cidx] &&
+		    !(--pool->trunks_c[cidx]->ref_cnt))
+			mlx5_free(pool->trunks_c[cidx]);
+		pool->trunks_c[cidx] = pool->trunks_g;
+		pool->trunks_c[cidx]->ref_cnt++;
+		mlx5_ipool_unlock(pool);
+	}
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = pool->trunks_c[cidx]->trunks[trunk_idx];
+	if (!trunk)
+		return NULL;
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	uint32_t i, ts_idx, fetch_size;
+	int cidx = 0;
+	union mlx5_indexed_qd qd;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1)
+		cidx = 0;
+	/* Try local cache firstly. */
+	if (!rte_ring_dequeue(pool->l_idx_c[cidx], &qd.ptr)) {
+		*idx = qd.idx;
+		return mlx5_ipool_get_cache(pool, qd.idx);
+	}
+	if (pool->idx_g) {
+		/*
+		 * Try fetch from the global cache. Check if global cache ring
+		 * updated first.
+		 */
+		if (pool->idx_c[cidx] != pool->idx_g) {
+			mlx5_ipool_lock(pool);
+			/* Rlease the old ring as last user. */
+			if (pool->idx_c[cidx] &&
+			    !(--pool->idx_c[cidx]->ref_cnt)) {
+				rte_ring_free(pool->idx_c[cidx]->ring);
+				pool->cfg.free(pool->idx_c[cidx]);
+			}
+			pool->idx_c[cidx] = pool->idx_g;
+			pool->idx_c[cidx]->ref_cnt++;
+			mlx5_ipool_unlock(pool);
+		}
+		fetch_size = pool->cfg.trunk_size;
+		while (!rte_ring_dequeue(pool->idx_g->ring, &qd.ptr)) {
+			if (unlikely(!(--fetch_size))) {
+				*idx = qd.idx;
+				return mlx5_ipool_get_cache(pool, qd.idx);
+			}
+			rte_ring_enqueue(pool->l_idx_c[cidx], qd.ptr);
+		}
+	}
+	/* Not enough idx in global cache. Keep fetching from new trunk. */
+	trunk = mlx5_ipool_grow_cache(pool);
+	if (!trunk)
+		return NULL;
+	/* Get trunk start index. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk->idx);
+	/* Enqueue trunk_size - 1 to local cache ring. */
+	for (i = 0; i < trunk->free - 1; i++) {
+		qd.idx = ts_idx + i + 1;
+		rte_ring_enqueue(pool->l_idx_c[cidx], qd.ptr);
+	}
+	/* Return trunk's final entry. */
+	*idx = ts_idx + i + 1;
+	return &trunk->data[i * pool->cfg.size];
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	uint32_t i, reclaim_num = 0;
+	union mlx5_indexed_qd qd;
+
+	if (!idx)
+		return;
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (cidx == -1)
+		cidx = 0;
+	qd.idx = idx;
+	/* Try to enqueue to local index cache. */
+	if (!rte_ring_enqueue(pool->l_idx_c[cidx], qd.ptr))
+		return;
+	/* Update the global index cache ring if needed. */
+	if (pool->idx_c[cidx] != pool->idx_g) {
+		mlx5_ipool_lock(pool);
+		/* Rlease the old ring if we are the last thread cache it. */
+		if (!(--pool->idx_c[cidx]->ref_cnt))
+			rte_ring_free(pool->idx_c[cidx]->ring);
+		pool->idx_c[cidx] = pool->idx_g;
+		pool->idx_c[cidx]->ref_cnt++;
+		mlx5_ipool_unlock(pool);
+	}
+	reclaim_num = pool->cfg.per_core_cache >> 4;
+	/* Local index cache full, try with global index cache. */
+	rte_ring_enqueue(pool->idx_c[cidx]->ring, qd.ptr);
+	/* Dequeue the index from local cache to global. */
+	for (i = 0; i < reclaim_num; i++) {
+		/* No need to check the return value. */
+		rte_ring_dequeue(pool->l_idx_c[cidx], &qd.ptr);
+		rte_ring_enqueue(pool->idx_c[cidx]->ring, qd.ptr);
+	}
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +605,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +676,8 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_free_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +743,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -524,7 +772,10 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
+	if (pool->cfg.per_core_cache)
+		trunks = pool->trunks_g->trunks;
+	else
+		trunks = pool->trunks;
 	for (i = 0; i < pool->n_trunk; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
drivers/net/mlx5/mlx5_utils.h +54 -1

95 lines not on this page · raw message

Signature · 1 line
2.25.1
#3 lore raw

PATCH 3/4 net/mlx5: add index pool cache flush

In reply to Suanming Mou

This commit adds the per-core cache indexed pool cache flush function.

---
 drivers/net/mlx5/mlx5_utils.c | 70 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h | 13 +++++++
 2 files changed, 83 insertions(+)
drivers/net/mlx5/mlx5_utils.c +70 -0
index 900de2a831..ee6c2e293e 100644
@@ -452,6 +452,21 @@ mlx5_ipool_grow_cache(struct mlx5_indexed_pool *pool)
 	pool->idx_c[cidx] = pi;
 	__atomic_fetch_add(&pool->n_trunk, n_grow, __ATOMIC_ACQUIRE);
 	mlx5_ipool_unlock(pool);
+	/* Pre-allocate the bitmap. */
+	if (pool->ibmp)
+		pool->cfg.free(pool->ibmp);
+	data_size = sizeof(*pool->ibmp);
+	data_size += rte_bitmap_get_memory_footprint(cur_max_idx);
+	/* rte_bitmap requires memory cacheline aligned. */
+	pool->ibmp = pool->cfg.malloc(MLX5_MEM_ZERO, data_size,
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->ibmp)
+		return NULL;
+	pool->ibmp->num = cur_max_idx;
+	pool->ibmp->mem_size = data_size - sizeof(*pool->ibmp);
+	pool->ibmp->mem = (void *)(pool->ibmp + 1);
+	pool->ibmp->bmp = rte_bitmap_init(pool->ibmp->num,
+			pool->ibmp->mem, pool->ibmp->mem_size);
 allocate_trunk:
 	/* Initialize the new trunk. */
 	trunk_size = sizeof(*trunk);
@@ -787,6 +802,61 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i;
+	struct rte_ring *ring_c;
+	char cache_name[64];
+	union mlx5_indexed_qd qd;
+	uint32_t bmp_num, mem_size;
+	uint32_t num = 0;
+
+	/* Create a new ring to save all cached index. */
+	snprintf(cache_name, RTE_DIM(cache_name), "Ip_%p",
+		 (void *)pool->idx_g->ring);
+	ring_c = rte_ring_create(cache_name, pool->ibmp->num,
+			SOCKET_ID_ANY, RING_F_SC_DEQ | RING_F_SP_ENQ);
+	if (!ring_c)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->ibmp->bmp = rte_bitmap_init_with_all_set(bmp_num,
+			pool->ibmp->mem, mem_size);
+	/* Flush core cache. */
+	for (i = 0; i < MLX5_IPOOL_MAX_CORES; i++) {
+		while (!rte_ring_dequeue(pool->l_idx_c[i], &qd.ptr)) {
+			rte_bitmap_clear(pool->ibmp->bmp, (qd.idx - 1));
+			rte_ring_enqueue(ring_c, qd.ptr);
+			num++;
+		}
+	}
+	/* Flush global cache. */
+	while (!rte_ring_dequeue(pool->idx_g->ring, &qd.ptr)) {
+		rte_bitmap_clear(pool->ibmp->bmp, (qd.idx - 1));
+		rte_ring_enqueue(ring_c, qd.ptr);
+		num++;
+	}
+	rte_ring_free(pool->idx_g->ring);
+	pool->idx_g->ring = ring_c;
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	if (!pool->ibmp || !rte_bitmap_scan(pool->ibmp->bmp, &iidx, &slab))
+		return NULL;
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(pool->ibmp->bmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get(pool, iidx);
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
drivers/net/mlx5/mlx5_utils.h +13 -0
index 4fe82d4a5f..03d5164485 100644
@@ -247,6 +247,13 @@ struct mlx5_indexed_cache {
 	uint32_t res;
 };
 
+struct mlx5_indexed_bmp {
+	struct rte_bitmap *bmp;
+	void *mem;
+	uint32_t mem_size;
+	uint32_t num;
+} __rte_cache_aligned;
+
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t lock; /* Pool lock for multiple thread usage. */
@@ -261,6 +268,7 @@ struct mlx5_indexed_pool {
 	struct mlx5_indexed_cache *idx_g;
 	struct mlx5_indexed_cache *idx_c[MLX5_IPOOL_MAX_CORES];
 	struct rte_ring *l_idx_c[MLX5_IPOOL_MAX_CORES];
+	struct mlx5_indexed_bmp *ibmp;
 	uint32_t free_list; /* Index to first free trunk. */
 #ifdef POOL_DEBUG
 	uint32_t n_entry;
@@ -861,4 +869,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry) \
+	for ((idx) = 1, mlx5_ipool_flush_cache((ipool)), \
+	     entry = mlx5_ipool_get_next((ipool), &idx); \
+	     (entry); idx++, entry = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
Signature · 1 line
2.25.1
#4 lore raw

PATCH 4/4 net/mlx5: replace flow list with index pool

In reply to Suanming Mou

The flow list is used to save the create flows and to be used only when port closes all the flows need to be flushed.

This commit creates a new function to get the available entries from the index pool. Then flows can be flushed from index pool.

---
 drivers/net/mlx5/linux/mlx5_os.c |  11 +++
 drivers/net/mlx5/mlx5.c          |   3 +-
 drivers/net/mlx5/mlx5.h          |   4 +-
 drivers/net/mlx5/mlx5_flow.c     | 111 +++++++++++--------------------
 drivers/net/mlx5/mlx5_flow.h     |   1 +
 drivers/net/mlx5/mlx5_flow_dv.c  |   5 ++
 drivers/net/mlx5/mlx5_trigger.c  |   8 +--
 7 files changed, 61 insertions(+), 82 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +11 -0
index 534a56a555..c56be18d24 100644
@@ -1608,6 +1608,17 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	struct mlx5_indexed_pool_config icfg = {
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 4096,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 23,
+		.type = "rte_flow_ipool",
+	};
+	priv->flows = mlx5_ipool_create(&icfg);
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
drivers/net/mlx5/mlx5.c +2 -1
index cf1815cb74..b0f9f1a45e 100644
@@ -323,6 +323,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_shift = 2,
 		.need_lock = 1,
 		.release_mem_en = 1,
+		.per_core_cache = 1 << 23,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -1528,7 +1529,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, false, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
drivers/net/mlx5/mlx5.h +2 -2
index 32b2817bf2..f6b72cd352 100644
@@ -1344,7 +1344,7 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows; /* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1596,7 +1596,7 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, bool control, bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
drivers/net/mlx5/mlx5_flow.c +37 -74

277 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +1 -0
index 2f2aa962f9..cb0b97a2d4 100644
@@ -1000,6 +1000,7 @@ struct rte_flow {
 	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t control:1;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index c50649a107..d039794c9a 100644
@@ -13830,6 +13830,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
drivers/net/mlx5/mlx5_trigger.c +3 -5
index ae7fcca229..48cf780786 100644
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, false, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, true, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, true, false);
 }
 
 /**
Signature · 1 line
2.25.1
#5 lore raw

v2 cover net/mlx5: insertion rate optimization

In reply to Suanming Mou

This patch series optimize the flow insertion rate with adding local cache to index pool and list.

For object which wants efficient index allocate and free, local cache will be very helpful.

For index pool, two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

For list, sub local core list is introduced. The allocated objects will be added and released only from local list without any locks. Only the objects need to be shared will be synced from global list.

---

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
net/mlx5: optimize modify header action memory net/mlx5: remove cache term from the list utility net/mlx5: add per lcore cache to the list utility net/mlx5: minimize list critical sections net/mlx5: manage list cache entries release net/mlx5: relax the list utility atomic operations net/mlx5: allocate list memory by the create API common/mlx5: add per-lcore cache to hash list utility net/mlx5: move modify header allocator to ipool

Suanming Mou (13):
net/mlx5: allow limiting the index pool maximum index net/mlx5: add indexed pool local cache net/mlx5: add index pool foreach define net/mlx5: replace flow list with index pool net/mlx5: adjust the hash bucket size common/mlx5: allocate cache list memory individually net/mlx5: enable index pool per-core cache net/mlx5: optimize hash list table allocate on demand common/mlx5: optimize cache list object memory net/mlx5: change memory release configuration net/mlx5: support index pool none local core operations net/mlx5: support list none local core operations net/mlx5: optimize Rx queue match

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    6 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  283 ++++--
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  305 +++---
 drivers/net/mlx5/mlx5_flow.h            |  210 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1203 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  617 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 20 files changed, 2518 insertions(+), 1411 deletions(-)
Signature · 1 line
2.25.1
#6 lore raw

PATCH v2 1/22 net/mlx5: allow limiting the index pool maximum index

Some ipool instances in the driver are used as ID\index allocator and added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value. The ipool will ensure that no index bigger than the maximum value is provided.

Use this configuration in ID allocator cases instead of the current logics. This patch add the maximum ID configurable for the index pool.

---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +12 -2
index 18fe23e4fb..bf2b2ebc72 100644
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
drivers/net/mlx5/mlx5_utils.h +1 -0
index b54517c6df..15870e14c2 100644
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
Signature · 1 line
2.25.1
#7 lore raw

PATCH v2 2/22 net/mlx5: add indexed pool local cache

In reply to Suanming Mou

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is introduced to allocate and free the index more efficient. One as local and the other as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_utils.c +313 -10
index bf2b2ebc72..215024632d 100644
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
drivers/net/mlx5/mlx5_utils.h +59 -5

85 lines not on this page · raw message

Signature · 1 line
2.25.1
#8 lore raw

PATCH v2 3/22 net/mlx5: add index pool foreach define

In reply to Suanming Mou

In some cases, application may want to know all the allocated index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach operation.

---
 drivers/net/mlx5/mlx5_utils.c | 86 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 ++++
 2 files changed, 94 insertions(+)
drivers/net/mlx5/mlx5_utils.c +86 -0
index 215024632d..0ed279e162 100644
@@ -839,6 +839,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
drivers/net/mlx5/mlx5_utils.h +8 -0
index 0469062695..737dd7052d 100644
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
Signature · 1 line
2.25.1
#9 lore raw

PATCH v2 4/22 net/mlx5: replace flow list with index pool

In reply to Suanming Mou

The flow list is used to save the create flows and to be used only when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to flush all the allocated flows.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 +++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 149 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 126 insertions(+), 110 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +47 -1
index 92b3009786..31cc8d9eb8 100644
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -823,6 +861,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1566,7 +1605,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1600,6 +1638,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
drivers/net/mlx5/mlx5.c +6 -3
index cf1815cb74..fcfc3dcdca 100644
@@ -322,7 +322,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -792,8 +793,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1528,7 +1531,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
drivers/net/mlx5/mlx5.h +12 -2
index 32b2817bf2..5fa5d3cb99 100644
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1344,7 +1352,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1596,7 +1605,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
drivers/net/mlx5/mlx5_flow.c +52 -97

343 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +1 -1
index 2f2aa962f9..d9b6acaafd 100644
@@ -997,9 +997,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index c5d4b01e57..67f7243503 100644
@@ -13844,6 +13844,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
drivers/net/mlx5/mlx5_trigger.c +3 -5
index ae7fcca229..7cb8920d6b 100644
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
drivers/net/mlx5/windows/mlx5_os.c +0 -1
index 3fe3f55f49..7d15c998bb 100644
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	TAILQ_INIT(&priv->flow_meter_profiles);
Signature · 1 line
2.25.1
#10 lore raw

PATCH v2 5/22 net/mlx5: optimize modify header action memory

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Define the types of the modify header action fields to be with the minimum size needed for the optional values range.

---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)
drivers/common/mlx5/linux/mlx5_glue.h +1 -0
index 840d8cf57f..a186ee577f 100644
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
drivers/net/mlx5/linux/mlx5_flow_os.h +2 -1
index cee685015b..1926d26410 100644
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
drivers/net/mlx5/mlx5_flow.h +3 -3
index d9b6acaafd..81c95e0beb 100644
@@ -523,11 +523,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
drivers/net/mlx5/mlx5_flow_dv.c +6 -7
index 67f7243503..784ec11dea 100644
@@ -5000,21 +5000,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5582,10 +5582,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
Signature · 1 line
2.25.1
#11 lore raw

PATCH v2 6/22 net/mlx5: remove cache term from the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The internal mlx5 list tool is used mainly when the list objects need to be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread cache management.

To prevent confusing, remove the current "cache" term from the API's names.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 558 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 454 insertions(+), 460 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -16
index 31cc8d9eb8..9aa57e38b7 100644
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -500,8 +500,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1702,7 +1702,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1761,7 +1761,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -1
index fcfc3dcdca..9aade013c5 100644
@@ -1611,7 +1611,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +7 -8
index 5fa5d3cb99..803387b98e 100644
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1221,7 +1221,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1369,7 +1369,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
drivers/net/mlx5/mlx5_flow.h +45 -43

130 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +276 -282

1,122 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +6 -6
index 1b264e5994..3dcc71d51d 100644
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +14 -14
index bb9a908087..8395332507 100644
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +39 -39

188 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +47 -47

215 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +3 -4
index 7d15c998bb..b10c47fee3 100644
@@ -608,10 +608,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#12 lore raw

PATCH v2 7/22 net/mlx5: add per lcore cache to the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When mlx5 list object is accessed by multiple cores, the list lock counter is all the time written by all the cores what increases cache misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup operation, all the other threads coming to do an operation in the list are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when the list objects are mostly reused.

Synchronization with atomic operations should be done in order to allow threads to unregister an entry from other thread cache.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +34 -24
index 9aa57e38b7..8a043526da 100644
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1702,10 +1710,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.h +1 -0
index 803387b98e..b01ce5cbfd 100644
@@ -1365,6 +1365,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
drivers/net/mlx5/mlx5_flow.h +20 -1
index 4dec703366..ce363355c1 100644
@@ -1634,7 +1634,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1642,6 +1646,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1650,6 +1659,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1658,6 +1672,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow_dv.c +179 -2

219 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -0
index 3dcc71d51d..5450ddd388 100644
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +52 -19

117 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +135 -79

288 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +22 -8
index 593793345d..24ae2b2ccb 100644
@@ -309,9 +309,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -339,6 +344,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -375,15 +387,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -393,8 +407,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -407,10 +419,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +3 -2
index b10c47fee3..f6cf1928b2 100644
@@ -608,9 +608,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#13 lore raw

PATCH v2 8/22 net/mlx5: minimize list critical sections

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical sections.

These operations are heavy and make the critical sections heavy because they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation counter in order to detect parallel allocations.

---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)
drivers/net/mlx5/mlx5_utils.c +45 -41
index 51cca68ea9..772b352af5 100644
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
drivers/net/mlx5/mlx5_utils.h +3 -2
index 24ae2b2ccb..684d1e8a2a 100644
@@ -387,8 +387,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
Signature · 1 line
2.25.1
#14 lore raw

PATCH v2 9/22 net/mlx5: manage list cache entries release

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When a cache entry is allocated by lcore A and is released by lcore B, the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be increased atomically when the non-original lcore decreases the reference counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in its cache, it will check the counter in order to free invalid entries in its cache.

---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)
drivers/net/mlx5/mlx5_utils.c +52 -27
index 772b352af5..7cdf44dcf7 100644
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/net/mlx5/mlx5_utils.h +2 -0
index 684d1e8a2a..ffa9cd5142 100644
@@ -310,11 +310,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
Signature · 1 line
2.25.1
#15 lore raw

PATCH v2 10/22 net/mlx5: relax the list utility atomic operations

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The atomic operation in the list utility no need a barriers because the critical part are managed by RW lock.

Relax them.

---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
drivers/net/mlx5/mlx5_utils.c +6 -6
index 7cdf44dcf7..29248c80ed 100644
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
Signature · 1 line
2.25.1
#16 lore raw

PATCH v2 11/22 net/mlx5: allocate list memory by the create API

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence with the hlist utility.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  56 ++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 137 insertions(+), 81 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +73 -32
index 8a043526da..87b63d852b 100644
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -447,6 +455,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -508,9 +532,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1710,11 +1748,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1771,7 +1811,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +2 -1
index 9aade013c5..775ea15adb 100644
@@ -1611,7 +1611,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +5 -5
index b01ce5cbfd..9319604d91 100644
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +1 -1
index ce363355c1..ce4d205e86 100644
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
drivers/net/mlx5/mlx5_flow_dv.c +33 -23

164 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxq.c +3 -3
index f8769da8dc..aa9e973d10 100644
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +12 -7
index 29248c80ed..a4526444f9 100644
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
drivers/net/mlx5/mlx5_utils.h +7 -8
index ffa9cd5142..0bf2f5f5ca 100644
@@ -419,15 +419,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index f6cf1928b2..97a8f04e39 100644
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#17 lore raw

PATCH v2 12/22 common/mlx5: add per-lcore cache to hash list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and hlist.

---
 doc/guides/nics/mlx5.rst                |   5 +
 doc/guides/rel_notes/release_21_08.rst  |   6 +
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 466 +++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h | 261 +++++++++----
 drivers/net/mlx5/linux/mlx5_os.c        |  46 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++---
 drivers/net/mlx5/mlx5_flow.h            | 185 +++++-----
 drivers/net/mlx5/mlx5_flow_dv.c         | 407 ++++++++++++---------
 drivers/net/mlx5/mlx5_rx.h              |  13 +-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +--
 drivers/net/mlx5/mlx5_utils.c           | 251 -------------
 drivers/net/mlx5/mlx5_utils.h           | 197 ----------
 drivers/net/mlx5/windows/mlx5_os.c      |   2 +-
 16 files changed, 1016 insertions(+), 1044 deletions(-)
doc/guides/nics/mlx5.rst +5 -0
index eb44a070b1..9bd3846e0d 100644
@@ -445,6 +445,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
doc/guides/rel_notes/release_21_08.rst +6 -0
index a6ecfdf3ce..f6cd1d137d 100644
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Updated Mellanox mlx5 driver.**
+
+  Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+  * Optimize multiple-thread flow insertion rate.
+
 
 Removed Items
 -------------
drivers/common/mlx5/mlx5_common.h +2 -0
index 1fbefe0fa6..1809ff1e95 100644
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
drivers/common/mlx5/mlx5_common_utils.c +328 -138

530 lines not on this page · raw message

drivers/common/mlx5/mlx5_common_utils.h +193 -68

343 lines not on this page · raw message

drivers/net/mlx5/linux/mlx5_os.c +24 -22

125 lines not on this page · raw message

drivers/net/mlx5/mlx5.c +6 -4
index 775ea15adb..0e80408511 100644
@@ -1358,20 +1358,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
drivers/net/mlx5/mlx5.h +1 -0
index 9319604d91..896c0c2da1 100644
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
drivers/net/mlx5/mlx5_flow.c +113 -42

298 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +90 -95

242 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +232 -175

764 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -8
index 5450ddd388..3f2b99fb65 100644
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
drivers/net/mlx5/mlx5_rxq.c +10 -43

91 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +0 -251

257 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +0 -197

203 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +1 -1
index 97a8f04e39..af2f684648 100644
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#18 lore raw

PATCH v2 13/22 net/mlx5: move modify header allocator to ipool

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Modify header actions are allocated by mlx5_malloc which has a big overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header handler.

---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.c +4 -0
index 0e80408511..713accf675 100644
@@ -801,6 +801,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -814,6 +815,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
drivers/net/mlx5/mlx5.h +14 -0
index 896c0c2da1..5774f63244 100644
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1123,6 +1136,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
drivers/net/mlx5/mlx5_flow.h +1 -13
index ab4e8c5c4f..4552aaa803 100644
@@ -504,23 +504,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
drivers/net/mlx5/mlx5_flow_dv.c +67 -12

138 lines not on this page · raw message

Signature · 1 line
2.25.1
#19 lore raw

PATCH v2 14/22 net/mlx5: adjust the hash bucket size

In reply to Suanming Mou

With the new per core optimization to the list, the hash bucket size can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index cf573a9a4d..a82dc4db00 100644
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
drivers/net/mlx5/mlx5.c +1 -1
index 713accf675..8fb7f4442d 100644
@@ -373,7 +373,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
drivers/net/mlx5/mlx5_defs.h +3 -3
index 906aa43c5a..ca67ce8213 100644
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
drivers/net/mlx5/mlx5_flow.c +2 -3
index e6b71a87a0..cb8161f668 100644
@@ -8625,7 +8625,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8734,8 +8734,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
Signature · 1 line
2.25.1
#20 lore raw

PATCH v2 15/22 common/mlx5: allocate cache list memory individually

In reply to Suanming Mou

Currently, the list's local cache instance memory is allocated with the list. As the local cache instance array size is RTE_MAX_LCORE, most of the cases the system will only have very limited cores. allocate the instance memory individually per core will be more economic to the memory.

This commit changes the instance array to pointer array, allocate the local cache memory only when the core is to be used.

---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +44 -18
index 4e385c616a..f75b1cb0da 100644
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
drivers/common/mlx5/mlx5_common_utils.h +1 -1
index 79d0bc1a50..42ca3cbc1e 100644
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
Signature · 1 line
2.25.1
#21 lore raw

PATCH v2 16/22 net/mlx5: enable index pool per-core cache

In reply to Suanming Mou

This commit enables the tag and header modify action index pool per-core cache in non-reclaim memory mode.

---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5.c +3 -1
index 8fb7f4442d..bf1463c289 100644
@@ -214,7 +214,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1128,6 +1129,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
drivers/net/mlx5/mlx5.h +1 -0
index 5774f63244..516f3ffae5 100644
@@ -1093,6 +1093,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index 08704d892a..f79c60e489 100644
@@ -5321,7 +5321,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
Signature · 1 line
2.25.1
#22 lore raw

PATCH v2 17/22 net/mlx5: optimize hash list table allocate on demand

In reply to Suanming Mou

Currently, all the hash list tables are allocated during start up. Since different applications may only use dedicated limited actions, optimized the hash list table allocate on demand will save initial memory.

This commit optimizes hash list table allocate on demand.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -43
index a82dc4db00..75324e35d8 100644
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
drivers/net/mlx5/mlx5_defs.h +6 -0
index ca67ce8213..fe86bb40d3 100644
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
drivers/net/mlx5/mlx5_flow_dv.c +75 -4

119 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +0 -2
index af2f684648..0e6f7003b0 100644
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
Signature · 1 line
2.25.1
#23 lore raw

PATCH v2 18/22 common/mlx5: optimize cache list object memory

In reply to Suanming Mou

Currently, hash list uses the cache list as bucket list. The list in the buckets have the same name, ctx and callbacks. This wastes the memory.

This commit abstracts all the name, ctx and callback members in the list to a constant struct and others to the inconstant struct, uses the wrapper functions to satisfy both hash list and cache list can set the list constant and inconstant struct individually.

---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +171 -124
index f75b1cb0da..858c8d8164 100644
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +30 -15

69 lines not on this page · raw message

Signature · 1 line
2.25.1
#24 lore raw

PATCH v2 19/22 net/mlx5: change memory release configuration

In reply to Suanming Mou

This commit changes the index pool memory release configuration to 0 when memory reclaim mode is not required.

---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/mlx5/mlx5.c +2 -0
index bf1463c289..6b7225e55d 100644
@@ -797,6 +797,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
Signature · 1 line
2.25.1
#25 lore raw

PATCH v2 20/22 net/mlx5: support index pool none local core operations

In reply to Suanming Mou

This commit supports the index pool none local core operations with an extra cache.

---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)
drivers/net/mlx5/mlx5_utils.c +54 -21
index 94abe79860..c34d6d62a8 100644
@@ -114,6 +114,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->nlcore_lock);
 	return pool;
 }
 
@@ -354,20 +355,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -378,15 +373,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -399,29 +406,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -464,6 +482,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -643,7 +676,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -712,7 +745,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
drivers/net/mlx5/mlx5_utils.h +2 -1
index 7d9b64c877..060c52f022 100644
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t nlcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
Signature · 1 line
2.25.1
#26 lore raw

PATCH v2 21/22 net/mlx5: support list none local core operations

In reply to Suanming Mou

This commit supports the list none local core operations with an extra sub-list.

---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +63 -29
index 858c8d8164..d58d0d08ab 100644
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.nlcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.nlcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
+	return ret;
 }
 
 void
drivers/common/mlx5/mlx5_common_utils.h +8 -1
index 6acd0d3754..66ce27464c 100644
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t nlcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
Signature · 1 line
2.25.1
#27 lore raw

PATCH v2 22/22 net/mlx5: optimize Rx queue match

In reply to Suanming Mou

As hrxq struct has the indirect table pointer, while matching the hrxq, better to use the hrxq indirect table instead of searching from the list.

This commit optimizes the hrxq indirect table matching.

---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5_rxq.c +7 -11
index 7893b3edd4..23685d7654 100644
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
Signature · 1 line
2.25.1
#28 lore raw

v3 cover net/mlx5: insertion rate optimization

In reply to Suanming Mou

This patch series optimize the flow insertion rate with adding local cache to index pool and list.

For object which wants efficient index allocate and free, local cache will be very helpful.

For index pool, two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

For list, sub local core list is introduced. The allocated objects will be added and released only from local list without any locks. Only the objects need to be shared will be synced from global list.

---

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
net/mlx5: optimize modify header action memory net/mlx5: remove cache term from the list utility net/mlx5: add per lcore cache to the list utility net/mlx5: minimize list critical sections net/mlx5: manage list cache entries release net/mlx5: relax the list utility atomic operations net/mlx5: allocate list memory by the create API common/mlx5: add per-lcore cache to hash list utility net/mlx5: move modify header allocator to ipool

Suanming Mou (13):
net/mlx5: allow limiting the index pool maximum index net/mlx5: add indexed pool local cache net/mlx5: add index pool foreach define net/mlx5: replace flow list with index pool net/mlx5: adjust the hash bucket size common/mlx5: allocate cache list memory individually net/mlx5: enable index pool per-core cache net/mlx5: optimize hash list table allocate on demand common/mlx5: optimize cache list object memory net/mlx5: change memory release configuration net/mlx5: support index pool none local core operations net/mlx5: support list none local core operations net/mlx5: optimize Rx queue match

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    6 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  289 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  305 +++---
 drivers/net/mlx5/mlx5_flow.h            |  210 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1203 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  617 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2531 insertions(+), 1411 deletions(-)
Signature · 1 line
2.25.1
#29 lore raw

PATCH v3 1/22 net/mlx5: allow limiting the index pool maximum index

Some ipool instances in the driver are used as ID\index allocator and added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value. The ipool will ensure that no index bigger than the maximum value is provided.

Use this configuration in ID allocator cases instead of the current logics. This patch add the maximum ID configurable for the index pool.

---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +12 -2
index 18fe23e4fb..bf2b2ebc72 100644
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
drivers/net/mlx5/mlx5_utils.h +1 -0
index b54517c6df..15870e14c2 100644
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
Signature · 1 line
2.25.1
#30 lore raw

PATCH v3 2/22 net/mlx5: add indexed pool local cache

In reply to Suanming Mou

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is introduced to allocate and free the index more efficient. One as local and the other as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_utils.c +313 -10
index bf2b2ebc72..215024632d 100644
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
drivers/net/mlx5/mlx5_utils.h +59 -5

85 lines not on this page · raw message

Signature · 1 line
2.25.1
#31 lore raw

PATCH v3 3/22 net/mlx5: add index pool foreach define

In reply to Suanming Mou

In some cases, application may want to know all the allocated index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach operation.

---
 drivers/net/mlx5/mlx5_utils.c | 86 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 ++++
 2 files changed, 94 insertions(+)
drivers/net/mlx5/mlx5_utils.c +86 -0
index 215024632d..0ed279e162 100644
@@ -839,6 +839,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
drivers/net/mlx5/mlx5_utils.h +8 -0
index 0469062695..737dd7052d 100644
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
Signature · 1 line
2.25.1
#32 lore raw

PATCH v3 4/22 net/mlx5: replace flow list with index pool

In reply to Suanming Mou

The flow list is used to save the create flows and to be used only when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to flush all the allocated flows.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 +++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 149 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 126 insertions(+), 110 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +47 -1
index 92b3009786..31cc8d9eb8 100644
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -823,6 +861,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1566,7 +1605,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1600,6 +1638,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
drivers/net/mlx5/mlx5.c +6 -3
index cf1815cb74..fcfc3dcdca 100644
@@ -322,7 +322,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -792,8 +793,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1528,7 +1531,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
drivers/net/mlx5/mlx5.h +12 -2
index 1b2dc8f815..380d35d420 100644
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1344,7 +1352,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1596,7 +1605,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
drivers/net/mlx5/mlx5_flow.c +52 -97

343 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +1 -1
index 2f2aa962f9..d9b6acaafd 100644
@@ -997,9 +997,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index a04a3c2bb8..e898c571da 100644
@@ -13844,6 +13844,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
drivers/net/mlx5/mlx5_trigger.c +3 -5
index ae7fcca229..7cb8920d6b 100644
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
drivers/net/mlx5/windows/mlx5_os.c +0 -1
index 3fe3f55f49..7d15c998bb 100644
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	TAILQ_INIT(&priv->flow_meter_profiles);
Signature · 1 line
2.25.1
#33 lore raw

PATCH v3 5/22 net/mlx5: optimize modify header action memory

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Define the types of the modify header action fields to be with the minimum size needed for the optional values range.

---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)
drivers/common/mlx5/linux/mlx5_glue.h +1 -0
index 840d8cf57f..a186ee577f 100644
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
drivers/net/mlx5/linux/mlx5_flow_os.h +2 -1
index cee685015b..1926d26410 100644
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
drivers/net/mlx5/mlx5_flow.h +3 -3
index d9b6acaafd..81c95e0beb 100644
@@ -523,11 +523,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
drivers/net/mlx5/mlx5_flow_dv.c +6 -7
index e898c571da..a7c1cf05da 100644
@@ -5000,21 +5000,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5582,10 +5582,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
Signature · 1 line
2.25.1
#34 lore raw

PATCH v3 6/22 net/mlx5: remove cache term from the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The internal mlx5 list tool is used mainly when the list objects need to be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread cache management.

To prevent confusing, remove the current "cache" term from the API's names.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 558 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 454 insertions(+), 460 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -16
index 31cc8d9eb8..9aa57e38b7 100644
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -500,8 +500,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1702,7 +1702,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1761,7 +1761,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -1
index fcfc3dcdca..9aade013c5 100644
@@ -1611,7 +1611,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +7 -8
index 380d35d420..58646da331 100644
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1221,7 +1221,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1369,7 +1369,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
drivers/net/mlx5/mlx5_flow.h +45 -43

130 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +276 -282

1,122 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +6 -6
index 1b264e5994..3dcc71d51d 100644
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +14 -14
index bb9a908087..8395332507 100644
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +39 -39

188 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +47 -47

215 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +3 -4
index 7d15c998bb..b10c47fee3 100644
@@ -608,10 +608,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#35 lore raw

PATCH v3 7/22 net/mlx5: add per lcore cache to the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When mlx5 list object is accessed by multiple cores, the list lock counter is all the time written by all the cores what increases cache misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup operation, all the other threads coming to do an operation in the list are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when the list objects are mostly reused.

Synchronization with atomic operations should be done in order to allow threads to unregister an entry from other thread cache.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +34 -24
index 9aa57e38b7..8a043526da 100644
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1702,10 +1710,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.h +1 -0
index 58646da331..546bee761e 100644
@@ -1365,6 +1365,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
drivers/net/mlx5/mlx5_flow.h +20 -1
index 4dec703366..ce363355c1 100644
@@ -1634,7 +1634,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1642,6 +1646,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1650,6 +1659,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1658,6 +1672,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow_dv.c +179 -2

219 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -0
index 3dcc71d51d..5450ddd388 100644
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +52 -19

117 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +135 -79

288 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +22 -8
index 593793345d..24ae2b2ccb 100644
@@ -309,9 +309,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -339,6 +344,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -375,15 +387,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -393,8 +407,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -407,10 +419,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +3 -2
index b10c47fee3..f6cf1928b2 100644
@@ -608,9 +608,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#36 lore raw

PATCH v3 8/22 net/mlx5: minimize list critical sections

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical sections.

These operations are heavy and make the critical sections heavy because they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation counter in order to detect parallel allocations.

---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)
drivers/net/mlx5/mlx5_utils.c +45 -41
index 51cca68ea9..772b352af5 100644
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
drivers/net/mlx5/mlx5_utils.h +3 -2
index 24ae2b2ccb..684d1e8a2a 100644
@@ -387,8 +387,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
Signature · 1 line
2.25.1
#37 lore raw

PATCH v3 9/22 net/mlx5: manage list cache entries release

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When a cache entry is allocated by lcore A and is released by lcore B, the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be increased atomically when the non-original lcore decreases the reference counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in its cache, it will check the counter in order to free invalid entries in its cache.

---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)
drivers/net/mlx5/mlx5_utils.c +52 -27
index 772b352af5..7cdf44dcf7 100644
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/net/mlx5/mlx5_utils.h +2 -0
index 684d1e8a2a..ffa9cd5142 100644
@@ -310,11 +310,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
Signature · 1 line
2.25.1
#38 lore raw

PATCH v3 10/22 net/mlx5: relax the list utility atomic operations

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The atomic operation in the list utility no need a barriers because the critical part are managed by RW lock.

Relax them.

---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
drivers/net/mlx5/mlx5_utils.c +6 -6
index 7cdf44dcf7..29248c80ed 100644
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
Signature · 1 line
2.25.1
#39 lore raw

PATCH v3 11/22 net/mlx5: allocate list memory by the create API

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence with the hlist utility.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  56 ++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 137 insertions(+), 81 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +73 -32
index 8a043526da..87b63d852b 100644
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -447,6 +455,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -508,9 +532,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1710,11 +1748,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1771,7 +1811,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_drop_action_destroy(eth_dev);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +2 -1
index 9aade013c5..775ea15adb 100644
@@ -1611,7 +1611,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +5 -5
index 546bee761e..df5cba3d45 100644
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +1 -1
index ce363355c1..ce4d205e86 100644
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
drivers/net/mlx5/mlx5_flow_dv.c +33 -23

164 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxq.c +3 -3
index f8769da8dc..aa9e973d10 100644
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +12 -7
index 29248c80ed..a4526444f9 100644
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
drivers/net/mlx5/mlx5_utils.h +7 -8
index ffa9cd5142..0bf2f5f5ca 100644
@@ -419,15 +419,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index f6cf1928b2..97a8f04e39 100644
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#40 lore raw

PATCH v3 12/22 common/mlx5: add per-lcore cache to hash list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and hlist.

---
 doc/guides/nics/mlx5.rst                |   5 +
 doc/guides/rel_notes/release_21_08.rst  |   6 +
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 466 +++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h | 267 ++++++++++----
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/linux/mlx5_os.c        |  46 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++---
 drivers/net/mlx5/mlx5_flow.h            | 185 +++++-----
 drivers/net/mlx5/mlx5_flow_dv.c         | 407 ++++++++++++---------
 drivers/net/mlx5/mlx5_rx.h              |  13 +-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +--
 drivers/net/mlx5/mlx5_utils.c           | 251 -------------
 drivers/net/mlx5/mlx5_utils.h           | 197 ----------
 drivers/net/mlx5/windows/mlx5_os.c      |   2 +-
 17 files changed, 1029 insertions(+), 1044 deletions(-)
doc/guides/nics/mlx5.rst +5 -0
index eb44a070b1..9bd3846e0d 100644
@@ -445,6 +445,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
doc/guides/rel_notes/release_21_08.rst +6 -0
index a6ecfdf3ce..f6cd1d137d 100644
@@ -55,6 +55,12 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Updated Mellanox mlx5 driver.**
+
+  Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+  * Optimize multiple-thread flow insertion rate.
+
 
 Removed Items
 -------------
drivers/common/mlx5/mlx5_common.h +2 -0
index 1fbefe0fa6..1809ff1e95 100644
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
drivers/common/mlx5/mlx5_common_utils.c +328 -138

530 lines not on this page · raw message

drivers/common/mlx5/mlx5_common_utils.h +199 -68

349 lines not on this page · raw message

drivers/common/mlx5/version.map +7 -0
index b8be73a77b..e6586d6f6f 100644
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
drivers/net/mlx5/linux/mlx5_os.c +24 -22

125 lines not on this page · raw message

drivers/net/mlx5/mlx5.c +6 -4
index 775ea15adb..0e80408511 100644
@@ -1358,20 +1358,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
drivers/net/mlx5/mlx5.h +1 -0
index df5cba3d45..f3768ee028 100644
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
drivers/net/mlx5/mlx5_flow.c +113 -42

298 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +90 -95

242 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +232 -175

764 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -8
index 5450ddd388..3f2b99fb65 100644
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
drivers/net/mlx5/mlx5_rxq.c +10 -43

91 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +0 -251

257 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +0 -197

203 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +1 -1
index 97a8f04e39..af2f684648 100644
@@ -608,7 +608,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#41 lore raw

PATCH v3 13/22 net/mlx5: move modify header allocator to ipool

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Modify header actions are allocated by mlx5_malloc which has a big overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header handler.

---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.c +4 -0
index 0e80408511..713accf675 100644
@@ -801,6 +801,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -814,6 +815,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
drivers/net/mlx5/mlx5.h +14 -0
index f3768ee028..c7239e1137 100644
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1123,6 +1136,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
drivers/net/mlx5/mlx5_flow.h +1 -13
index ab4e8c5c4f..4552aaa803 100644
@@ -504,23 +504,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
drivers/net/mlx5/mlx5_flow_dv.c +67 -12

138 lines not on this page · raw message

Signature · 1 line
2.25.1
#42 lore raw

PATCH v3 14/22 net/mlx5: adjust the hash bucket size

In reply to Suanming Mou

With the new per core optimization to the list, the hash bucket size can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index cf573a9a4d..a82dc4db00 100644
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
drivers/net/mlx5/mlx5.c +1 -1
index 713accf675..8fb7f4442d 100644
@@ -373,7 +373,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
drivers/net/mlx5/mlx5_defs.h +3 -3
index 906aa43c5a..ca67ce8213 100644
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
drivers/net/mlx5/mlx5_flow.c +2 -3
index 7bd45d3895..15a895ea48 100644
@@ -8625,7 +8625,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8734,8 +8734,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
Signature · 1 line
2.25.1
#43 lore raw

PATCH v3 15/22 common/mlx5: allocate cache list memory individually

In reply to Suanming Mou

Currently, the list's local cache instance memory is allocated with the list. As the local cache instance array size is RTE_MAX_LCORE, most of the cases the system will only have very limited cores. allocate the instance memory individually per core will be more economic to the memory.

This commit changes the instance array to pointer array, allocate the local cache memory only when the core is to be used.

---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +44 -18
index 4e385c616a..f75b1cb0da 100644
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
drivers/common/mlx5/mlx5_common_utils.h +1 -1
index 61b30a45ca..979dfafad4 100644
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
Signature · 1 line
2.25.1
#44 lore raw

PATCH v3 16/22 net/mlx5: enable index pool per-core cache

In reply to Suanming Mou

This commit enables the tag and header modify action index pool per-core cache in non-reclaim memory mode.

---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5.c +3 -1
index 8fb7f4442d..bf1463c289 100644
@@ -214,7 +214,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1128,6 +1129,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
drivers/net/mlx5/mlx5.h +1 -0
index c7239e1137..01198b6cc7 100644
@@ -1093,6 +1093,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index e702b78358..f84a2b1a5d 100644
@@ -5321,7 +5321,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
Signature · 1 line
2.25.1
#45 lore raw

PATCH v3 17/22 net/mlx5: optimize hash list table allocate on demand

In reply to Suanming Mou

Currently, all the hash list tables are allocated during start up. Since different applications may only use dedicated limited actions, optimized the hash list table allocate on demand will save initial memory.

This commit optimizes hash list table allocate on demand.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -43
index a82dc4db00..75324e35d8 100644
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
drivers/net/mlx5/mlx5_defs.h +6 -0
index ca67ce8213..fe86bb40d3 100644
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
drivers/net/mlx5/mlx5_flow_dv.c +75 -4

119 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +0 -2
index af2f684648..0e6f7003b0 100644
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
Signature · 1 line
2.25.1
#46 lore raw

PATCH v3 18/22 common/mlx5: optimize cache list object memory

In reply to Suanming Mou

Currently, hash list uses the cache list as bucket list. The list in the buckets have the same name, ctx and callbacks. This wastes the memory.

This commit abstracts all the name, ctx and callback members in the list to a constant struct and others to the inconstant struct, uses the wrapper functions to satisfy both hash list and cache list can set the list constant and inconstant struct individually.

---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +171 -124
index f75b1cb0da..858c8d8164 100644
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +30 -15

69 lines not on this page · raw message

Signature · 1 line
2.25.1
#47 lore raw

PATCH v3 19/22 net/mlx5: change memory release configuration

In reply to Suanming Mou

This commit changes the index pool memory release configuration to 0 when memory reclaim mode is not required.

---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/mlx5/mlx5.c +2 -0
index bf1463c289..6b7225e55d 100644
@@ -797,6 +797,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
Signature · 1 line
2.25.1
#48 lore raw

PATCH v3 20/22 net/mlx5: support index pool none local core operations

In reply to Suanming Mou

This commit supports the index pool none local core operations with an extra cache.

---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)
drivers/net/mlx5/mlx5_utils.c +54 -21
index 94abe79860..c34d6d62a8 100644
@@ -114,6 +114,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->nlcore_lock);
 	return pool;
 }
 
@@ -354,20 +355,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -378,15 +373,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -399,29 +406,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -464,6 +482,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->nlcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->nlcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -643,7 +676,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -712,7 +745,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
drivers/net/mlx5/mlx5_utils.h +2 -1
index 7d9b64c877..060c52f022 100644
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t nlcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
Signature · 1 line
2.25.1
#49 lore raw

PATCH v3 21/22 net/mlx5: support list none local core operations

In reply to Suanming Mou

This commit supports the list none local core operations with an extra sub-list.

---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +63 -29
index 858c8d8164..d58d0d08ab 100644
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.nlcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.nlcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.nlcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.nlcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.nlcore_lock);
+	return ret;
 }
 
 void
drivers/common/mlx5/mlx5_common_utils.h +8 -1
index 9e8ebe772a..95106afc5e 100644
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t nlcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
Signature · 1 line
2.25.1
#50 lore raw

PATCH v3 22/22 net/mlx5: optimize Rx queue match

In reply to Suanming Mou

As hrxq struct has the indirect table pointer, while matching the hrxq, better to use the hrxq indirect table instead of searching from the list.

This commit optimizes the hrxq indirect table matching.

---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5_rxq.c +7 -11
index 7893b3edd4..23685d7654 100644
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
Signature · 1 line
2.25.1
#51 lore raw

v4 cover net/mlx5: insertion rate optimization

In reply to Suanming Mou

This patch series optimize the flow insertion rate with adding local cache to index pool and list.

For object which wants efficient index allocate and free, local cache will be very helpful.

For index pool, two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

For list, sub local core list is introduced. The allocated objects will be added and released only from local list without any locks. Only the objects need to be shared will be synced from global list.

---

v4:
- split the list utility to commom patch
- fix one ipool crash issue in mlx5_ipool_get_cache() function.
- reorganize the common and mlx5 patches.
- split the doc update to an individual patch.
- update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
net/mlx5: optimize modify header action memory net/mlx5: remove cache term from the list utility net/mlx5: add per lcore cache to the list utility net/mlx5: minimize list critical sections net/mlx5: manage list cache entries release net/mlx5: relax the list utility atomic operations net/mlx5: allocate list memory by the create API common/mlx5: add per-lcore cache to hash list utility net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
net/mlx5: allow limiting the index pool maximum index net/mlx5: add indexed pool local cache
net/mlx5: add index pool foreach define net/mlx5: support index pool non-lcore operations net/mlx5: replace flow list with index pool common/mlx5: move list utility to common common/mlx5: add list lcore share
common/mlx5: call list callbacks with context common/mlx5: allocate cache list memory individually common/mlx5: optimize cache list object memory common/mlx5: support list non-lcore operations net/mlx5: adjust the hash bucket size
net/mlx5: enable index pool per-core cache net/mlx5: optimize hash list table allocate on demand net/mlx5: change memory release configuration net/mlx5: optimize Rx queue match
doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    6 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  305 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1203 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2545 insertions(+), 1411 deletions(-)
Signature · 1 line
2.25.1
#52 lore raw

PATCH v4 1/26 net/mlx5: allow limiting the index pool maximum index

Some ipool instances in the driver are used as ID\index allocator and added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value. The ipool will ensure that no index bigger than the maximum value is provided.

Use this configuration in ID allocator cases instead of the current logics. This patch add the maximum ID configurable for the index pool.

---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +12 -2
index 18fe23e4fb..bf2b2ebc72 100644
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
drivers/net/mlx5/mlx5_utils.h +1 -0
index b54517c6df..15870e14c2 100644
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
Signature · 1 line
2.25.1
#53 lore raw

PATCH v4 2/26 net/mlx5: add indexed pool local cache

In reply to Suanming Mou

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is introduced to allocate and free the index more efficient. One as local and the other as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_utils.c +313 -10
index bf2b2ebc72..215024632d 100644
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
drivers/net/mlx5/mlx5_utils.h +59 -5

85 lines not on this page · raw message

Signature · 1 line
2.25.1
#54 lore raw

PATCH v4 3/26 net/mlx5: add index pool foreach define

In reply to Suanming Mou

In some cases, application may want to know all the allocated index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach operation.

---
 drivers/net/mlx5/mlx5_utils.c | 96 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 +++
 2 files changed, 104 insertions(+)
drivers/net/mlx5/mlx5_utils.c +96 -0
index 215024632d..32f8d65073 100644
@@ -529,6 +529,16 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -839,6 +849,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
drivers/net/mlx5/mlx5_utils.h +8 -0
index 0469062695..737dd7052d 100644
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
Signature · 1 line
2.25.1
#55 lore raw

PATCH v4 4/26 net/mlx5: support index pool non-lcore operations

In reply to Suanming Mou

This commit supports the index pool non-lcore operations with an extra cache and lcore lock.

---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)
drivers/net/mlx5/mlx5_utils.c +54 -21
index 32f8d65073..f9557c09ff 100644
@@ -275,6 +275,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->lcore_lock);
 	return pool;
 }
 
@@ -515,20 +516,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -549,15 +544,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -570,29 +577,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -635,6 +653,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -814,7 +847,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -883,7 +916,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
drivers/net/mlx5/mlx5_utils.h +2 -1
index 737dd7052d..a509b0a4eb 100644
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t lcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
Signature · 1 line
2.25.1
#56 lore raw

PATCH v4 5/26 net/mlx5: replace flow list with index pool

In reply to Suanming Mou

The flow list is used to save the create flows and to be used only when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to flush all the allocated flows.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 +++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 149 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 126 insertions(+), 110 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +47 -1
index b3f9e392ab..4b050b4f4a 100644
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -823,6 +861,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1566,7 +1605,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1602,6 +1640,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
drivers/net/mlx5/mlx5.c +6 -3
index abd573e8ed..2c99da0fac 100644
@@ -322,7 +322,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -790,8 +791,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1526,7 +1529,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
drivers/net/mlx5/mlx5.h +12 -2
index 0f4b239142..b196fd365f 100644
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1344,7 +1352,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1597,7 +1606,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
drivers/net/mlx5/mlx5_flow.c +52 -97

343 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +1 -1
index 2f2aa962f9..d9b6acaafd 100644
@@ -997,9 +997,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index 75ef6216ac..eee3926c06 100644
@@ -13881,6 +13881,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
drivers/net/mlx5/mlx5_trigger.c +3 -5
index ae7fcca229..7cb8920d6b 100644
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
drivers/net/mlx5/windows/mlx5_os.c +0 -1
index e30b682822..17716b66c9 100644
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
Signature · 1 line
2.25.1
#57 lore raw

PATCH v4 6/26 net/mlx5: optimize modify header action memory

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Define the types of the modify header action fields to be with the minimum size needed for the optional values range.

---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)
drivers/common/mlx5/linux/mlx5_glue.h +1 -0
index 840d8cf57f..a186ee577f 100644
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
drivers/net/mlx5/linux/mlx5_flow_os.h +2 -1
index cee685015b..1926d26410 100644
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
drivers/net/mlx5/mlx5_flow.h +3 -3
index d9b6acaafd..81c95e0beb 100644
@@ -523,11 +523,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
drivers/net/mlx5/mlx5_flow_dv.c +6 -7
index eee3926c06..bca8339361 100644
@@ -5000,21 +5000,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5616,10 +5616,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
Signature · 1 line
2.25.1
#58 lore raw

PATCH v4 7/26 net/mlx5: remove cache term from the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The internal mlx5 list tool is used mainly when the list objects need to be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread cache management.

To prevent confusing, remove the current "cache" term from the API's names.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 558 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 454 insertions(+), 460 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -16
index 4b050b4f4a..57b0a1c57f 100644
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -500,8 +500,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1704,7 +1704,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1765,7 +1765,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -1
index 2c99da0fac..f51be5ace6 100644
@@ -1609,7 +1609,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +7 -8
index b196fd365f..bf1fbb530b 100644
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1221,7 +1221,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1369,7 +1369,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
drivers/net/mlx5/mlx5_flow.h +45 -43

130 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +276 -282

1,122 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +6 -6
index 1b264e5994..3dcc71d51d 100644
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +14 -14
index bb9a908087..8395332507 100644
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +39 -39

188 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +47 -47

215 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +3 -4
index 17716b66c9..bcf72dc6db 100644
@@ -610,10 +610,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#59 lore raw

PATCH v4 8/26 net/mlx5: add per lcore cache to the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When mlx5 list object is accessed by multiple cores, the list lock counter is all the time written by all the cores what increases cache misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup operation, all the other threads coming to do an operation in the list are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when the list objects are mostly reused.

Synchronization with atomic operations should be done in order to allow threads to unregister an entry from other thread cache.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +34 -24
index 57b0a1c57f..d7119dd561 100644
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1704,10 +1712,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.h +1 -0
index bf1fbb530b..740695ea5c 100644
@@ -1365,6 +1365,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
drivers/net/mlx5/mlx5_flow.h +20 -1
index 4dec703366..ce363355c1 100644
@@ -1634,7 +1634,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1642,6 +1646,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1650,6 +1659,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1658,6 +1672,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow_dv.c +179 -2

219 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -0
index 3dcc71d51d..5450ddd388 100644
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +52 -19

117 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +135 -79

288 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +22 -8
index cfb3cb6180..9e3fe0cb85 100644
@@ -310,9 +310,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -340,6 +345,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -376,15 +388,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -394,8 +408,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -408,10 +420,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +3 -2
index bcf72dc6db..8ced98f0dc 100644
@@ -610,9 +610,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#60 lore raw

PATCH v4 9/26 net/mlx5: minimize list critical sections

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical sections.

These operations are heavy and make the critical sections heavy because they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation counter in order to detect parallel allocations.

---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)
drivers/net/mlx5/mlx5_utils.c +45 -41
index f505caed4e..c4c9adb039 100644
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
drivers/net/mlx5/mlx5_utils.h +3 -2
index 9e3fe0cb85..6dade8238d 100644
@@ -388,8 +388,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
Signature · 1 line
2.25.1
#61 lore raw

PATCH v4 10/26 net/mlx5: manage list cache entries release

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When a cache entry is allocated by lcore A and is released by lcore B, the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be increased atomically when the non-original lcore decreases the reference counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in its cache, it will check the counter in order to free invalid entries in its cache.

---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)
drivers/net/mlx5/mlx5_utils.c +52 -27
index c4c9adb039..13c7dbe1c2 100644
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/net/mlx5/mlx5_utils.h +2 -0
index 6dade8238d..71da5ab4f9 100644
@@ -311,11 +311,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
Signature · 1 line
2.25.1
#62 lore raw

PATCH v4 11/26 net/mlx5: relax the list utility atomic operations

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The atomic operation in the list utility no need a barriers because the critical part are managed by RW lock.

Relax them.

---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
drivers/net/mlx5/mlx5_utils.c +6 -6
index 13c7dbe1c2..daecf37575 100644
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
Signature · 1 line
2.25.1
#63 lore raw

PATCH v4 12/26 net/mlx5: allocate list memory by the create API

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence with the hlist utility.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  56 ++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 137 insertions(+), 81 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +73 -32
index d7119dd561..da4d2fdadc 100644
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -447,6 +455,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -508,9 +532,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1712,11 +1750,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1775,7 +1815,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +2 -1
index f51be5ace6..6661d041ed 100644
@@ -1609,7 +1609,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +5 -5
index 740695ea5c..5da3d93a5b 100644
@@ -1114,10 +1114,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1359,7 +1359,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +1 -1
index ce363355c1..ce4d205e86 100644
@@ -590,7 +590,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
drivers/net/mlx5/mlx5_flow_dv.c +33 -23

164 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxq.c +3 -3
index f8769da8dc..aa9e973d10 100644
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +12 -7
index daecf37575..0be778935f 100644
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
drivers/net/mlx5/mlx5_utils.h +7 -8
index 71da5ab4f9..ea64bb75c9 100644
@@ -420,15 +420,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index 8ced98f0dc..e6176e70d2 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#64 lore raw

PATCH v4 13/26 common/mlx5: move list utility to common

In reply to Suanming Mou

Hash list is planned to be implemented with the cache list code.

This commit moves the list utility to common directory.

---
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 250 +++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h | 205 +++++++++++++++++++
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/mlx5_utils.c           | 251 ------------------------
 drivers/net/mlx5/mlx5_utils.h           | 197 -------------------
 6 files changed, 464 insertions(+), 448 deletions(-)
drivers/common/mlx5/mlx5_common.h +2 -0
index 306f2f1ab7..7fb7d40b38 100644
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
drivers/common/mlx5/mlx5_common_utils.c +250 -0

256 lines not on this page · raw message

drivers/common/mlx5/mlx5_common_utils.h +205 -0

211 lines not on this page · raw message

drivers/common/mlx5/version.map +7 -0
index b8be73a77b..e6586d6f6f 100644
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
drivers/net/mlx5/mlx5_utils.c +0 -251

257 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +0 -197

203 lines not on this page · raw message

Signature · 1 line
2.25.1
#65 lore raw

PATCH v4 14/26 common/mlx5: add list lcore share

In reply to Suanming Mou

As some actions in SW-steering is only memory and can be allowed to create duplicate objects, for lists which no need to check if there are existing same objects in other sub local lists, search the object only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list register.

---
 drivers/common/mlx5/mlx5_common_utils.c | 46 +++++++++++++++++++------
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 55 insertions(+), 22 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +35 -11
index 8bb8a6016d..6ac78ba97f 100644
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/common/mlx5/mlx5_common_utils.h +12 -4
index 96add6d003..000279d236 100644
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
drivers/net/mlx5/linux/mlx5_os.c +6 -5
index da4d2fdadc..ced88f5394 100644
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1750,7 +1750,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
drivers/net/mlx5/mlx5_flow_dv.c +1 -1
index 982056cb41..4f7cdb0622 100644
@@ -10047,7 +10047,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index e6176e70d2..a04f93e1d4 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#66 lore raw

PATCH v4 15/26 common/mlx5: call list callbacks with context

In reply to Suanming Mou

This commit optimizes to call the list callback functions with global context directly.

---
 drivers/common/mlx5/mlx5_common_utils.c |  24 ++---
 drivers/common/mlx5/mlx5_common_utils.h |  36 ++++----
 drivers/net/mlx5/mlx5_flow.h            |  94 +++++++++----------
 drivers/net/mlx5/mlx5_flow_dv.c         | 115 +++++++++++-------------
 drivers/net/mlx5/mlx5_rx.h              |  13 ++-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +++--------
 6 files changed, 132 insertions(+), 203 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +12 -12
index 6ac78ba97f..50c98143f7 100644
@@ -55,7 +55,7 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
@@ -96,7 +96,7 @@ static struct mlx5_list_entry *
 mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
@@ -121,9 +121,9 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
 			if (list->lcores_share)
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			else
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -162,7 +162,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_rwlock_read_unlock(&list->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
+	entry = list->cb_create(list->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
@@ -174,9 +174,9 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list, entry, ctx);
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
+		list->cb_remove(list->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
@@ -192,8 +192,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
 			return mlx5_list_cache_insert(list, lcore_index, oentry,
 						      ctx);
 		}
@@ -223,9 +223,9 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (list->lcores_share)
-			list->cb_clone_free(list, entry);
+			list->cb_clone_free(list->ctx, entry);
 		else
-			list->cb_remove(list, entry);
+			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
@@ -244,7 +244,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
+		list->cb_remove(list->ctx, gentry);
 		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
drivers/common/mlx5/mlx5_common_utils.h +15 -21
index 000279d236..a691ff8f0d 100644
@@ -33,19 +33,19 @@ struct mlx5_list_cache {
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -54,34 +54,28 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
 				   struct mlx5_list_entry *entry, void *ctx);
 
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
 
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
 
 /**
  * Linked mlx5 list structure.
drivers/net/mlx5/mlx5_flow.h +40 -54

104 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +50 -65

294 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -8
index 5450ddd388..3f2b99fb65 100644
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
drivers/net/mlx5/mlx5_rxq.c +10 -43

91 lines not on this page · raw message

Signature · 1 line
2.25.1
#67 lore raw

PATCH v4 16/26 common/mlx5: add per-lcore cache to hash list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and hlist.

---
 drivers/common/mlx5/mlx5_common_utils.c | 238 +++++++------------
 drivers/common/mlx5/mlx5_common_utils.h | 141 +++---------
 drivers/net/mlx5/linux/mlx5_os.c        |  35 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++++++----
 drivers/net/mlx5/mlx5_flow.h            |  92 ++++----
 drivers/net/mlx5/mlx5_flow_dv.c         | 292 +++++++++++++++---------
 8 files changed, 482 insertions(+), 482 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +77 -161
index 50c98143f7..4e385c616a 100644
@@ -13,25 +13,21 @@
 
 /********************* mlx5 list ************************/
 
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx, bool lcores_share,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	struct mlx5_list *list;
 	int i;
 
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
-		return NULL;
+		return -EINVAL;
 	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -45,6 +41,28 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
 	return list;
 }
 
@@ -254,8 +272,8 @@ mlx5_list_unregister(struct mlx5_list *list,
 	return 1;
 }
 
-void
-mlx5_list_destroy(struct mlx5_list *list)
+static void
+mlx5_list_uninit(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
 	int i;
@@ -266,15 +284,21 @@ mlx5_list_destroy(struct mlx5_list *list)
 			entry = LIST_FIRST(&list->cache[i].h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", list->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			}
 		}
 	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
 	mlx5_free(list);
 }
 
@@ -287,37 +311,24 @@ mlx5_list_get_entry_num(struct mlx5_list *list)
 
 /********************* Hash List **********************/
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -331,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -394,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
+
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +35 -106

197 lines not on this page · raw message

drivers/net/mlx5/linux/mlx5_os.c +18 -17
index ced88f5394..468d8173c4 100644
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1783,15 +1783,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
drivers/net/mlx5/mlx5.c +6 -4
index 6661d041ed..2d4330198b 100644
@@ -1356,20 +1356,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
drivers/net/mlx5/mlx5.h +1 -0
index 5da3d93a5b..56ac018c7e 100644
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
drivers/net/mlx5/mlx5_flow.c +113 -42

298 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +50 -42

143 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +182 -110

464 lines not on this page · raw message

Signature · 1 line
2.25.1