Skip to contentSkip to conversation

net/mlx5: add indexed pool local cache

PATCH New Suanming Mou 2 participants 135 messages May 27, 2021 – Jul 13, 2021 Network drivers Page 2 of 2

lore
Share & tools
lore
#68 lore raw

PATCH v4 17/26 common/mlx5: allocate cache list memory individually

In reply to Suanming Mou

Currently, the list's local cache instance memory is allocated with the list. As the local cache instance array size is RTE_MAX_LCORE, most of the cases the system will only have very limited cores. allocate the instance memory individually per core will be more economic to the memory.

This commit changes the instance array to pointer array, allocate the local cache memory only when the core is to be used.

---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +44 -18
index 4e385c616a..f75b1cb0da 100644
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
drivers/common/mlx5/mlx5_common_utils.h +1 -1
index 4bb974fa3e..d49fb64457 100644
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
Signature · 1 line
2.25.1
#69 lore raw

PATCH v4 18/26 common/mlx5: optimize cache list object memory

In reply to Suanming Mou

Currently, hash list uses the cache list as bucket list. The list in the buckets have the same name, ctx and callbacks. This wastes the memory.

This commit abstracts all the name, ctx and callback members in the list to a constant struct and others to the inconstant struct, uses the wrapper functions to satisfy both hash list and cache list can set the list constant and inconstant struct individually.

---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +171 -124
index f75b1cb0da..858c8d8164 100644
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +30 -15

69 lines not on this page · raw message

Signature · 1 line
2.25.1
#70 lore raw

PATCH v4 19/26 common/mlx5: support list non-lcore operations

In reply to Suanming Mou

This commit supports the list non-lcore operations with an extra sub-list and lock.

---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +63 -29
index 858c8d8164..c8b48f3afc 100644
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.lcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.lcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
+	return ret;
 }
 
 void
drivers/common/mlx5/mlx5_common_utils.h +8 -1
index 5718a21be0..613d29de0c 100644
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t lcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
Signature · 1 line
2.25.1
#71 lore raw

PATCH v4 20/26 net/mlx5: move modify header allocator to ipool

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Modify header actions are allocated by mlx5_malloc which has a big overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header handler.

---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.c +4 -0
index 2d4330198b..499d8449ac 100644
@@ -799,6 +799,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -812,6 +813,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
drivers/net/mlx5/mlx5.h +14 -0
index 56ac018c7e..d88abef12c 100644
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1123,6 +1136,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
drivers/net/mlx5/mlx5_flow.h +1 -13
index 98e90fec0d..da6367c212 100644
@@ -504,23 +504,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
drivers/net/mlx5/mlx5_flow_dv.c +67 -12

138 lines not on this page · raw message

Signature · 1 line
2.25.1
#72 lore raw

PATCH v4 21/26 net/mlx5: adjust the hash bucket size

In reply to Suanming Mou

With the new per core optimization to the list, the hash bucket size can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index 468d8173c4..cfb33251cf 100644
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
drivers/net/mlx5/mlx5.c +1 -1
index 499d8449ac..08e795ea4c 100644
@@ -373,7 +373,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
drivers/net/mlx5/mlx5_defs.h +3 -3
index 906aa43c5a..ca67ce8213 100644
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
drivers/net/mlx5/mlx5_flow.c +2 -3
index 367fbc8e4a..b92ebe9029 100644
@@ -8642,7 +8642,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8751,8 +8751,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
Signature · 1 line
2.25.1
#73 lore raw

PATCH v4 22/26 net/mlx5: enable index pool per-core cache

In reply to Suanming Mou

This commit enables the tag and header modify action index pool per-core cache in non-reclaim memory mode.

---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5.c +3 -1
index 08e795ea4c..53dee9164a 100644
@@ -214,7 +214,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1126,6 +1127,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
drivers/net/mlx5/mlx5.h +1 -0
index d88abef12c..2e2a98345f 100644
@@ -1093,6 +1093,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index 9666c11861..b30e5a82ce 100644
@@ -5355,7 +5355,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
Signature · 1 line
2.25.1
#74 lore raw

PATCH v4 23/26 net/mlx5: optimize hash list table allocate on demand

In reply to Suanming Mou

Currently, all the hash list tables are allocated during start up. Since different applications may only use dedicated limited actions, optimized the hash list table allocate on demand will save initial memory.

This commit optimizes hash list table allocate on demand.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -43
index cfb33251cf..38c5672436 100644
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
drivers/net/mlx5/mlx5_defs.h +6 -0
index ca67ce8213..fe86bb40d3 100644
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
drivers/net/mlx5/mlx5_flow_dv.c +75 -4

119 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +0 -2
index a04f93e1d4..5da362a9d5 100644
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
Signature · 1 line
2.25.1
#75 lore raw

PATCH v4 24/26 net/mlx5: change memory release configuration

In reply to Suanming Mou

This commit changes the index pool memory release configuration to 0 when memory reclaim mode is not required.

---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/mlx5/mlx5.c +2 -0
index 53dee9164a..10594f5961 100644
@@ -795,6 +795,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
Signature · 1 line
2.25.1
#76 lore raw

PATCH v4 25/26 net/mlx5: optimize Rx queue match

In reply to Suanming Mou

As hrxq struct has the indirect table pointer, while matching the hrxq, better to use the hrxq indirect table instead of searching from the list.

This commit optimizes the hrxq indirect table matching.

---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5_rxq.c +7 -11
index 7893b3edd4..23685d7654 100644
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
Signature · 1 line
2.25.1
#77 lore raw

PATCH v4 26/26 doc: add mlx5 multiple-thread flow insertion optimization

In reply to Suanming Mou

This commit adds the multiple-thread flow insertion optimization description.

---
 doc/guides/nics/mlx5.rst               | 5 +++++
 doc/guides/rel_notes/release_21_08.rst | 6 ++++++
 2 files changed, 11 insertions(+)
doc/guides/nics/mlx5.rst +5 -0
index a16af32e67..d7d7ba9741 100644
@@ -445,6 +445,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
doc/guides/rel_notes/release_21_08.rst +6 -0
index 0a05cb02fa..d187edc303 100644
@@ -68,6 +68,12 @@ New Features
   usecases. Configuration happens via standard rawdev enq/deq operations. See
   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.
 
+* **Updated Mellanox mlx5 driver.**
+
+  Updated the Mellanox mlx5 driver with new features and improvements, including:
+
+  * Optimize multiple-thread flow insertion rate.
+
 
 Removed Items
 -------------
Signature · 1 line
2.25.1
#78 lore raw

v5 cover net/mlx5: insertion rate optimization

In reply to Suanming Mou

This patch series optimize the flow insertion rate with adding local cache to index pool and list.

For object which wants efficient index allocate and free, local cache will be very helpful.

For index pool, two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

For list, sub local core list is introduced. The allocated objects will be added and released only from local list without any locks. Only the objects need to be shared will be synced from global list.

---

v5: code rebase to the latest version.

v4:
- split the list utility to commom patch
- fix one ipool crash issue in mlx5_ipool_get_cache() function.
- reorganize the common and mlx5 patches.
- split the doc update to an individual patch.
- update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
net/mlx5: optimize modify header action memory net/mlx5: remove cache term from the list utility net/mlx5: add per lcore cache to the list utility net/mlx5: minimize list critical sections net/mlx5: manage list cache entries release net/mlx5: relax the list utility atomic operations net/mlx5: allocate list memory by the create API common/mlx5: add per-lcore cache to hash list utility net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
net/mlx5: allow limiting the index pool maximum index net/mlx5: add indexed pool local cache
net/mlx5: add index pool foreach define net/mlx5: support index pool non-lcore operations net/mlx5: replace flow list with index pool common/mlx5: move list utility to common common/mlx5: add list lcore share
common/mlx5: call list callbacks with context common/mlx5: allocate cache list memory individually common/mlx5: optimize cache list object memory common/mlx5: support list non-lcore operations net/mlx5: adjust the hash bucket size
net/mlx5: enable index pool per-core cache net/mlx5: optimize hash list table allocate on demand net/mlx5: change memory release configuration net/mlx5: optimize Rx queue match
doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    1 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  308 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1206 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2543 insertions(+), 1414 deletions(-)
Signature · 1 line
2.25.1
#79 lore raw

PATCH v5 1/26 net/mlx5: allow limiting the index pool maximum index

Some ipool instances in the driver are used as ID\index allocator and added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value. The ipool will ensure that no index bigger than the maximum value is provided.

Use this configuration in ID allocator cases instead of the current logics. This patch add the maximum ID configurable for the index pool.

---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +12 -2
index 18fe23e4fb..bf2b2ebc72 100644
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
drivers/net/mlx5/mlx5_utils.h +1 -0
index b54517c6df..15870e14c2 100644
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
Signature · 1 line
2.25.1
#80 lore raw

PATCH v5 2/26 net/mlx5: add indexed pool local cache

In reply to Suanming Mou

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is introduced to allocate and free the index more efficient. One as local and the other as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_utils.c +313 -10
index bf2b2ebc72..215024632d 100644
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
drivers/net/mlx5/mlx5_utils.h +59 -5

85 lines not on this page · raw message

Signature · 1 line
2.25.1
#81 lore raw

PATCH v5 3/26 net/mlx5: add index pool foreach define

In reply to Suanming Mou

In some cases, application may want to know all the allocated index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach operation.

---
 drivers/net/mlx5/mlx5_utils.c | 96 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 +++
 2 files changed, 104 insertions(+)
drivers/net/mlx5/mlx5_utils.c +96 -0
index 215024632d..32f8d65073 100644
@@ -529,6 +529,16 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -839,6 +849,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
drivers/net/mlx5/mlx5_utils.h +8 -0
index 0469062695..737dd7052d 100644
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
Signature · 1 line
2.25.1
#82 lore raw

PATCH v5 4/26 net/mlx5: support index pool non-lcore operations

In reply to Suanming Mou

This commit supports the index pool non-lcore operations with an extra cache and lcore lock.

---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)
drivers/net/mlx5/mlx5_utils.c +54 -21
index 32f8d65073..f9557c09ff 100644
@@ -275,6 +275,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->lcore_lock);
 	return pool;
 }
 
@@ -515,20 +516,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -549,15 +544,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -570,29 +577,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -635,6 +653,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -814,7 +847,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -883,7 +916,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
drivers/net/mlx5/mlx5_utils.h +2 -1
index 737dd7052d..a509b0a4eb 100644
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t lcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
Signature · 1 line
2.25.1
#83 lore raw

PATCH v5 5/26 net/mlx5: replace flow list with index pool

In reply to Suanming Mou

The flow list is used to save the create flows and to be used only when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to flush all the allocated flows.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 ++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 152 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 127 insertions(+), 112 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +47 -1
index be22d9cbd2..867f450cab 100644
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -832,6 +870,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1575,7 +1614,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1611,6 +1649,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
drivers/net/mlx5/mlx5.c +6 -3
index 44fbc2da83..b5c05b12e8 100644
@@ -325,7 +325,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -793,8 +794,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1529,7 +1532,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
drivers/net/mlx5/mlx5.h +12 -2
index f864c1d701..facab913bf 100644
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1367,7 +1375,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1627,7 +1636,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
drivers/net/mlx5/mlx5_flow.c +53 -99

352 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +1 -1
index 7d97c5880f..adadcfa425 100644
@@ -998,9 +998,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index 2f4c0eeb5b..49bc85f7b3 100644
@@ -13901,6 +13901,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
drivers/net/mlx5/mlx5_trigger.c +3 -5
index ae7fcca229..7cb8920d6b 100644
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
drivers/net/mlx5/windows/mlx5_os.c +0 -1
index e30b682822..17716b66c9 100644
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
Signature · 1 line
2.25.1
#84 lore raw

PATCH v5 6/26 net/mlx5: optimize modify header action memory

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Define the types of the modify header action fields to be with the minimum size needed for the optional values range.

---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)
drivers/common/mlx5/linux/mlx5_glue.h +1 -0
index 61f40d5478..f39ef2dac7 100644
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
drivers/net/mlx5/linux/mlx5_flow_os.h +2 -1
index cee685015b..1926d26410 100644
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
drivers/net/mlx5/mlx5_flow.h +3 -3
index adadcfa425..f1ebc153f2 100644
@@ -524,11 +524,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
drivers/net/mlx5/mlx5_flow_dv.c +6 -7
index 49bc85f7b3..0f8b916991 100644
@@ -5006,21 +5006,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5622,10 +5622,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
Signature · 1 line
2.25.1
#85 lore raw

PATCH v5 7/26 net/mlx5: remove cache term from the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The internal mlx5 list tool is used mainly when the list objects need to be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread cache management.

To prevent confusing, remove the current "cache" term from the API's names.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 561 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 456 insertions(+), 461 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -16
index 867f450cab..5d537fa3e4 100644
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -509,8 +509,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1713,7 +1713,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1774,7 +1774,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -1
index b5c05b12e8..5cd2cfb07c 100644
@@ -1612,7 +1612,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +7 -8
index facab913bf..de5cd2e06e 100644
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1244,7 +1244,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1392,7 +1392,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
drivers/net/mlx5/mlx5_flow.h +45 -43

130 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +278 -283

1,131 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +6 -6
index 1b264e5994..3dcc71d51d 100644
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +14 -14
index bb9a908087..8395332507 100644
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +39 -39

188 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +47 -47

215 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +3 -4
index 17716b66c9..bcf72dc6db 100644
@@ -610,10 +610,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#86 lore raw

PATCH v5 8/26 net/mlx5: add per lcore cache to the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When mlx5 list object is accessed by multiple cores, the list lock counter is all the time written by all the cores what increases cache misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup operation, all the other threads coming to do an operation in the list are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when the list objects are mostly reused.

Synchronization with atomic operations should be done in order to allow threads to unregister an entry from other thread cache.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +34 -24
index 5d537fa3e4..25e4bdb7a6 100644
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1713,10 +1721,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.h +1 -0
index de5cd2e06e..1141e12977 100644
@@ -1388,6 +1388,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
drivers/net/mlx5/mlx5_flow.h +20 -1
index cbae4f1872..2016adc6ee 100644
@@ -1646,7 +1646,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1654,6 +1658,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1662,6 +1671,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1670,6 +1684,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow_dv.c +179 -2

219 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -0
index 3dcc71d51d..5450ddd388 100644
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +52 -19

117 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +135 -79

288 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +22 -8
index cfb3cb6180..9e3fe0cb85 100644
@@ -310,9 +310,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -340,6 +345,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -376,15 +388,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -394,8 +408,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -408,10 +420,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +3 -2
index bcf72dc6db..8ced98f0dc 100644
@@ -610,9 +610,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#87 lore raw

PATCH v5 9/26 net/mlx5: minimize list critical sections

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical sections.

These operations are heavy and make the critical sections heavy because they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation counter in order to detect parallel allocations.

---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)
drivers/net/mlx5/mlx5_utils.c +45 -41
index f505caed4e..c4c9adb039 100644
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
drivers/net/mlx5/mlx5_utils.h +3 -2
index 9e3fe0cb85..6dade8238d 100644
@@ -388,8 +388,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
Signature · 1 line
2.25.1
#88 lore raw

PATCH v5 10/26 net/mlx5: manage list cache entries release

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When a cache entry is allocated by lcore A and is released by lcore B, the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be increased atomically when the non-original lcore decreases the reference counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in its cache, it will check the counter in order to free invalid entries in its cache.

---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)
drivers/net/mlx5/mlx5_utils.c +52 -27
index c4c9adb039..13c7dbe1c2 100644
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/net/mlx5/mlx5_utils.h +2 -0
index 6dade8238d..71da5ab4f9 100644
@@ -311,11 +311,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
Signature · 1 line
2.25.1
#89 lore raw

PATCH v5 11/26 net/mlx5: relax the list utility atomic operations

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The atomic operation in the list utility no need a barriers because the critical part are managed by RW lock.

Relax them.

---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
drivers/net/mlx5/mlx5_utils.c +6 -6
index 13c7dbe1c2..daecf37575 100644
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
Signature · 1 line
2.25.1
#90 lore raw

PATCH v5 12/26 net/mlx5: allocate list memory by the create API

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence with the hlist utility.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  60 ++++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 139 insertions(+), 83 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +73 -32
index 25e4bdb7a6..2a9a6c3bf8 100644
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -456,6 +464,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -517,9 +541,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1721,11 +1759,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1784,7 +1824,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +2 -1
index 5cd2cfb07c..d04b5d616f 100644
@@ -1612,7 +1612,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +5 -5
index 1141e12977..88b84d749a 100644
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +1 -1
index 2016adc6ee..59a6f31d2e 100644
@@ -591,7 +591,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
drivers/net/mlx5/mlx5_flow_dv.c +35 -25

180 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxq.c +3 -3
index f8769da8dc..aa9e973d10 100644
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +12 -7
index daecf37575..0be778935f 100644
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
drivers/net/mlx5/mlx5_utils.h +7 -8
index 71da5ab4f9..ea64bb75c9 100644
@@ -420,15 +420,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index 8ced98f0dc..e6176e70d2 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#91 lore raw

PATCH v5 13/26 common/mlx5: move list utility to common

In reply to Suanming Mou

Hash list is planned to be implemented with the cache list code.

This commit moves the list utility to common directory.

---
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 250 +++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h | 205 +++++++++++++++++++
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/mlx5_utils.c           | 251 ------------------------
 drivers/net/mlx5/mlx5_utils.h           | 197 -------------------
 6 files changed, 464 insertions(+), 448 deletions(-)
drivers/common/mlx5/mlx5_common.h +2 -0
index 306f2f1ab7..7fb7d40b38 100644
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
drivers/common/mlx5/mlx5_common_utils.c +250 -0

256 lines not on this page · raw message

drivers/common/mlx5/mlx5_common_utils.h +205 -0

211 lines not on this page · raw message

drivers/common/mlx5/version.map +7 -0
index b8be73a77b..e6586d6f6f 100644
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
drivers/net/mlx5/mlx5_utils.c +0 -251

257 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +0 -197

203 lines not on this page · raw message

Signature · 1 line
2.25.1
#92 lore raw

PATCH v5 14/26 common/mlx5: add list lcore share

In reply to Suanming Mou

As some actions in SW-steering is only memory and can be allowed to create duplicate objects, for lists which no need to check if there are existing same objects in other sub local lists, search the object only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list register.

---
 drivers/common/mlx5/mlx5_common_utils.c | 46 +++++++++++++++++++------
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 55 insertions(+), 22 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +35 -11
index 8bb8a6016d..6ac78ba97f 100644
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/common/mlx5/mlx5_common_utils.h +12 -4
index 96add6d003..000279d236 100644
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
drivers/net/mlx5/linux/mlx5_os.c +6 -5
index 2a9a6c3bf8..ce41fb34a0 100644
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1759,7 +1759,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
drivers/net/mlx5/mlx5_flow_dv.c +1 -1
index 5a536e3dff..4a45172a12 100644
@@ -10054,7 +10054,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index e6176e70d2..a04f93e1d4 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#93 lore raw

Re: [dpdk-dev] [PATCH v5 14/26] common/mlx5: add list lcore share

Hi Suanming,

This patch will cause the following failure in compilation with CLANG :
[1443/3183] Compiling C object drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
FAILED: drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o
clang -Idrivers/libtmp_rte_common_mlx5.a.p -Idrivers -I../../root/dpdk/drivers -Idrivers/common/mlx5 -I../../root/dpdk/drivers/common/mlx5 -Idrivers/common/mlx5/linux -I../../root/dpdk/drivers/common/mlx5/linux -Ilib/hash -I../../root/dpdk/lib/hash -I. -I../../root/dpdk -Iconfig -I../../root/dpdk/config -Ilib/eal/include -I../../root/dpdk/lib/eal/include -Ilib/eal/linux/include -I../../root/dpdk/lib/eal/linux/include -Ilib/eal/x86/include -I../../root/dpdk/lib/eal/x86/include -Ilib/eal/common -I../../root/dpdk/lib/eal/common -Ilib/eal -I../../root/dpdk/lib/eal -Ilib/kvargs -I../../root/dpdk/lib/kvargs -Ilib/metrics -I../../root/dpdk/lib/metrics -Ilib/telemetry -I../../root/dpdk/lib/telemetry -Ilib/net -I../../root/dpdk/lib/net -Ilib/mbuf -I../../root/dpdk/lib/mbuf -Ilib/mempool -I../../root/dpdk/lib/mempool -Ilib/ring -I../../root/dpdk/lib/ring -Ilib/rcu -I../../root/dpdk/lib/rcu -Ilib/pci -I../../root/dpdk/lib/pci -Idrivers/bus/pci -I../../root/dpdk/drivers/bus/pci -I../../root/dpdk/drivers/bus/pci/linux -I/usr/usr/include -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual -Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings -Wno-address-of-packed-member -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -std=c11 -Wno-strict-prototypes -D_BSD_SOURCE -D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -pedantic -DPEDANTIC -DRTE_LOG_DEFAULT_LOGTYPE=pmd.common.mlx5 -MD -MQ drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o -MF drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o.d -o drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o -c ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:6: error: variable 'entry' is used uninitialized whenever 'if' condition is false [-Werror,-Wsometimes-uninitialized]

        if (list->lcores_share) {
            ^~~~~~~~~~~~~~~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:165:32: note: uninitialized use occurs here
        entry = list->cb_create(list, entry, ctx);
                                      ^~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:2: note: remove the 'if' if its condition is always true
        if (list->lcores_share) {
        ^~~~~~~~~~~~~~~~~~~~~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:136:31: note: initialize the variable 'entry' to silence this warning
        struct mlx5_list_entry *entry, *local_entry;
                                     ^
                                      = NULL

1 error generated.
[1500/3183] Compiling C object drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_tx_empw.c.o
ninja: build stopped: subcommand failed.

########################
Build failed!

        CC: clang version 12.0.0 (Fedora 12.0.0-2.fc34)

Kindest regards,
Raslan Darawsheh

Quoted 284 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Monday, July 12, 2021 4:47 AM
To: Slava Ovsiienko <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; Ori Kam <[email protected]>; [email protected]
Subject: [PATCH v5 14/26] common/mlx5: add list lcore share

As some actions in SW-steering is only memory and can be allowed to create duplicate objects, for lists which no need to check if there are existing same objects in other sub local lists, search the object only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list register.

---
 drivers/common/mlx5/mlx5_common_utils.c | 46 +++++++++++++++++++-
-----
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 55 insertions(+), 22 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +6 -2
index 8bb8a6016d..6ac78ba97f 100644
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int
 lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED)
 == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
drivers/common/mlx5/mlx5_common_utils.c +19 -8
 		}
 		entry = nentry;
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index,
 entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry,
 next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry-
 >ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
drivers/common/mlx5/mlx5_common_utils.c +10 -1
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt,
 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED)
 != 0)
drivers/common/mlx5/mlx5_common_utils.c +0 -0
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/common/mlx5/mlx5_common_utils.h +1 -4
index 96add6d003..000279d236 100644
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry
 *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores.
 */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
drivers/common/mlx5/mlx5_common_utils.h +11 -0
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index 2a9a6c3bf8..ce41fb34a0 100644
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 
 flow_dv_port_id_create_cb,
 

flow_dv_port_id_match_cb,

flow_dv_port_id_remove_cb,

drivers/net/mlx5/linux/mlx5_os.c +1 -1
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 
 flow_dv_push_vlan_create_cb,
 

flow_dv_push_vlan_match_cb,

flow_dv_push_vlan_remove_cb,

drivers/net/mlx5/linux/mlx5_os.c +1 -1
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 

flow_dv_sample_remove_cb,

drivers/net/mlx5/linux/mlx5_os.c +3 -2
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1759,7 +1759,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
drivers/net/mlx5/linux/mlx5_os.c +0 -0
 				       mlx5_hrxq_clone_cb,
drivers/net/mlx5/mlx5_flow_dv.c +1 -1
index 5a536e3dff..4a45172a12 100644
@@ -10054,7 +10054,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list,
 uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
drivers/net/mlx5/mlx5_flow_dv.c +0 -0
 					      flow_dv_matcher_remove_cb,
drivers/net/mlx5/windows/mlx5_os.c +1 -2
index e6176e70d2..a04f93e1d4 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
--

2.25.1

#94 lore raw

Re: [dpdk-dev] [PATCH v5 14/26] common/mlx5: add list lcore share

Hi Raslan,

Quoted 69 lines

-----Original Message-----
From: Raslan Darawsheh <[email protected]>
Sent: Monday, July 12, 2021 10:59 PM
To: Suanming Mou <[email protected]>; Slava Ovsiienko <[email protected]>; Matan Azrad <[email protected]>
Cc: Ori Kam <[email protected]>; [email protected]
Subject: RE: [PATCH v5 14/26] common/mlx5: add list lcore share

Hi Suanming,

This patch will cause the following failure in compilation with CLANG :
[1443/3183] Compiling C object drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o FAILED:
drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o clang -Idrivers/libtmp_rte_common_mlx5.a.p -Idrivers -I../../root/dpdk/drivers - Idrivers/common/mlx5 -I../../root/dpdk/drivers/common/mlx5 - Idrivers/common/mlx5/linux -I../../root/dpdk/drivers/common/mlx5/linux - Ilib/hash -I../../root/dpdk/lib/hash -I. -I../../root/dpdk -Iconfig - I../../root/dpdk/config -Ilib/eal/include -I../../root/dpdk/lib/eal/include - Ilib/eal/linux/include -I../../root/dpdk/lib/eal/linux/include -Ilib/eal/x86/include -I../../root/dpdk/lib/eal/x86/include -Ilib/eal/common - I../../root/dpdk/lib/eal/common -Ilib/eal -I../../root/dpdk/lib/eal -Ilib/kvargs - I../../root/dpdk/lib/kvargs -Ilib/metrics -I../../root/dpdk/lib/metrics - Ilib/telemetry -I../../root/dpdk/lib/telemetry -Ilib/net -I../../root/dpdk/lib/net - Ilib/mbuf -I../../root/dpdk/lib/mbuf -Ilib/mempool - I../../root/dpdk/lib/mempool -Ilib/ring -I../../root/dpdk/lib/ring -Ilib/rcu - I../../root/dpdk/lib/rcu -Ilib/pci -I../../root/dpdk/lib/pci -Idrivers/bus/pci - I../../root/dpdk/drivers/bus/pci -I../../root/dpdk/drivers/bus/pci/linux - I/usr/usr/include -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 - Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual - Wdeprecated -Wformat -Wformat-nonliteral -Wformat-security -Wmissing- declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition - Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwrite-strings - Wno-address-of-packed-member -Wno-missing-field-initializers - D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API - DALLOW_INTERNAL_API -std=c11 -Wno-strict-prototypes -D_BSD_SOURCE - D_DEFAULT_SOURCE -D_XOPEN_SOURCE=600 -pedantic -DPEDANTIC - DRTE_LOG_DEFAULT_LOGTYPE=pmd.common.mlx5 -MD -MQ drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o -MF drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o. d -o drivers/libtmp_rte_common_mlx5.a.p/common_mlx5_mlx5_common_utils.c.o -c ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c ../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:6: error:
variable 'entry' is used uninitialized whenever 'if' condition is false [-Werror,- Wsometimes-uninitialized]

        if (list->lcores_share) {
            ^~~~~~~~~~~~~~~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:165:32: note:
uninitialized use occurs here
        entry = list->cb_create(list, entry, ctx);
                                      ^~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:152:2: note:
remove the 'if' if its condition is always true
        if (list->lcores_share) {
        ^~~~~~~~~~~~~~~~~~~~~~~~
../../root/dpdk/drivers/common/mlx5/mlx5_common_utils.c:136:31: note:
initialize the variable 'entry' to silence this warning
        struct mlx5_list_entry *entry, *local_entry;
                                     ^
                                      = NULL

1 error generated.
[1500/3183] Compiling C object drivers/libtmp_rte_net_mlx5.a.p/net_mlx5_mlx5_tx_empw.c.o
ninja: build stopped: subcommand failed.

Quoted 3 lines

########################
Build failed!

        CC: clang version 12.0.0 (Fedora 12.0.0-2.fc34)

OK, I will check it.

Quoted 5 lines

Kindest regards,
Raslan Darawsheh

#95 lore raw

PATCH v5 15/26 common/mlx5: call list callbacks with context

In reply to Suanming Mou

This commit optimizes to call the list callback functions with global context directly.

---
 drivers/common/mlx5/mlx5_common_utils.c |  24 ++---
 drivers/common/mlx5/mlx5_common_utils.h |  36 ++++----
 drivers/net/mlx5/mlx5_flow.h            |  94 +++++++++----------
 drivers/net/mlx5/mlx5_flow_dv.c         | 115 +++++++++++-------------
 drivers/net/mlx5/mlx5_rx.h              |  13 ++-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +++--------
 6 files changed, 132 insertions(+), 203 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +12 -12
index 6ac78ba97f..50c98143f7 100644
@@ -55,7 +55,7 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
@@ -96,7 +96,7 @@ static struct mlx5_list_entry *
 mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
@@ -121,9 +121,9 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
 			if (list->lcores_share)
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			else
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -162,7 +162,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_rwlock_read_unlock(&list->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
+	entry = list->cb_create(list->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
@@ -174,9 +174,9 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list, entry, ctx);
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
+		list->cb_remove(list->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
@@ -192,8 +192,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
 			return mlx5_list_cache_insert(list, lcore_index, oentry,
 						      ctx);
 		}
@@ -223,9 +223,9 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (list->lcores_share)
-			list->cb_clone_free(list, entry);
+			list->cb_clone_free(list->ctx, entry);
 		else
-			list->cb_remove(list, entry);
+			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
@@ -244,7 +244,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
+		list->cb_remove(list->ctx, gentry);
 		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
drivers/common/mlx5/mlx5_common_utils.h +15 -21
index 000279d236..a691ff8f0d 100644
@@ -33,19 +33,19 @@ struct mlx5_list_cache {
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -54,34 +54,28 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
 				   struct mlx5_list_entry *entry, void *ctx);
 
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
 
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
 
 /**
  * Linked mlx5 list structure.
drivers/net/mlx5/mlx5_flow.h +40 -54

104 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +50 -65

294 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -8
index 5450ddd388..3f2b99fb65 100644
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
drivers/net/mlx5/mlx5_rxq.c +10 -43

91 lines not on this page · raw message

Signature · 1 line
2.25.1
#96 lore raw

PATCH v5 16/26 common/mlx5: add per-lcore cache to hash list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and hlist.

---
 drivers/common/mlx5/mlx5_common_utils.c | 238 +++++++------------
 drivers/common/mlx5/mlx5_common_utils.h | 141 +++---------
 drivers/net/mlx5/linux/mlx5_os.c        |  35 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++++++----
 drivers/net/mlx5/mlx5_flow.h            |  92 ++++----
 drivers/net/mlx5/mlx5_flow_dv.c         | 292 +++++++++++++++---------
 8 files changed, 482 insertions(+), 482 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +77 -161
index 50c98143f7..4e385c616a 100644
@@ -13,25 +13,21 @@
 
 /********************* mlx5 list ************************/
 
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx, bool lcores_share,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	struct mlx5_list *list;
 	int i;
 
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
-		return NULL;
+		return -EINVAL;
 	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -45,6 +41,28 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
 	return list;
 }
 
@@ -254,8 +272,8 @@ mlx5_list_unregister(struct mlx5_list *list,
 	return 1;
 }
 
-void
-mlx5_list_destroy(struct mlx5_list *list)
+static void
+mlx5_list_uninit(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
 	int i;
@@ -266,15 +284,21 @@ mlx5_list_destroy(struct mlx5_list *list)
 			entry = LIST_FIRST(&list->cache[i].h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", list->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			}
 		}
 	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
 	mlx5_free(list);
 }
 
@@ -287,37 +311,24 @@ mlx5_list_get_entry_num(struct mlx5_list *list)
 
 /********************* Hash List **********************/
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -331,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -394,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
+
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +35 -106

197 lines not on this page · raw message

drivers/net/mlx5/linux/mlx5_os.c +18 -17
index ce41fb34a0..b04e5f8ba6 100644
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1792,15 +1792,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
drivers/net/mlx5/mlx5.c +6 -4
index d04b5d616f..fd675c9262 100644
@@ -1359,20 +1359,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
drivers/net/mlx5/mlx5.h +1 -0
index 88b84d749a..e9b08094a6 100644
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
drivers/net/mlx5/mlx5_flow.c +113 -42

298 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +50 -42

143 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +182 -110

464 lines not on this page · raw message

Signature · 1 line
2.25.1
#97 lore raw

PATCH v5 17/26 common/mlx5: allocate cache list memory individually

In reply to Suanming Mou

Currently, the list's local cache instance memory is allocated with the list. As the local cache instance array size is RTE_MAX_LCORE, most of the cases the system will only have very limited cores. allocate the instance memory individually per core will be more economic to the memory.

This commit changes the instance array to pointer array, allocate the local cache memory only when the core is to be used.

---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +44 -18
index 4e385c616a..f75b1cb0da 100644
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
drivers/common/mlx5/mlx5_common_utils.h +1 -1
index 4bb974fa3e..d49fb64457 100644
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
Signature · 1 line
2.25.1
#98 lore raw

PATCH v5 18/26 common/mlx5: optimize cache list object memory

In reply to Suanming Mou

Currently, hash list uses the cache list as bucket list. The list in the buckets have the same name, ctx and callbacks. This wastes the memory.

This commit abstracts all the name, ctx and callback members in the list to a constant struct and others to the inconstant struct, uses the wrapper functions to satisfy both hash list and cache list can set the list constant and inconstant struct individually.

---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +171 -124
index f75b1cb0da..858c8d8164 100644
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +30 -15

69 lines not on this page · raw message

Signature · 1 line
2.25.1
#99 lore raw

PATCH v5 19/26 common/mlx5: support list non-lcore operations

In reply to Suanming Mou

This commit supports the list non-lcore operations with an extra sub-list and lock.

---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +63 -29
index 858c8d8164..c8b48f3afc 100644
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.lcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.lcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
+	return ret;
 }
 
 void
drivers/common/mlx5/mlx5_common_utils.h +8 -1
index 5718a21be0..613d29de0c 100644
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t lcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
Signature · 1 line
2.25.1
#100 lore raw

PATCH v5 20/26 net/mlx5: move modify header allocator to ipool

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Modify header actions are allocated by mlx5_malloc which has a big overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header handler.

---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.c +4 -0
index fd675c9262..640d36c6be 100644
@@ -802,6 +802,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -815,6 +816,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
drivers/net/mlx5/mlx5.h +14 -0
index e9b08094a6..e7e4749824 100644
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1146,6 +1159,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
drivers/net/mlx5/mlx5_flow.h +1 -13
index e3a29297ba..7027012220 100644
@@ -505,23 +505,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
drivers/net/mlx5/mlx5_flow_dv.c +67 -12

138 lines not on this page · raw message

Signature · 1 line
2.25.1
#101 lore raw

PATCH v5 21/26 net/mlx5: adjust the hash bucket size

In reply to Suanming Mou

With the new per core optimization to the list, the hash bucket size can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index b04e5f8ba6..969d265350 100644
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
drivers/net/mlx5/mlx5.c +1 -1
index 640d36c6be..3b5c7840a8 100644
@@ -376,7 +376,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
drivers/net/mlx5/mlx5_defs.h +3 -3
index 906aa43c5a..ca67ce8213 100644
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
drivers/net/mlx5/mlx5_flow.c +2 -3
index 45e90bb245..1c5d4c283a 100644
@@ -8881,7 +8881,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8990,8 +8990,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
Signature · 1 line
2.25.1
#102 lore raw

PATCH v5 22/26 net/mlx5: enable index pool per-core cache

In reply to Suanming Mou

This commit enables the tag and header modify action index pool per-core cache in non-reclaim memory mode.

---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5.c +3 -1
index 3b5c7840a8..f79a094a35 100644
@@ -217,7 +217,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1129,6 +1130,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
drivers/net/mlx5/mlx5.h +1 -0
index e7e4749824..30893c7f38 100644
@@ -1116,6 +1116,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index e39fe43854..739c387c0a 100644
@@ -5361,7 +5361,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
Signature · 1 line
2.25.1
#103 lore raw

PATCH v5 23/26 net/mlx5: optimize hash list table allocate on demand

In reply to Suanming Mou

Currently, all the hash list tables are allocated during start up. Since different applications may only use dedicated limited actions, optimized the hash list table allocate on demand will save initial memory.

This commit optimizes hash list table allocate on demand.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -43
index 969d265350..ec2f8a4aed 100644
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
drivers/net/mlx5/mlx5_defs.h +6 -0
index ca67ce8213..fe86bb40d3 100644
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
drivers/net/mlx5/mlx5_flow_dv.c +75 -4

119 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +0 -2
index a04f93e1d4..5da362a9d5 100644
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
Signature · 1 line
2.25.1
#104 lore raw

PATCH v5 24/26 net/mlx5: change memory release configuration

In reply to Suanming Mou

This commit changes the index pool memory release configuration to 0 when memory reclaim mode is not required.

---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/mlx5/mlx5.c +2 -0
index f79a094a35..021a34dd4d 100644
@@ -798,6 +798,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
Signature · 1 line
2.25.1
#105 lore raw

PATCH v5 25/26 net/mlx5: optimize Rx queue match

In reply to Suanming Mou

As hrxq struct has the indirect table pointer, while matching the hrxq, better to use the hrxq indirect table instead of searching from the list.

This commit optimizes the hrxq indirect table matching.

---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5_rxq.c +7 -11
index 7893b3edd4..23685d7654 100644
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
Signature · 1 line
2.25.1
#106 lore raw

PATCH v5 26/26 doc: add mlx5 multiple-thread flow insertion optimization

In reply to Suanming Mou

This commit adds the multiple-thread flow insertion optimization description.

---
 doc/guides/nics/mlx5.rst               | 5 +++++
 doc/guides/rel_notes/release_21_08.rst | 1 +
 2 files changed, 6 insertions(+)
doc/guides/nics/mlx5.rst +5 -0
index 5a3c4f198a..199beb2549 100644
@@ -448,6 +448,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
doc/guides/rel_notes/release_21_08.rst +1 -0
index 6a902ef9ac..68496496e8 100644
@@ -88,6 +88,7 @@ New Features
 
   * Added support for meter hierarchy.
   * Added devargs options ``allow_duplicate_pattern``.
+  * Optimize multiple-thread flow insertion rate.
 
 * **Added support for Marvell CNXK crypto driver.**
 
Signature · 1 line
2.25.1
#107 lore raw

v6 cover net/mlx5: insertion rate optimization

In reply to Suanming Mou

This patch series optimize the flow insertion rate with adding local cache to index pool and list.

For object which wants efficient index allocate and free, local cache will be very helpful.

For index pool, two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

For list, sub local core list is introduced. The allocated objects will be added and released only from local list without any locks. Only the objects need to be shared will be synced from global list.

---

v6: fix compilation issue in "common/mlx5: add list lcore share" patch.

v5: code rebase to the latest version.

v4:
- split the list utility to commom patch
- fix one ipool crash issue in mlx5_ipool_get_cache() function.
- reorganize the common and mlx5 patches.
- split the doc update to an individual patch.
- update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
net/mlx5: optimize modify header action memory net/mlx5: remove cache term from the list utility net/mlx5: add per lcore cache to the list utility net/mlx5: minimize list critical sections net/mlx5: manage list cache entries release net/mlx5: relax the list utility atomic operations net/mlx5: allocate list memory by the create API common/mlx5: add per-lcore cache to hash list utility net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
net/mlx5: allow limiting the index pool maximum index net/mlx5: add indexed pool local cache
net/mlx5: add index pool foreach define net/mlx5: support index pool non-lcore operations net/mlx5: replace flow list with index pool common/mlx5: move list utility to common common/mlx5: add list lcore share
common/mlx5: call list callbacks with context common/mlx5: allocate cache list memory individually common/mlx5: optimize cache list object memory common/mlx5: support list non-lcore operations net/mlx5: adjust the hash bucket size
net/mlx5: enable index pool per-core cache net/mlx5: optimize hash list table allocate on demand net/mlx5: change memory release configuration net/mlx5: optimize Rx queue match
doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    1 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
 drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  308 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1206 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2543 insertions(+), 1414 deletions(-)
Signature · 1 line
2.25.1
#108 lore raw

PATCH v6 1/26 net/mlx5: allow limiting the index pool maximum index

Some ipool instances in the driver are used as ID\index allocator and added other logic in order to work with limited index values.

Add a new configuration for ipool specify the maximum index value. The ipool will ensure that no index bigger than the maximum value is provided.

Use this configuration in ID allocator cases instead of the current logics. This patch add the maximum ID configurable for the index pool.

---
 drivers/net/mlx5/mlx5_utils.c | 14 ++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  1 +
 2 files changed, 13 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5_utils.c +12 -2
index 18fe23e4fb..bf2b2ebc72 100644
@@ -270,6 +270,9 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		if (i > 0)
 			pool->grow_tbl[i] += pool->grow_tbl[i - 1];
 	}
+	if (!pool->cfg.max_idx)
+		pool->cfg.max_idx =
+			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	return pool;
 }
 
@@ -282,9 +285,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	size_t trunk_size = 0;
 	size_t data_size;
 	size_t bmp_size;
-	uint32_t idx;
+	uint32_t idx, cur_max_idx, i;
 
-	if (pool->n_trunk_valid == TRUNK_MAX_IDX)
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, pool->n_trunk_valid);
+	if (pool->n_trunk_valid == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
 		return -ENOMEM;
 	if (pool->n_trunk_valid == pool->n_trunk) {
 		/* No free trunk flags, expand trunk list. */
@@ -336,6 +341,11 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
 		     [RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
 		     bmp_size);
+	/* Clear the overhead bits in the trunk if it happens. */
+	if (cur_max_idx + data_size > pool->cfg.max_idx) {
+		for (i = pool->cfg.max_idx - cur_max_idx; i < data_size; i++)
+			rte_bitmap_clear(trunk->bmp, i);
+	}
 	MLX5_ASSERT(trunk->bmp);
 	pool->n_trunk_valid++;
 #ifdef POOL_DEBUG
drivers/net/mlx5/mlx5_utils.h +1 -0
index b54517c6df..15870e14c2 100644
@@ -208,6 +208,7 @@ struct mlx5_indexed_pool_config {
 	uint32_t need_lock:1;
 	/* Lock is needed for multiple thread usage. */
 	uint32_t release_mem_en:1; /* Rlease trunk when it is free. */
+	uint32_t max_idx; /* The maximum index can be allocated. */
 	const char *type; /* Memory allocate type name. */
 	void *(*malloc)(uint32_t flags, size_t size, unsigned int align,
 			int socket);
Signature · 1 line
2.25.1
#109 lore raw

PATCH v6 2/26 net/mlx5: add indexed pool local cache

In reply to Suanming Mou

For object which wants efficient index allocate and free, local cache will be very helpful.

Two level cache is introduced to allocate and free the index more efficient. One as local and the other as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk with more index.

This commit adds new local cache mechanism for indexed pool.

---
 drivers/net/mlx5/mlx5_utils.c | 323 ++++++++++++++++++++++++++++++++--
 drivers/net/mlx5/mlx5_utils.h |  64 ++++++-
 2 files changed, 372 insertions(+), 15 deletions(-)
drivers/net/mlx5/mlx5_utils.c +313 -10
index bf2b2ebc72..215024632d 100644
@@ -175,14 +175,14 @@ static inline void
 mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_lock(&pool->lock);
+		rte_spinlock_lock(&pool->rsz_lock);
 }
 
 static inline void
 mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
 {
 	if (pool->cfg.need_lock)
-		rte_spinlock_unlock(&pool->lock);
+		rte_spinlock_unlock(&pool->rsz_lock);
 }
 
 static inline uint32_t
@@ -243,6 +243,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	uint32_t i;
 
 	if (!cfg || (!cfg->malloc ^ !cfg->free) ||
+	    (cfg->per_core_cache && cfg->release_mem_en) ||
 	    (cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
 	    ((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
 		return NULL;
@@ -258,9 +259,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 		pool->cfg.malloc = mlx5_malloc;
 		pool->cfg.free = mlx5_free;
 	}
-	pool->free_list = TRUNK_INVALID;
 	if (pool->cfg.need_lock)
-		rte_spinlock_init(&pool->lock);
+		rte_spinlock_init(&pool->rsz_lock);
 	/*
 	 * Initialize the dynamic grow trunk size lookup table to have a quick
 	 * lookup for the trunk entry index offset.
@@ -273,6 +273,8 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 	if (!pool->cfg.max_idx)
 		pool->cfg.max_idx =
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
+	if (!cfg->per_core_cache)
+		pool->free_list = TRUNK_INVALID;
 	return pool;
 }
 
@@ -355,6 +357,274 @@ mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+static inline struct mlx5_indexed_cache *
+mlx5_ipool_update_global_cache(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_cache *gc, *lc, *olc = NULL;
+
+	lc = pool->cache[cidx]->lc;
+	gc = __atomic_load_n(&pool->gc, __ATOMIC_RELAXED);
+	if (gc && lc != gc) {
+		mlx5_ipool_lock(pool);
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = pool->gc;
+		lc->ref_cnt++;
+		pool->cache[cidx]->lc = lc;
+		mlx5_ipool_unlock(pool);
+		if (olc)
+			pool->cfg.free(olc);
+	}
+	return lc;
+}
+
+static uint32_t
+mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *p, *lc, *olc = NULL;
+	size_t trunk_size = 0;
+	size_t data_size;
+	uint32_t cur_max_idx, trunk_idx, trunk_n;
+	uint32_t fetch_size, ts_idx, i;
+	int n_grow;
+
+check_again:
+	p = NULL;
+	fetch_size = 0;
+	/*
+	 * Fetch new index from global if possible. First round local
+	 * cache will be NULL.
+	 */
+	lc = pool->cache[cidx]->lc;
+	mlx5_ipool_lock(pool);
+	/* Try to update local cache first. */
+	if (likely(pool->gc)) {
+		if (lc != pool->gc) {
+			if (lc && !(--lc->ref_cnt))
+				olc = lc;
+			lc = pool->gc;
+			lc->ref_cnt++;
+			pool->cache[cidx]->lc = lc;
+		}
+		if (lc->len) {
+			/* Use the updated local cache to fetch index. */
+			fetch_size = pool->cfg.per_core_cache >> 2;
+			if (lc->len < fetch_size)
+				fetch_size = lc->len;
+			lc->len -= fetch_size;
+			memcpy(pool->cache[cidx]->idx, &lc->idx[lc->len],
+			       sizeof(uint32_t) * fetch_size);
+		}
+	}
+	mlx5_ipool_unlock(pool);
+	if (unlikely(olc)) {
+		pool->cfg.free(olc);
+		olc = NULL;
+	}
+	if (fetch_size) {
+		pool->cache[cidx]->len = fetch_size - 1;
+		return pool->cache[cidx]->idx[pool->cache[cidx]->len];
+	}
+	trunk_idx = lc ? __atomic_load_n(&lc->n_trunk_valid,
+			 __ATOMIC_ACQUIRE) : 0;
+	trunk_n = lc ? lc->n_trunk : 0;
+	cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	/* Check if index reach maximum. */
+	if (trunk_idx == TRUNK_MAX_IDX ||
+	    cur_max_idx >= pool->cfg.max_idx)
+		return 0;
+	/* No enough space in trunk array, resize the trunks array. */
+	if (trunk_idx == trunk_n) {
+		n_grow = trunk_idx ? trunk_idx :
+			     RTE_CACHE_LINE_SIZE / sizeof(void *);
+		cur_max_idx = mlx5_trunk_idx_offset_get(pool, trunk_n + n_grow);
+		/* Resize the trunk array. */
+		p = pool->cfg.malloc(0, ((trunk_idx + n_grow) *
+			sizeof(struct mlx5_indexed_trunk *)) +
+			(cur_max_idx * sizeof(uint32_t)) + sizeof(*p),
+			RTE_CACHE_LINE_SIZE, rte_socket_id());
+		if (!p)
+			return 0;
+		p->trunks = (struct mlx5_indexed_trunk **)&p->idx[cur_max_idx];
+		if (lc)
+			memcpy(p->trunks, lc->trunks, trunk_idx *
+		       sizeof(struct mlx5_indexed_trunk *));
+#ifdef RTE_LIBRTE_MLX5_DEBUG
+		memset(RTE_PTR_ADD(p->trunks, trunk_idx * sizeof(void *)), 0,
+			n_grow * sizeof(void *));
+#endif
+		p->n_trunk_valid = trunk_idx;
+		p->n_trunk = trunk_n + n_grow;
+		p->len = 0;
+	}
+	/* Prepare the new trunk. */
+	trunk_size = sizeof(*trunk);
+	data_size = mlx5_trunk_size_get(pool, trunk_idx);
+	trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
+	trunk = pool->cfg.malloc(0, trunk_size,
+				 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (unlikely(!trunk)) {
+		pool->cfg.free(p);
+		return 0;
+	}
+	trunk->idx = trunk_idx;
+	trunk->free = data_size;
+	mlx5_ipool_lock(pool);
+	/*
+	 * Double check if trunks has been updated or have available index.
+	 * During the new trunk allocate, index may still be flushed to the
+	 * global cache. So also need to check the pool->gc->len.
+	 */
+	if (pool->gc && (lc != pool->gc ||
+	    lc->n_trunk_valid != trunk_idx ||
+	    pool->gc->len)) {
+		mlx5_ipool_unlock(pool);
+		if (p)
+			pool->cfg.free(p);
+		pool->cfg.free(trunk);
+		goto check_again;
+	}
+	/* Resize the trunk array and update local cache first.  */
+	if (p) {
+		if (lc && !(--lc->ref_cnt))
+			olc = lc;
+		lc = p;
+		lc->ref_cnt = 1;
+		pool->cache[cidx]->lc = lc;
+		__atomic_store_n(&pool->gc, p, __ATOMIC_RELAXED);
+	}
+	/* Add trunk to trunks array. */
+	lc->trunks[trunk_idx] = trunk;
+	__atomic_fetch_add(&lc->n_trunk_valid, 1, __ATOMIC_RELAXED);
+	/* Enqueue half of the index to global. */
+	ts_idx = mlx5_trunk_idx_offset_get(pool, trunk_idx) + 1;
+	fetch_size = trunk->free >> 1;
+	for (i = 0; i < fetch_size; i++)
+		lc->idx[i] = ts_idx + i;
+	lc->len = fetch_size;
+	mlx5_ipool_unlock(pool);
+	/* Copy left half - 1 to local cache index array. */
+	pool->cache[cidx]->len = trunk->free - fetch_size - 1;
+	ts_idx += fetch_size;
+	for (i = 0; i < pool->cache[cidx]->len; i++)
+		pool->cache[cidx]->idx[i] = ts_idx + i;
+	if (olc)
+		pool->cfg.free(olc);
+	return ts_idx + i;
+}
+
+static void *
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	struct mlx5_indexed_trunk *trunk;
+	struct mlx5_indexed_cache *lc;
+	uint32_t trunk_idx;
+	uint32_t entry_idx;
+	int cidx;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	lc = mlx5_ipool_update_global_cache(pool, cidx);
+	idx -= 1;
+	trunk_idx = mlx5_trunk_idx_get(pool, idx);
+	trunk = lc->trunks[trunk_idx];
+	MLX5_ASSERT(trunk);
+	entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk_idx);
+	return &trunk->data[entry_idx * pool->cfg.size];
+}
+
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return NULL;
+	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	} else if (pool->cache[cidx]->len) {
+		pool->cache[cidx]->len--;
+		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
+		return mlx5_ipool_get_cache(pool, *idx);
+	}
+	/* Not enough idx in global cache. Keep fetching from global. */
+	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
+	if (unlikely(!(*idx)))
+		return NULL;
+	return mlx5_ipool_get_cache(pool, *idx);
+}
+
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+	struct mlx5_ipool_per_lcore *ilc;
+	struct mlx5_indexed_cache *gc, *olc = NULL;
+	uint32_t reclaim_num = 0;
+
+	MLX5_ASSERT(idx);
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		rte_errno = ENOTSUP;
+		return;
+	}
+	/*
+	 * When index was allocated on core A but freed on core B. In this
+	 * case check if local cache on core B was allocated before.
+	 */
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return;
+		}
+	}
+	/* Try to enqueue to local index cache. */
+	if (pool->cache[cidx]->len < pool->cfg.per_core_cache) {
+		pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+		pool->cache[cidx]->len++;
+		return;
+	}
+	ilc = pool->cache[cidx];
+	reclaim_num = pool->cfg.per_core_cache >> 2;
+	ilc->len -= reclaim_num;
+	/* Local index cache full, try with global index cache. */
+	mlx5_ipool_lock(pool);
+	gc = pool->gc;
+	if (ilc->lc != gc) {
+		if (!(--ilc->lc->ref_cnt))
+			olc = ilc->lc;
+		gc->ref_cnt++;
+		ilc->lc = gc;
+	}
+	memcpy(&gc->idx[gc->len], &ilc->idx[ilc->len],
+	       reclaim_num * sizeof(uint32_t));
+	gc->len += reclaim_num;
+	mlx5_ipool_unlock(pool);
+	if (olc)
+		pool->cfg.free(olc);
+	pool->cache[cidx]->idx[pool->cache[cidx]->len] = idx;
+	pool->cache[cidx]->len++;
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -363,6 +633,8 @@ mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	uint32_t iidx = 0;
 	void *p;
 
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_malloc_cache(pool, idx);
 	mlx5_ipool_lock(pool);
 	if (pool->free_list == TRUNK_INVALID) {
 		/* If no available trunks, grow new. */
@@ -432,6 +704,10 @@ mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return;
+	if (pool->cfg.per_core_cache) {
+		mlx5_ipool_free_cache(pool, idx);
+		return;
+	}
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -497,6 +773,8 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 
 	if (!idx)
 		return NULL;
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_cache(pool, idx);
 	idx -= 1;
 	mlx5_ipool_lock(pool);
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -519,18 +797,43 @@ mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
 int
 mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 {
-	struct mlx5_indexed_trunk **trunks;
-	uint32_t i;
+	struct mlx5_indexed_trunk **trunks = NULL;
+	struct mlx5_indexed_cache *gc = pool->gc;
+	uint32_t i, n_trunk_valid = 0;
 
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
-	trunks = pool->trunks;
-	for (i = 0; i < pool->n_trunk; i++) {
+	if (pool->cfg.per_core_cache) {
+		for (i = 0; i < RTE_MAX_LCORE; i++) {
+			/*
+			 * Free only old global cache. Pool gc will be
+			 * freed at last.
+			 */
+			if (pool->cache[i]) {
+				if (pool->cache[i]->lc &&
+				    pool->cache[i]->lc != pool->gc &&
+				    (!(--pool->cache[i]->lc->ref_cnt)))
+					pool->cfg.free(pool->cache[i]->lc);
+				pool->cfg.free(pool->cache[i]);
+			}
+		}
+		if (gc) {
+			trunks = gc->trunks;
+			n_trunk_valid = gc->n_trunk_valid;
+		}
+	} else {
+		gc = NULL;
+		trunks = pool->trunks;
+		n_trunk_valid = pool->n_trunk_valid;
+	}
+	for (i = 0; i < n_trunk_valid; i++) {
 		if (trunks[i])
 			pool->cfg.free(trunks[i]);
 	}
-	if (!pool->trunks)
-		pool->cfg.free(pool->trunks);
+	if (!gc && trunks)
+		pool->cfg.free(trunks);
+	if (gc)
+		pool->cfg.free(gc);
 	mlx5_ipool_unlock(pool);
 	mlx5_free(pool);
 	return 0;
drivers/net/mlx5/mlx5_utils.h +59 -5

85 lines not on this page · raw message

Signature · 1 line
2.25.1
#110 lore raw

PATCH v6 3/26 net/mlx5: add index pool foreach define

In reply to Suanming Mou

In some cases, application may want to know all the allocated index in order to apply some operations to the allocated index.

This commit adds the indexed pool functions to support foreach operation.

---
 drivers/net/mlx5/mlx5_utils.c | 96 +++++++++++++++++++++++++++++++++++
 drivers/net/mlx5/mlx5_utils.h |  8 +++
 2 files changed, 104 insertions(+)
drivers/net/mlx5/mlx5_utils.c +96 -0
index 215024632d..32f8d65073 100644
@@ -529,6 +529,16 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!pool->cache[cidx])) {
+		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
+			sizeof(struct mlx5_ipool_per_lcore) +
+			(pool->cfg.per_core_cache * sizeof(uint32_t)),
+			RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!pool->cache[cidx]) {
+			DRV_LOG(ERR, "Ipool cache%d allocate failed\n", cidx);
+			return NULL;
+		}
+	}
 	lc = mlx5_ipool_update_global_cache(pool, cidx);
 	idx -= 1;
 	trunk_idx = mlx5_trunk_idx_get(pool, idx);
@@ -839,6 +849,92 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	return 0;
 }
 
+void
+mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
+{
+	uint32_t i, j;
+	struct mlx5_indexed_cache *gc;
+	struct rte_bitmap *ibmp;
+	uint32_t bmp_num, mem_size;
+
+	if (!pool->cfg.per_core_cache)
+		return;
+	gc = pool->gc;
+	if (!gc)
+		return;
+	/* Reset bmp. */
+	bmp_num = mlx5_trunk_idx_offset_get(pool, gc->n_trunk_valid);
+	mem_size = rte_bitmap_get_memory_footprint(bmp_num);
+	pool->bmp_mem = pool->cfg.malloc(MLX5_MEM_ZERO, mem_size,
+					 RTE_CACHE_LINE_SIZE, rte_socket_id());
+	if (!pool->bmp_mem) {
+		DRV_LOG(ERR, "Ipool bitmap mem allocate failed.\n");
+		return;
+	}
+	ibmp = rte_bitmap_init_with_all_set(bmp_num, pool->bmp_mem, mem_size);
+	if (!ibmp) {
+		pool->cfg.free(pool->bmp_mem);
+		pool->bmp_mem = NULL;
+		DRV_LOG(ERR, "Ipool bitmap create failed.\n");
+		return;
+	}
+	pool->ibmp = ibmp;
+	/* Clear global cache. */
+	for (i = 0; i < gc->len; i++)
+		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
+	/* Clear core cache. */
+	for (i = 0; i < RTE_MAX_LCORE; i++) {
+		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
+
+		if (!ilc)
+			continue;
+		for (j = 0; j < ilc->len; j++)
+			rte_bitmap_clear(ibmp, ilc->idx[j] - 1);
+	}
+}
+
+static void *
+mlx5_ipool_get_next_cache(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	struct rte_bitmap *ibmp;
+	uint64_t slab = 0;
+	uint32_t iidx = *pos;
+
+	ibmp = pool->ibmp;
+	if (!ibmp || !rte_bitmap_scan(ibmp, &iidx, &slab)) {
+		if (pool->bmp_mem) {
+			pool->cfg.free(pool->bmp_mem);
+			pool->bmp_mem = NULL;
+			pool->ibmp = NULL;
+		}
+		return NULL;
+	}
+	iidx += __builtin_ctzll(slab);
+	rte_bitmap_clear(ibmp, iidx);
+	iidx++;
+	*pos = iidx;
+	return mlx5_ipool_get_cache(pool, iidx);
+}
+
+void *
+mlx5_ipool_get_next(struct mlx5_indexed_pool *pool, uint32_t *pos)
+{
+	uint32_t idx = *pos;
+	void *entry;
+
+	if (pool->cfg.per_core_cache)
+		return mlx5_ipool_get_next_cache(pool, pos);
+	while (idx <= mlx5_trunk_idx_offset_get(pool, pool->n_trunk)) {
+		entry = mlx5_ipool_get(pool, idx);
+		if (entry) {
+			*pos = idx;
+			return entry;
+		}
+		idx++;
+	}
+	return NULL;
+}
+
 void
 mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
 {
drivers/net/mlx5/mlx5_utils.h +8 -0
index 0469062695..737dd7052d 100644
@@ -261,6 +261,9 @@ struct mlx5_indexed_pool {
 			/* Global cache. */
 			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
 			/* Local cache. */
+			struct rte_bitmap *ibmp;
+			void *bmp_mem;
+			/* Allocate objects bitmap. Use during flush. */
 		};
 	};
 #ifdef POOL_DEBUG
@@ -862,4 +865,9 @@ struct {								\
 	     (entry);							\
 	     idx++, (entry) = mlx5_l3t_get_next((tbl), &idx))
 
+#define MLX5_IPOOL_FOREACH(ipool, idx, entry)				\
+	for ((idx) = 0, mlx5_ipool_flush_cache((ipool)),		\
+	    (entry) = mlx5_ipool_get_next((ipool), &idx);		\
+	    (entry); idx++, (entry) = mlx5_ipool_get_next((ipool), &idx))
+
 #endif /* RTE_PMD_MLX5_UTILS_H_ */
Signature · 1 line
2.25.1
#111 lore raw

PATCH v6 4/26 net/mlx5: support index pool non-lcore operations

In reply to Suanming Mou

This commit supports the index pool non-lcore operations with an extra cache and lcore lock.

---
 drivers/net/mlx5/mlx5_utils.c | 75 +++++++++++++++++++++++++----------
 drivers/net/mlx5/mlx5_utils.h |  3 +-
 2 files changed, 56 insertions(+), 22 deletions(-)
drivers/net/mlx5/mlx5_utils.c +54 -21
index 32f8d65073..f9557c09ff 100644
@@ -275,6 +275,7 @@ mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
 			mlx5_trunk_idx_offset_get(pool, TRUNK_MAX_IDX + 1);
 	if (!cfg->per_core_cache)
 		pool->free_list = TRUNK_INVALID;
+	rte_spinlock_init(&pool->lcore_lock);
 	return pool;
 }
 
@@ -515,20 +516,14 @@ mlx5_ipool_allocate_from_global(struct mlx5_indexed_pool *pool, int cidx)
 }
 
 static void *
-mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+_mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
 {
 	struct mlx5_indexed_trunk *trunk;
 	struct mlx5_indexed_cache *lc;
 	uint32_t trunk_idx;
 	uint32_t entry_idx;
-	int cidx;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -549,15 +544,27 @@ mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 }
 
 static void *
-mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
+mlx5_ipool_get_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 {
+	void *entry;
 	int cidx;
 
 	cidx = rte_lcore_index(rte_lcore_id());
 	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
 	}
+	entry = _mlx5_ipool_get_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+
+static void *
+_mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, int cidx,
+			 uint32_t *idx)
+{
 	if (unlikely(!pool->cache[cidx])) {
 		pool->cache[cidx] = pool->cfg.malloc(MLX5_MEM_ZERO,
 			sizeof(struct mlx5_ipool_per_lcore) +
@@ -570,29 +577,40 @@ mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 	} else if (pool->cache[cidx]->len) {
 		pool->cache[cidx]->len--;
 		*idx = pool->cache[cidx]->idx[pool->cache[cidx]->len];
-		return mlx5_ipool_get_cache(pool, *idx);
+		return _mlx5_ipool_get_cache(pool, cidx, *idx);
 	}
 	/* Not enough idx in global cache. Keep fetching from global. */
 	*idx = mlx5_ipool_allocate_from_global(pool, cidx);
 	if (unlikely(!(*idx)))
 		return NULL;
-	return mlx5_ipool_get_cache(pool, *idx);
+	return _mlx5_ipool_get_cache(pool, cidx, *idx);
 }
 
-static void
-mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+static void *
+mlx5_ipool_malloc_cache(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
+	void *entry;
 	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	entry = _mlx5_ipool_malloc_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+	return entry;
+}
+
+static void
+_mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, int cidx, uint32_t idx)
+{
 	struct mlx5_ipool_per_lcore *ilc;
 	struct mlx5_indexed_cache *gc, *olc = NULL;
 	uint32_t reclaim_num = 0;
 
 	MLX5_ASSERT(idx);
-	cidx = rte_lcore_index(rte_lcore_id());
-	if (unlikely(cidx == -1)) {
-		rte_errno = ENOTSUP;
-		return;
-	}
 	/*
 	 * When index was allocated on core A but freed on core B. In this
 	 * case check if local cache on core B was allocated before.
@@ -635,6 +653,21 @@ mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
 	pool->cache[cidx]->len++;
 }
 
+static void
+mlx5_ipool_free_cache(struct mlx5_indexed_pool *pool, uint32_t idx)
+{
+	int cidx;
+
+	cidx = rte_lcore_index(rte_lcore_id());
+	if (unlikely(cidx == -1)) {
+		cidx = RTE_MAX_LCORE;
+		rte_spinlock_lock(&pool->lcore_lock);
+	}
+	_mlx5_ipool_free_cache(pool, cidx, idx);
+	if (unlikely(cidx == RTE_MAX_LCORE))
+		rte_spinlock_unlock(&pool->lcore_lock);
+}
+
 void *
 mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
 {
@@ -814,7 +847,7 @@ mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
 	MLX5_ASSERT(pool);
 	mlx5_ipool_lock(pool);
 	if (pool->cfg.per_core_cache) {
-		for (i = 0; i < RTE_MAX_LCORE; i++) {
+		for (i = 0; i <= RTE_MAX_LCORE; i++) {
 			/*
 			 * Free only old global cache. Pool gc will be
 			 * freed at last.
@@ -883,7 +916,7 @@ mlx5_ipool_flush_cache(struct mlx5_indexed_pool *pool)
 	for (i = 0; i < gc->len; i++)
 		rte_bitmap_clear(ibmp, gc->idx[i] - 1);
 	/* Clear core cache. */
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < RTE_MAX_LCORE + 1; i++) {
 		struct mlx5_ipool_per_lcore *ilc = pool->cache[i];
 
 		if (!ilc)
drivers/net/mlx5/mlx5_utils.h +2 -1
index 737dd7052d..a509b0a4eb 100644
@@ -248,6 +248,7 @@ struct mlx5_ipool_per_lcore {
 struct mlx5_indexed_pool {
 	struct mlx5_indexed_pool_config cfg; /* Indexed pool configuration. */
 	rte_spinlock_t rsz_lock; /* Pool lock for multiple thread usage. */
+	rte_spinlock_t lcore_lock;
 	/* Dim of trunk pointer array. */
 	union {
 		struct {
@@ -259,7 +260,7 @@ struct mlx5_indexed_pool {
 		struct {
 			struct mlx5_indexed_cache *gc;
 			/* Global cache. */
-			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE];
+			struct mlx5_ipool_per_lcore *cache[RTE_MAX_LCORE + 1];
 			/* Local cache. */
 			struct rte_bitmap *ibmp;
 			void *bmp_mem;
Signature · 1 line
2.25.1
#112 lore raw

PATCH v6 5/26 net/mlx5: replace flow list with index pool

In reply to Suanming Mou

The flow list is used to save the create flows and to be used only when port closes all the flows need to be flushed.

This commit takes advantage of the index pool foreach operation to flush all the allocated flows.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  48 ++++++++-
 drivers/net/mlx5/mlx5.c            |   9 +-
 drivers/net/mlx5/mlx5.h            |  14 ++-
 drivers/net/mlx5/mlx5_flow.c       | 152 ++++++++++-------------------
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |   5 +
 drivers/net/mlx5/mlx5_trigger.c    |   8 +-
 drivers/net/mlx5/windows/mlx5_os.c |   1 -
 8 files changed, 127 insertions(+), 112 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +47 -1
index be22d9cbd2..867f450cab 100644
@@ -69,6 +69,44 @@ static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
 /* Process local data for secondary processes. */
 static struct mlx5_local_data mlx5_local_data;
 
+/* rte flow indexed pool configuration. */
+static struct mlx5_indexed_pool_config icfg[] = {
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "ctl_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 1 << 14,
+		.type = "rte_flow_ipool",
+	},
+	{
+		.size = sizeof(struct rte_flow),
+		.trunk_size = 64,
+		.grow_trunk = 3,
+		.grow_shift = 2,
+		.need_lock = 1,
+		.release_mem_en = 0,
+		.malloc = mlx5_malloc,
+		.free = mlx5_free,
+		.per_core_cache = 0,
+		.type = "mcp_flow_ipool",
+	},
+};
+
 /**
  * Set the completion channel file descriptor interrupt as non-blocking.
  *
@@ -832,6 +870,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	int own_domain_id = 0;
 	uint16_t port_id;
 	struct mlx5_port_info vport_info = { .query_flags = 0 };
+	int i;
 
 	/* Determine if this port representor is supposed to be spawned. */
 	if (switch_info->representor && dpdk_dev->devargs &&
@@ -1575,7 +1614,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 				      mlx5_ifindex(eth_dev),
 				      eth_dev->data->mac_addrs,
 				      MLX5_MAX_MAC_ADDRESSES);
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	rte_spinlock_init(&priv->flow_list_lock);
 	TAILQ_INIT(&priv->flow_meters);
@@ -1611,6 +1649,14 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	mlx5_set_min_inline(spawn, config);
 	/* Store device configuration on private structure. */
 	priv->config = *config;
+	for (i = 0; i < MLX5_FLOW_TYPE_MAXI; i++) {
+		icfg[i].release_mem_en = !!config->reclaim_mode;
+		if (config->reclaim_mode)
+			icfg[i].per_core_cache = 0;
+		priv->flows[i] = mlx5_ipool_create(&icfg[i]);
+		if (!priv->flows[i])
+			goto error;
+	}
 	/* Create context for virtual machine VLAN workaround. */
 	priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex);
 	if (config->dv_flow_en) {
drivers/net/mlx5/mlx5.c +6 -3
index 44fbc2da83..b5c05b12e8 100644
@@ -325,7 +325,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = 1 << 19,
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_flow_handle_ipool",
@@ -793,8 +794,10 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 				MLX5_FLOW_HANDLE_VERBS_SIZE;
 			break;
 		}
-		if (config->reclaim_mode)
+		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
+			cfg.per_core_cache = 0;
+		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
 }
@@ -1529,7 +1532,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	 * If all the flows are already flushed in the device stop stage,
 	 * then this will return directly without any action.
 	 */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_action_handle_flush(dev);
 	mlx5_flow_meter_flush(dev, NULL);
 	/* Prevent crashes when queues are still in use. */
drivers/net/mlx5/mlx5.h +12 -2
index f864c1d701..facab913bf 100644
@@ -71,6 +71,14 @@ enum mlx5_reclaim_mem_mode {
 	MLX5_RCM_AGGR, /* Reclaim PMD and rdma-core level. */
 };
 
+/* The type of flow. */
+enum mlx5_flow_type {
+	MLX5_FLOW_TYPE_CTL, /* Control flow. */
+	MLX5_FLOW_TYPE_GEN, /* General flow. */
+	MLX5_FLOW_TYPE_MCP, /* MCP flow. */
+	MLX5_FLOW_TYPE_MAXI,
+};
+
 /* Hash and cache list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
@@ -1367,7 +1375,8 @@ struct mlx5_priv {
 	unsigned int (*reta_idx)[]; /* RETA index table. */
 	unsigned int reta_idx_n; /* RETA index size. */
 	struct mlx5_drop drop_queue; /* Flow drop queues. */
-	uint32_t flows; /* RTE Flow rules. */
+	struct mlx5_indexed_pool *flows[MLX5_FLOW_TYPE_MAXI];
+	/* RTE Flow rules. */
 	uint32_t ctrl_flows; /* Control flow rules. */
 	rte_spinlock_t flow_list_lock;
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
@@ -1627,7 +1636,8 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
 				  struct rte_flow_error *error);
 int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
 		      struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, enum mlx5_flow_type type,
+			  bool active);
 int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
 int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
 		    const struct rte_flow_action *action, void *data,
drivers/net/mlx5/mlx5_flow.c +53 -99

352 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +1 -1
index 7d97c5880f..adadcfa425 100644
@@ -998,9 +998,9 @@ flow_items_to_tunnel(const struct rte_flow_item items[])
 
 /* Flow structure. */
 struct rte_flow {
-	ILIST_ENTRY(uint32_t)next; /**< Index to the next flow structure. */
 	uint32_t dev_handles;
 	/**< Device flow handles that are part of the flow. */
+	uint32_t type:2;
 	uint32_t drv_type:2; /**< Driver type. */
 	uint32_t tunnel:1;
 	uint32_t meter:24; /**< Holds flow meter id. */
drivers/net/mlx5/mlx5_flow_dv.c +5 -0
index 2f4c0eeb5b..49bc85f7b3 100644
@@ -13901,6 +13901,11 @@ flow_dv_destroy(struct rte_eth_dev *dev, struct rte_flow *flow)
 		    dev_handle->split_flow_id)
 			mlx5_ipool_free(fm->flow_ipool,
 					dev_handle->split_flow_id);
+		else if (dev_handle->split_flow_id &&
+		    !dev_handle->is_meter_flow_id)
+			mlx5_ipool_free(priv->sh->ipool
+					[MLX5_IPOOL_RSS_EXPANTION_FLOW_ID],
+					dev_handle->split_flow_id);
 		mlx5_ipool_free(priv->sh->ipool[MLX5_IPOOL_MLX5_FLOW],
 			   tmp_idx);
 	}
drivers/net/mlx5/mlx5_trigger.c +3 -5
index ae7fcca229..7cb8920d6b 100644
@@ -1187,7 +1187,7 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
 	/* Control flows for default traffic can be removed firstly. */
 	mlx5_traffic_disable(dev);
 	/* All RX queue flags will be cleared in the flush interface. */
-	mlx5_flow_list_flush(dev, &priv->flows, true);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_GEN, true);
 	mlx5_flow_meter_rxq_flush(dev);
 	mlx5_rx_intr_vec_disable(dev);
 	priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
@@ -1370,7 +1370,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 	return 0;
 error:
 	ret = rte_errno; /* Save rte_errno before cleanup. */
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1385,9 +1385,7 @@ mlx5_traffic_enable(struct rte_eth_dev *dev)
 void
 mlx5_traffic_disable(struct rte_eth_dev *dev)
 {
-	struct mlx5_priv *priv = dev->data->dev_private;
-
-	mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
+	mlx5_flow_list_flush(dev, MLX5_FLOW_TYPE_CTL, false);
 }
 
 /**
drivers/net/mlx5/windows/mlx5_os.c +0 -1
index e30b682822..17716b66c9 100644
@@ -563,7 +563,6 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	eth_dev->rx_queue_count = mlx5_rx_queue_count;
 	/* Register MAC address. */
 	claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
-	priv->flows = 0;
 	priv->ctrl_flows = 0;
 	TAILQ_INIT(&priv->flow_meters);
 	priv->mtr_profile_tbl = mlx5_l3t_create(MLX5_L3T_TYPE_PTR);
Signature · 1 line
2.25.1
#113 lore raw

PATCH v6 6/26 net/mlx5: optimize modify header action memory

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Define the types of the modify header action fields to be with the minimum size needed for the optional values range.

---
 drivers/common/mlx5/linux/mlx5_glue.h |  1 +
 drivers/net/mlx5/linux/mlx5_flow_os.h |  3 ++-
 drivers/net/mlx5/mlx5_flow.h          |  6 +++---
 drivers/net/mlx5/mlx5_flow_dv.c       | 13 ++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)
drivers/common/mlx5/linux/mlx5_glue.h +1 -0
index 61f40d5478..f39ef2dac7 100644
@@ -78,6 +78,7 @@ struct mlx5dv_devx_async_cmd_hdr;
 enum  mlx5dv_dr_domain_type { unused, };
 struct mlx5dv_dr_domain;
 struct mlx5dv_dr_action;
+#define MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL 1
 #endif
 
 #ifndef HAVE_MLX5DV_DR_DEVX_PORT
drivers/net/mlx5/linux/mlx5_flow_os.h +2 -1
index cee685015b..1926d26410 100644
@@ -225,7 +225,8 @@ mlx5_flow_os_create_flow_action_modify_header(void *ctx, void *domain,
 			(struct mlx5_flow_dv_modify_hdr_resource *)resource;
 
 	*action = mlx5_glue->dv_create_flow_action_modify_header
-					(ctx, res->ft_type, domain, res->flags,
+					(ctx, res->ft_type, domain, res->root ?
+					 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL : 0,
 					 actions_len, (uint64_t *)res->actions);
 	return (*action) ? 0 : -1;
 }
drivers/net/mlx5/mlx5_flow.h +3 -3
index adadcfa425..f1ebc153f2 100644
@@ -524,11 +524,11 @@ struct mlx5_flow_dv_modify_hdr_resource {
 	void *action; /**< Modify header action object. */
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
-	uint32_t actions_num; /**< Number of modification actions. */
-	uint64_t flags; /**< Flags for RDMA API. */
+	uint8_t actions_num; /**< Number of modification actions. */
+	bool root; /**< Whether action is in root table. */
 	struct mlx5_modification_cmd actions[];
 	/**< Modification actions. */
-};
+} __rte_packed;
 
 /* Modify resource key of the hash organization. */
 union mlx5_flow_modify_hdr_key {
drivers/net/mlx5/mlx5_flow_dv.c +6 -7
index 49bc85f7b3..0f8b916991 100644
@@ -5006,21 +5006,21 @@ flow_dv_validate_action_port_id(struct rte_eth_dev *dev,
  *
  * @param dev
  *   Pointer to rte_eth_dev structure.
- * @param flags
- *   Flags bits to check if root level.
+ * @param root
+ *   Whether action is on root table.
  *
  * @return
  *   Max number of modify header actions device can support.
  */
 static inline unsigned int
 flow_dv_modify_hdr_action_max(struct rte_eth_dev *dev __rte_unused,
-			      uint64_t flags)
+			      bool root)
 {
 	/*
 	 * There's no way to directly query the max capacity from FW.
 	 * The maximal value on root table should be assumed to be supported.
 	 */
-	if (!(flags & MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL))
+	if (!root)
 		return MLX5_MAX_MODIFY_NUM;
 	else
 		return MLX5_ROOT_TBL_MODIFY_NUM;
@@ -5622,10 +5622,9 @@ flow_dv_modify_hdr_resource_register
 	};
 	uint64_t key64;
 
-	resource->flags = dev_flow->dv.group ? 0 :
-			  MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL;
+	resource->root = !dev_flow->dv.group;
 	if (resource->actions_num > flow_dv_modify_hdr_action_max(dev,
-				    resource->flags))
+								resource->root))
 		return rte_flow_error_set(error, EOVERFLOW,
 					  RTE_FLOW_ERROR_TYPE_ACTION, NULL,
 					  "too many modify header items");
Signature · 1 line
2.25.1
#114 lore raw

PATCH v6 7/26 net/mlx5: remove cache term from the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The internal mlx5 list tool is used mainly when the list objects need to be synchronized between multiple threads.

The "cache" term is used in the internal mlx5 list API.

Next enhancements on this tool will use the "cache" term for per thread cache management.

To prevent confusing, remove the current "cache" term from the API's names.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  32 +-
 drivers/net/mlx5/mlx5.c            |   2 +-
 drivers/net/mlx5/mlx5.h            |  15 +-
 drivers/net/mlx5/mlx5_flow.h       |  88 ++---
 drivers/net/mlx5/mlx5_flow_dv.c    | 561 ++++++++++++++---------------
 drivers/net/mlx5/mlx5_rx.h         |  12 +-
 drivers/net/mlx5/mlx5_rxq.c        |  28 +-
 drivers/net/mlx5/mlx5_utils.c      |  78 ++--
 drivers/net/mlx5/mlx5_utils.h      |  94 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   7 +-
 10 files changed, 456 insertions(+), 461 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +16 -16
index 867f450cab..5d537fa3e4 100644
@@ -272,27 +272,27 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action cache list. */
-	snprintf(s, sizeof(s), "%s_port_id_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->port_id_action_list, s, 0, sh,
+	/* Init port id action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
 			     flow_dv_port_id_create_cb,
 			     flow_dv_port_id_match_cb,
 			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action cache list. */
-	snprintf(s, sizeof(s), "%s_push_vlan_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->push_vlan_action_list, s, 0, sh,
+	/* Init push vlan action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
 			     flow_dv_push_vlan_create_cb,
 			     flow_dv_push_vlan_match_cb,
 			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action cache list. */
-	snprintf(s, sizeof(s), "%s_sample_action_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->sample_action_list, s, 0, sh,
+	/* Init sample action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
+	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
 			     flow_dv_sample_create_cb,
 			     flow_dv_sample_match_cb,
 			     flow_dv_sample_remove_cb);
-	/* Init dest array action cache list. */
-	snprintf(s, sizeof(s), "%s_dest_array_cache", sh->ibdev_name);
-	mlx5_cache_list_init(&sh->dest_array_list, s, 0, sh,
+	/* Init dest array action mlx5 list. */
+	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
+	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
 			     flow_dv_dest_array_create_cb,
 			     flow_dv_dest_array_match_cb,
 			     flow_dv_dest_array_remove_cb);
@@ -509,8 +509,8 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_cache_list_destroy(&sh->port_id_action_list);
-	mlx5_cache_list_destroy(&sh->push_vlan_action_list);
+	mlx5_list_destroy(&sh->port_id_action_list);
+	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
 }
 
@@ -1713,7 +1713,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
 			     mlx5_hrxq_create_cb,
 			     mlx5_hrxq_match_cb,
 			     mlx5_hrxq_remove_cb);
@@ -1774,7 +1774,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_cache_list_destroy(&priv->hrxqs);
+		mlx5_list_destroy(&priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +1 -1
index b5c05b12e8..5cd2cfb07c 100644
@@ -1612,7 +1612,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_cache_list_destroy(&priv->hrxqs);
+	mlx5_list_destroy(&priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +7 -8
index facab913bf..de5cd2e06e 100644
@@ -79,7 +79,7 @@ enum mlx5_flow_type {
 	MLX5_FLOW_TYPE_MAXI,
 };
 
-/* Hash and cache list callback context. */
+/* Hlist and list callback context. */
 struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_cache_list port_id_action_list; /* Port ID action cache. */
-	struct mlx5_cache_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_cache_list sample_action_list; /* List of sample actions. */
-	struct mlx5_cache_list dest_array_list;
+	struct mlx5_list port_id_action_list; /* Port ID action list. */
+	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list sample_action_list; /* List of sample actions. */
+	struct mlx5_list dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1244,7 +1244,7 @@ struct mlx5_ind_table_obj {
 /* Hash Rx queue. */
 __extension__
 struct mlx5_hrxq {
-	struct mlx5_cache_entry entry; /* Cache entry. */
+	struct mlx5_list_entry entry; /* List entry. */
 	uint32_t standalone:1; /* This object used in shared action. */
 	struct mlx5_ind_table_obj *ind_table; /* Indirection table. */
 	RTE_STD_C11
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_cache_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
@@ -1392,7 +1392,6 @@ struct mlx5_priv {
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t max_lro_msg_size;
-	/* Tags resources cache. */
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */
drivers/net/mlx5/mlx5_flow.h +45 -43

130 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +278 -283

1,131 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +6 -6
index 1b264e5994..3dcc71d51d 100644
@@ -222,13 +222,13 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_cache_entry *mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		struct mlx5_cache_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		       struct mlx5_cache_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
+		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
+int mlx5_hrxq_match_cb(struct mlx5_list *list,
+		       struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-			 struct mlx5_cache_entry *entry);
+void mlx5_hrxq_remove_cb(struct mlx5_list *list,
+			 struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +14 -14
index bb9a908087..8395332507 100644
@@ -2093,7 +2093,7 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  * Match an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2103,8 +2103,8 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
  *   0 if match, none zero if not match.
  */
 int
-mlx5_hrxq_match_cb(struct mlx5_cache_list *list,
-		   struct mlx5_cache_entry *entry,
+mlx5_hrxq_match_cb(struct mlx5_list *list,
+		   struct mlx5_list_entry *entry,
 		   void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2242,13 +2242,13 @@ __mlx5_hrxq_remove(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
  *   Index to Hash Rx queue to release.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  */
 void
-mlx5_hrxq_remove_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry)
+mlx5_hrxq_remove_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry)
 {
 	struct rte_eth_dev *dev = list->ctx;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2305,7 +2305,7 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * Create an Rx Hash queue.
  *
  * @param list
- *   Cache list pointer.
+ *   mlx5 list pointer.
  * @param entry
  *   Hash queue entry pointer.
  * @param cb_ctx
@@ -2314,9 +2314,9 @@ __mlx5_hrxq_create(struct rte_eth_dev *dev,
  * @return
  *   queue entry on success, NULL otherwise.
  */
-struct mlx5_cache_entry *
-mlx5_hrxq_create_cb(struct mlx5_cache_list *list,
-		    struct mlx5_cache_entry *entry __rte_unused,
+struct mlx5_list_entry *
+mlx5_hrxq_create_cb(struct mlx5_list *list,
+		    struct mlx5_list_entry *entry __rte_unused,
 		    void *cb_ctx)
 {
 	struct rte_eth_dev *dev = list->ctx;
@@ -2344,7 +2344,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 	struct mlx5_hrxq *hrxq;
-	struct mlx5_cache_entry *entry;
+	struct mlx5_list_entry *entry;
 	struct mlx5_flow_cb_ctx ctx = {
 		.data = rss_desc,
 	};
@@ -2352,7 +2352,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_cache_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(&priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2382,7 +2382,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_cache_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2470,7 +2470,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_cache_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(&priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +39 -39

188 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +47 -47

215 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +3 -4
index 17716b66c9..bcf72dc6db 100644
@@ -610,10 +610,9 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_cache_list_init(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
+		mlx5_hrxq_remove_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#115 lore raw

PATCH v6 8/26 net/mlx5: add per lcore cache to the list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When mlx5 list object is accessed by multiple cores, the list lock counter is all the time written by all the cores what increases cache misses in the memory caches.

In addition, when one thread accesses the list for add\remove\lookup operation, all the other threads coming to do an operation in the list are stuck in the lock.

Add per lcore cache to allow thread manipulations to be lockless when the list objects are mostly reused.

Synchronization with atomic operations should be done in order to allow threads to unregister an entry from other thread cache.

---
 drivers/net/mlx5/linux/mlx5_os.c   |  58 ++++----
 drivers/net/mlx5/mlx5.h            |   1 +
 drivers/net/mlx5/mlx5_flow.h       |  21 ++-
 drivers/net/mlx5/mlx5_flow_dv.c    | 181 +++++++++++++++++++++++-
 drivers/net/mlx5/mlx5_rx.h         |   5 +
 drivers/net/mlx5/mlx5_rxq.c        |  71 +++++++---
 drivers/net/mlx5/mlx5_utils.c      | 214 ++++++++++++++++++-----------
 drivers/net/mlx5/mlx5_utils.h      |  30 ++--
 drivers/net/mlx5/windows/mlx5_os.c |   5 +-
 9 files changed, 451 insertions(+), 135 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +34 -24
index 5d537fa3e4..25e4bdb7a6 100644
@@ -272,30 +272,38 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* The resources below are only valid with DV support. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
-	/* Init port id action mlx5 list. */
+	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, 0, sh,
-			     flow_dv_port_id_create_cb,
-			     flow_dv_port_id_match_cb,
-			     flow_dv_port_id_remove_cb);
-	/* Init push vlan action mlx5 list. */
+	mlx5_list_create(&sh->port_id_action_list, s, sh,
+			 flow_dv_port_id_create_cb,
+			 flow_dv_port_id_match_cb,
+			 flow_dv_port_id_remove_cb,
+			 flow_dv_port_id_clone_cb,
+			 flow_dv_port_id_clone_free_cb);
+	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, 0, sh,
-			     flow_dv_push_vlan_create_cb,
-			     flow_dv_push_vlan_match_cb,
-			     flow_dv_push_vlan_remove_cb);
-	/* Init sample action mlx5 list. */
+	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
+			 flow_dv_push_vlan_create_cb,
+			 flow_dv_push_vlan_match_cb,
+			 flow_dv_push_vlan_remove_cb,
+			 flow_dv_push_vlan_clone_cb,
+			 flow_dv_push_vlan_clone_free_cb);
+	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, 0, sh,
-			     flow_dv_sample_create_cb,
-			     flow_dv_sample_match_cb,
-			     flow_dv_sample_remove_cb);
-	/* Init dest array action mlx5 list. */
+	mlx5_list_create(&sh->sample_action_list, s, sh,
+			 flow_dv_sample_create_cb,
+			 flow_dv_sample_match_cb,
+			 flow_dv_sample_remove_cb,
+			 flow_dv_sample_clone_cb,
+			 flow_dv_sample_clone_free_cb);
+	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, 0, sh,
-			     flow_dv_dest_array_create_cb,
-			     flow_dv_dest_array_match_cb,
-			     flow_dv_dest_array_remove_cb);
+	mlx5_list_create(&sh->dest_array_list, s, sh,
+			 flow_dv_dest_array_create_cb,
+			 flow_dv_dest_array_match_cb,
+			 flow_dv_dest_array_remove_cb,
+			 flow_dv_dest_array_clone_cb,
+			 flow_dv_dest_array_clone_free_cb);
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -1713,10 +1721,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
-			     mlx5_hrxq_create_cb,
-			     mlx5_hrxq_match_cb,
-			     mlx5_hrxq_remove_cb);
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
+			 mlx5_hrxq_match_cb,
+			 mlx5_hrxq_remove_cb,
+			 mlx5_hrxq_clone_cb,
+			 mlx5_hrxq_clone_free_cb);
+	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
drivers/net/mlx5/mlx5.h +1 -0
index de5cd2e06e..1141e12977 100644
@@ -1388,6 +1388,7 @@ struct mlx5_priv {
 	/* Indirection tables. */
 	LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
 	/* Pointer to next element. */
+	rte_rwlock_t ind_tbls_lock;
 	uint32_t refcnt; /**< Reference counter. */
 	/**< Verbs modify header action object. */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
drivers/net/mlx5/mlx5_flow.h +20 -1
index cbae4f1872..2016adc6ee 100644
@@ -1646,7 +1646,11 @@ struct mlx5_list_entry *flow_dv_port_id_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_port_id_remove_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry);
-
+struct mlx5_list_entry *flow_dv_port_id_clone_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused,
+				 void *cb_ctx);
+void flow_dv_port_id_clone_free_cb(struct mlx5_list *list,
+				struct mlx5_list_entry *entry __rte_unused);
 int flow_dv_push_vlan_match_cb(struct mlx5_list *list,
 			       struct mlx5_list_entry *entry, void *cb_ctx);
 struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
@@ -1654,6 +1658,11 @@ struct mlx5_list_entry *flow_dv_push_vlan_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_push_vlan_remove_cb(struct mlx5_list *list,
 				 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_push_vlan_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_push_vlan_clone_free_cb(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry);
 
 int flow_dv_sample_match_cb(struct mlx5_list *list,
 			    struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1662,6 +1671,11 @@ struct mlx5_list_entry *flow_dv_sample_create_cb(struct mlx5_list *list,
 						 void *cb_ctx);
 void flow_dv_sample_remove_cb(struct mlx5_list *list,
 			      struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_sample_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_sample_clone_free_cb(struct mlx5_list *list,
+			      struct mlx5_list_entry *entry);
 
 int flow_dv_dest_array_match_cb(struct mlx5_list *list,
 				struct mlx5_list_entry *entry, void *cb_ctx);
@@ -1670,6 +1684,11 @@ struct mlx5_list_entry *flow_dv_dest_array_create_cb(struct mlx5_list *list,
 						  void *cb_ctx);
 void flow_dv_dest_array_remove_cb(struct mlx5_list *list,
 				  struct mlx5_list_entry *entry);
+struct mlx5_list_entry *flow_dv_dest_array_clone_cb
+				(struct mlx5_list *list,
+				 struct mlx5_list_entry *entry, void *cb_ctx);
+void flow_dv_dest_array_clone_free_cb(struct mlx5_list *list,
+				  struct mlx5_list_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
 int flow_dev_geneve_tlv_option_resource_register(struct rte_eth_dev *dev,
drivers/net/mlx5/mlx5_flow_dv.c +179 -2

219 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -0
index 3dcc71d51d..5450ddd388 100644
@@ -229,6 +229,11 @@ int mlx5_hrxq_match_cb(struct mlx5_list *list,
 		       void *cb_ctx);
 void mlx5_hrxq_remove_cb(struct mlx5_list *list,
 			 struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+					   struct mlx5_list_entry *entry,
+					   void *cb_ctx __rte_unused);
+void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
 int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hxrq_idx);
drivers/net/mlx5/mlx5_rxq.c +52 -19

117 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.c +135 -79

288 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +22 -8
index cfb3cb6180..9e3fe0cb85 100644
@@ -310,9 +310,14 @@ struct mlx5_list;
  */
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
-	uint32_t ref_cnt; /* Reference count. */
+	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	struct mlx5_list_entry *gentry;
 };
 
+struct mlx5_list_cache {
+	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+} __rte_cache_aligned;
+
 /**
  * Type of callback function for entry removal.
  *
@@ -340,6 +345,13 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
 typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
 				   struct mlx5_list_entry *entry, void *ctx);
 
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
+				 (struct mlx5_list *list,
+				  struct mlx5_list_entry *entry, void *ctx);
+
+typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
+					 struct mlx5_list_entry *entry);
+
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
@@ -376,15 +388,17 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t entry_sz; /**< Entry size, 0: use create callback. */
-	rte_rwlock_t lock; /* read/write lock. */
 	uint32_t gen_cnt; /* List modification will update generation count. */
 	uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
+	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
-	LIST_HEAD(mlx5_list_head, mlx5_list_entry) head;
+	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
+	mlx5_list_clone_free_cb cb_clone_free;
+	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	/* Lcore cache, last index is the global cache. */
 };
 
 /**
@@ -394,8 +408,6 @@ struct mlx5_list {
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
- * @param entry_size
- *   Entry size to allocate, 0 to allocate by creation callback.
  * @param ctx
  *   Pointer to the list context data.
  * @param cb_create
@@ -408,10 +420,12 @@ struct mlx5_list {
  *   0 on success, otherwise failure.
  */
 int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, uint32_t entry_size, void *ctx,
+			 const char *name, void *ctx,
 			 mlx5_list_create_cb cb_create,
 			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove);
+			 mlx5_list_remove_cb cb_remove,
+			 mlx5_list_clone_cb cb_clone,
+			 mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +3 -2
index bcf72dc6db..8ced98f0dc 100644
@@ -610,9 +610,10 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", 0, eth_dev,
+	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
-		mlx5_hrxq_remove_cb);
+		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
+		mlx5_hrxq_clone_free_cb);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
 	if (err < 0) {
Signature · 1 line
2.25.1
#116 lore raw

PATCH v6 9/26 net/mlx5: minimize list critical sections

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The mlx5 internal list utility is thread safe.

In order to synchronize list access between the threads, a RW lock is taken for the critical sections.

The create\remove\clone\clone_free operations are in the critical sections.

These operations are heavy and make the critical sections heavy because they are used for memory and other resources allocations\deallocations.

Moved out the operations from the critical sections and use generation counter in order to detect parallel allocations.

---
 drivers/net/mlx5/mlx5_utils.c | 86 ++++++++++++++++++-----------------
 drivers/net/mlx5/mlx5_utils.h |  5 +-
 2 files changed, 48 insertions(+), 43 deletions(-)
drivers/net/mlx5/mlx5_utils.c +45 -41
index f505caed4e..c4c9adb039 100644
@@ -101,7 +101,7 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 {
 	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
 
-	if (!lentry)
+	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
@@ -112,8 +112,8 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *lentry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry, *local_entry;
+	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	MLX5_ASSERT(list);
@@ -122,51 +122,56 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	/* Lookup in local cache. */
-	lentry = __list_lookup(list, lcore_index, ctx, true);
-	if (lentry)
-		return lentry;
-	/* Lookup with read lock, reuse if found. */
+	/* 1. Lookup in local cache. */
+	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	if (local_entry)
+		return local_entry;
+	/* 2. Lookup with read lock on global list, reuse if found. */
 	rte_rwlock_read_lock(&list->lock);
 	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (entry == NULL) {
-		prev_gen_cnt = __atomic_load_n(&list->gen_cnt,
-					       __ATOMIC_ACQUIRE);
-		rte_rwlock_read_unlock(&list->lock);
-	} else {
+	if (likely(entry)) {
 		rte_rwlock_read_unlock(&list->lock);
 		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	/* Not found, append with write lock - block read from other threads. */
+	prev_gen_cnt = list->gen_cnt;
+	rte_rwlock_read_unlock(&list->lock);
+	/* 3. Prepare new entry for global list and for cache. */
+	entry = list->cb_create(list, entry, ctx);
+	if (unlikely(!entry))
+		return NULL;
+	local_entry = list->cb_clone(list, entry, ctx);
+	if (unlikely(!local_entry)) {
+		list->cb_remove(list, entry);
+		return NULL;
+	}
+	entry->ref_cnt = 1u;
+	local_entry->ref_cnt = 1u;
+	local_entry->gentry = entry;
 	rte_rwlock_write_lock(&list->lock);
-	/* If list changed by other threads before lock, search again. */
-	if (prev_gen_cnt != __atomic_load_n(&list->gen_cnt, __ATOMIC_ACQUIRE)) {
-		/* Lookup and reuse w/o read lock. */
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-		if (entry) {
+	/* 4. Make sure the same entry was not created before the write lock. */
+	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(list,
+							       RTE_MAX_LCORE,
+							       ctx, true);
+
+		if (unlikely(oentry)) {
+			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
-		}
-	}
-	entry = list->cb_create(list, entry, ctx);
-	if (entry) {
-		lentry = mlx5_list_cache_insert(list, lcore_index, entry, ctx);
-		if (!lentry) {
 			list->cb_remove(list, entry);
-		} else {
-			entry->ref_cnt = 1u;
-			LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry,
-					 next);
-			__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_RELEASE);
-			__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			list->cb_clone_free(list, local_entry);
+			return mlx5_list_cache_insert(list, lcore_index, oentry,
+						      ctx);
 		}
-
 	}
+	/* 5. Update lists. */
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	return lentry;
+	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
+		list->name, (void *)entry, entry->ref_cnt);
+	return local_entry;
 }
 
 int
@@ -180,12 +185,11 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
-	if (__atomic_load_n(&gentry->ref_cnt, __ATOMIC_ACQUIRE) == 0) {
-		__atomic_add_fetch(&list->gen_cnt, 1, __ATOMIC_ACQUIRE);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		list->cb_remove(list, gentry);
 		rte_rwlock_write_unlock(&list->lock);
+		list->cb_remove(list, gentry);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
drivers/net/mlx5/mlx5_utils.h +3 -2
index 9e3fe0cb85..6dade8238d 100644
@@ -388,8 +388,9 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	uint32_t gen_cnt; /* List modification will update generation count. */
-	uint32_t count; /* number of entries in list. */
+	volatile uint32_t gen_cnt;
+	/* List modification will update generation count. */
+	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
 	rte_rwlock_t lock; /* read/write lock. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
Signature · 1 line
2.25.1
#117 lore raw

PATCH v6 10/26 net/mlx5: manage list cache entries release

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

When a cache entry is allocated by lcore A and is released by lcore B, the driver should synchronize the cache list access of lcore A.

The design decision is to manage a counter per lcore cache that will be increased atomically when the non-original lcore decreases the reference counter of cache entry to 0.

In list register operation, before the running lcore starts a lookup in its cache, it will check the counter in order to free invalid entries in its cache.

---
 drivers/net/mlx5/mlx5_utils.c | 79 +++++++++++++++++++++++------------
 drivers/net/mlx5/mlx5_utils.h |  2 +
 2 files changed, 54 insertions(+), 27 deletions(-)
drivers/net/mlx5/mlx5_utils.c +52 -27
index c4c9adb039..13c7dbe1c2 100644
@@ -47,36 +47,25 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
-
-		if (list->cb_match(list, entry, ctx)) {
-			if (lcore_index < RTE_MAX_LCORE) {
+		if (list->cb_match(list, entry, ctx) == 0) {
+			if (reuse) {
+				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
+							 __ATOMIC_ACQUIRE) - 1;
+				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
+					list->name, (void *)entry,
+					entry->ref_cnt);
+			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_ACQUIRE);
-				if (ret == 0) {
-					LIST_REMOVE(entry, next);
-					list->cb_clone_free(list, entry);
-				}
-			}
-			entry = nentry;
-			continue;
-		}
-		if (reuse) {
-			ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-						 __ATOMIC_ACQUIRE);
-			if (ret == 1u) {
-				/* Entry was invalid before, free it. */
-				LIST_REMOVE(entry, next);
-				list->cb_clone_free(list, entry);
-				entry = nentry;
-				continue;
 			}
-			DRV_LOG(DEBUG, "mlx5 list %s entry %p ref++: %u.",
-				list->name, (void *)entry, entry->ref_cnt);
+			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+				return entry;
+			if (reuse && ret == 0)
+				entry->ref_cnt--; /* Invalid entry. */
 		}
-		break;
+		entry = LIST_NEXT(entry, next);
 	}
-	return entry;
+	return NULL;
 }
 
 struct mlx5_list_entry *
@@ -105,10 +94,31 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
+	lentry->lcore_idx = (uint32_t)lcore_index;
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
 	return lentry;
 }
 
+static void
+__list_cache_clean(struct mlx5_list *list, int lcore_index)
+{
+	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
+	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
+					       __ATOMIC_RELAXED);
+
+	while (inv_cnt != 0 && entry != NULL) {
+		struct mlx5_list_entry *nentry = LIST_NEXT(entry, next);
+
+		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
+			LIST_REMOVE(entry, next);
+			list->cb_clone_free(list, entry);
+			inv_cnt--;
+		}
+		entry = nentry;
+	}
+}
+
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
@@ -122,6 +132,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	/* 0. Free entries that was invalidated by other lcores. */
+	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
@@ -147,6 +159,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
+	local_entry->lcore_idx = (uint32_t)lcore_index;
 	rte_rwlock_write_lock(&list->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
 	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
@@ -169,8 +182,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.",
-		list->name, (void *)entry, entry->ref_cnt);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
@@ -179,9 +192,21 @@ mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
+	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
+	lcore_idx = rte_lcore_index(rte_lcore_id());
+	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
+	if (entry->lcore_idx == (uint32_t)lcore_idx) {
+		LIST_REMOVE(entry, next);
+		list->cb_clone_free(list, entry);
+	} else if (likely(lcore_idx != -1)) {
+		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+				   __ATOMIC_RELAXED);
+	} else {
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/net/mlx5/mlx5_utils.h +2 -0
index 6dade8238d..71da5ab4f9 100644
@@ -311,11 +311,13 @@ struct mlx5_list;
 struct mlx5_list_entry {
 	LIST_ENTRY(mlx5_list_entry) next; /* Entry pointers in the list. */
 	uint32_t ref_cnt; /* 0 means, entry is invalid. */
+	uint32_t lcore_idx;
 	struct mlx5_list_entry *gentry;
 };
 
 struct mlx5_list_cache {
 	LIST_HEAD(mlx5_list_head, mlx5_list_entry) h;
+	uint32_t inv_cnt; /* Invalid entries counter. */
 } __rte_cache_aligned;
 
 /**
Signature · 1 line
2.25.1
#118 lore raw

PATCH v6 11/26 net/mlx5: relax the list utility atomic operations

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

The atomic operation in the list utility no need a barriers because the critical part are managed by RW lock.

Relax them.

---
 drivers/net/mlx5/mlx5_utils.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
drivers/net/mlx5/mlx5_utils.c +6 -6
index 13c7dbe1c2..daecf37575 100644
@@ -50,13 +50,13 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 		if (list->cb_match(list, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
-							 __ATOMIC_ACQUIRE) - 1;
+							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					list->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
-						      __ATOMIC_ACQUIRE);
+						      __ATOMIC_RELAXED);
 			}
 			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
 				return entry;
@@ -181,7 +181,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
 	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
@@ -194,7 +194,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	struct mlx5_list_entry *gentry = entry->gentry;
 	int lcore_idx;
 
-	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	lcore_idx = rte_lcore_index(rte_lcore_id());
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
@@ -207,14 +207,14 @@ mlx5_list_unregister(struct mlx5_list *list,
 	} else {
 		return 0;
 	}
-	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_ACQUIRE) != 0)
+	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
 		list->cb_remove(list, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_ACQUIRE);
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
 		return 0;
Signature · 1 line
2.25.1
#119 lore raw

PATCH v6 12/26 net/mlx5: allocate list memory by the create API

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Currently, the list memory was allocated by the list API caller.

Move it to be allocated by the create API in order to save consistence with the hlist utility.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 105 ++++++++++++++++++++---------
 drivers/net/mlx5/mlx5.c            |   3 +-
 drivers/net/mlx5/mlx5.h            |  10 +--
 drivers/net/mlx5/mlx5_flow.h       |   2 +-
 drivers/net/mlx5/mlx5_flow_dv.c    |  60 ++++++++++-------
 drivers/net/mlx5/mlx5_rxq.c        |   6 +-
 drivers/net/mlx5/mlx5_utils.c      |  19 ++++--
 drivers/net/mlx5/mlx5_utils.h      |  15 ++---
 drivers/net/mlx5/windows/mlx5_os.c |   2 +-
 9 files changed, 139 insertions(+), 83 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +73 -32
index 25e4bdb7a6..2a9a6c3bf8 100644
@@ -274,36 +274,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->port_id_action_list, s, sh,
-			 flow_dv_port_id_create_cb,
-			 flow_dv_port_id_match_cb,
-			 flow_dv_port_id_remove_cb,
-			 flow_dv_port_id_clone_cb,
-			 flow_dv_port_id_clone_free_cb);
+	sh->port_id_action_list = mlx5_list_create(s, sh,
+						   flow_dv_port_id_create_cb,
+						   flow_dv_port_id_match_cb,
+						   flow_dv_port_id_remove_cb,
+						   flow_dv_port_id_clone_cb,
+						 flow_dv_port_id_clone_free_cb);
+	if (!sh->port_id_action_list)
+		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->push_vlan_action_list, s, sh,
-			 flow_dv_push_vlan_create_cb,
-			 flow_dv_push_vlan_match_cb,
-			 flow_dv_push_vlan_remove_cb,
-			 flow_dv_push_vlan_clone_cb,
-			 flow_dv_push_vlan_clone_free_cb);
+	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+						    flow_dv_push_vlan_create_cb,
+						    flow_dv_push_vlan_match_cb,
+						    flow_dv_push_vlan_remove_cb,
+						    flow_dv_push_vlan_clone_cb,
+					       flow_dv_push_vlan_clone_free_cb);
+	if (!sh->push_vlan_action_list)
+		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	mlx5_list_create(&sh->sample_action_list, s, sh,
-			 flow_dv_sample_create_cb,
-			 flow_dv_sample_match_cb,
-			 flow_dv_sample_remove_cb,
-			 flow_dv_sample_clone_cb,
-			 flow_dv_sample_clone_free_cb);
+	sh->sample_action_list = mlx5_list_create(s, sh,
+						  flow_dv_sample_create_cb,
+						  flow_dv_sample_match_cb,
+						  flow_dv_sample_remove_cb,
+						  flow_dv_sample_clone_cb,
+						  flow_dv_sample_clone_free_cb);
+	if (!sh->sample_action_list)
+		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	mlx5_list_create(&sh->dest_array_list, s, sh,
-			 flow_dv_dest_array_create_cb,
-			 flow_dv_dest_array_match_cb,
-			 flow_dv_dest_array_remove_cb,
-			 flow_dv_dest_array_clone_cb,
-			 flow_dv_dest_array_clone_free_cb);
+	sh->dest_array_list = mlx5_list_create(s, sh,
+					       flow_dv_dest_array_create_cb,
+					       flow_dv_dest_array_match_cb,
+					       flow_dv_dest_array_remove_cb,
+					       flow_dv_dest_array_clone_cb,
+					      flow_dv_dest_array_clone_free_cb);
+	if (!sh->dest_array_list)
+		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
 	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
@@ -456,6 +464,22 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		sh->tunnel_hub = NULL;
 	}
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 	return err;
 }
 
@@ -517,9 +541,23 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv)
 		mlx5_release_tunnel_hub(sh, priv->dev_port);
 		sh->tunnel_hub = NULL;
 	}
-	mlx5_list_destroy(&sh->port_id_action_list);
-	mlx5_list_destroy(&sh->push_vlan_action_list);
 	mlx5_free_table_hash_list(priv);
+	if (sh->port_id_action_list) {
+		mlx5_list_destroy(sh->port_id_action_list);
+		sh->port_id_action_list = NULL;
+	}
+	if (sh->push_vlan_action_list) {
+		mlx5_list_destroy(sh->push_vlan_action_list);
+		sh->push_vlan_action_list = NULL;
+	}
+	if (sh->sample_action_list) {
+		mlx5_list_destroy(sh->sample_action_list);
+		sh->sample_action_list = NULL;
+	}
+	if (sh->dest_array_list) {
+		mlx5_list_destroy(sh->dest_array_list);
+		sh->dest_array_list = NULL;
+	}
 }
 
 /**
@@ -1721,11 +1759,13 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev, mlx5_hrxq_create_cb,
-			 mlx5_hrxq_match_cb,
-			 mlx5_hrxq_remove_cb,
-			 mlx5_hrxq_clone_cb,
-			 mlx5_hrxq_clone_free_cb);
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+				       mlx5_hrxq_match_cb,
+				       mlx5_hrxq_remove_cb,
+				       mlx5_hrxq_clone_cb,
+				       mlx5_hrxq_clone_free_cb);
+	if (!priv->hrxqs)
+		goto error;
 	rte_rwlock_init(&priv->ind_tbls_lock);
 	/* Query availability of metadata reg_c's. */
 	err = mlx5_flow_discover_mreg_c(eth_dev);
@@ -1784,7 +1824,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			mlx5_l3t_destroy(priv->mtr_profile_tbl);
 		if (own_domain_id)
 			claim_zero(rte_eth_switch_domain_free(priv->domain_id));
-		mlx5_list_destroy(&priv->hrxqs);
+		if (priv->hrxqs)
+			mlx5_list_destroy(priv->hrxqs);
 		mlx5_free(priv);
 		if (eth_dev != NULL)
 			eth_dev->data->dev_private = NULL;
drivers/net/mlx5/mlx5.c +2 -1
index 5cd2cfb07c..d04b5d616f 100644
@@ -1612,7 +1612,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
 	if (ret)
 		DRV_LOG(WARNING, "port %u some flows still remain",
 			dev->data->port_id);
-	mlx5_list_destroy(&priv->hrxqs);
+	if (priv->hrxqs)
+		mlx5_list_destroy(priv->hrxqs);
 	/*
 	 * Free the shared context in last turn, because the cleanup
 	 * routines above may use some shared fields, like
drivers/net/mlx5/mlx5.h +5 -5
index 1141e12977..88b84d749a 100644
@@ -1137,10 +1137,10 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_hlist *encaps_decaps; /* Encap/decap action hash list. */
 	struct mlx5_hlist *modify_cmds;
 	struct mlx5_hlist *tag_table;
-	struct mlx5_list port_id_action_list; /* Port ID action list. */
-	struct mlx5_list push_vlan_action_list; /* Push VLAN actions. */
-	struct mlx5_list sample_action_list; /* List of sample actions. */
-	struct mlx5_list dest_array_list;
+	struct mlx5_list *port_id_action_list; /* Port ID action list. */
+	struct mlx5_list *push_vlan_action_list; /* Push VLAN actions. */
+	struct mlx5_list *sample_action_list; /* List of sample actions. */
+	struct mlx5_list *dest_array_list;
 	/* List of destination array actions. */
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
@@ -1382,7 +1382,7 @@ struct mlx5_priv {
 	struct mlx5_obj_ops obj_ops; /* HW objects operations. */
 	LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
 	LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
-	struct mlx5_list hrxqs; /* Hash Rx queues. */
+	struct mlx5_list *hrxqs; /* Hash Rx queues. */
 	LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
 	LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
 	/* Indirection tables. */
drivers/net/mlx5/mlx5_flow.h +1 -1
index 2016adc6ee..59a6f31d2e 100644
@@ -591,7 +591,7 @@ struct mlx5_flow_tbl_data_entry {
 	/**< hash list entry, 64-bits key inside. */
 	struct mlx5_flow_tbl_resource tbl;
 	/**< flow table resource. */
-	struct mlx5_list matchers;
+	struct mlx5_list *matchers;
 	/**< matchers' header associated with the flow table. */
 	struct mlx5_flow_dv_jump_tbl_resource jump;
 	/**< jump resource, at most one for each table created. */
drivers/net/mlx5/mlx5_flow_dv.c +35 -25

180 lines not on this page · raw message

drivers/net/mlx5/mlx5_rxq.c +3 -3
index f8769da8dc..aa9e973d10 100644
@@ -2385,7 +2385,7 @@ uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 	if (rss_desc->shared_rss) {
 		hrxq = __mlx5_hrxq_create(dev, rss_desc);
 	} else {
-		entry = mlx5_list_register(&priv->hrxqs, &ctx);
+		entry = mlx5_list_register(priv->hrxqs, &ctx);
 		if (!entry)
 			return 0;
 		hrxq = container_of(entry, typeof(*hrxq), entry);
@@ -2415,7 +2415,7 @@ int mlx5_hrxq_release(struct rte_eth_dev *dev, uint32_t hrxq_idx)
 	if (!hrxq)
 		return 0;
 	if (!hrxq->standalone)
-		return mlx5_list_unregister(&priv->hrxqs, &hrxq->entry);
+		return mlx5_list_unregister(priv->hrxqs, &hrxq->entry);
 	__mlx5_hrxq_remove(dev, hrxq);
 	return 0;
 }
@@ -2503,7 +2503,7 @@ mlx5_hrxq_verify(struct rte_eth_dev *dev)
 {
 	struct mlx5_priv *priv = dev->data->dev_private;
 
-	return mlx5_list_get_entry_num(&priv->hrxqs);
+	return mlx5_list_get_entry_num(priv->hrxqs);
 }
 
 /**
drivers/net/mlx5/mlx5_utils.c +12 -7
index daecf37575..0be778935f 100644
@@ -11,20 +11,25 @@
 
 /********************* mlx5 list ************************/
 
-int
-mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
 		 mlx5_list_clone_cb cb_clone,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
+	struct mlx5_list *list;
 	int i;
 
-	MLX5_ASSERT(list);
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free)
-		return -1;
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -37,7 +42,7 @@ mlx5_list_create(struct mlx5_list *list, const char *name, void *ctx,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
-	return 0;
+	return list;
 }
 
 static struct mlx5_list_entry *
@@ -244,7 +249,7 @@ mlx5_list_destroy(struct mlx5_list *list)
 			}
 		}
 	}
-	memset(list, 0, sizeof(*list));
+	mlx5_free(list);
 }
 
 uint32_t
drivers/net/mlx5/mlx5_utils.h +7 -8
index 71da5ab4f9..ea64bb75c9 100644
@@ -420,15 +420,14 @@ struct mlx5_list {
  * @param cb_remove
  *   Callback function for entry remove.
  * @return
- *   0 on success, otherwise failure.
+ *   List pointer on success, otherwise NULL.
  */
-int mlx5_list_create(struct mlx5_list *list,
-			 const char *name, void *ctx,
-			 mlx5_list_create_cb cb_create,
-			 mlx5_list_match_cb cb_match,
-			 mlx5_list_remove_cb cb_remove,
-			 mlx5_list_clone_cb cb_clone,
-			 mlx5_list_clone_free_cb cb_clone_free);
+struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   mlx5_list_create_cb cb_create,
+				   mlx5_list_match_cb cb_match,
+				   mlx5_list_remove_cb cb_remove,
+				   mlx5_list_clone_cb cb_clone,
+				   mlx5_list_clone_free_cb cb_clone_free);
 
 /**
  * Search an entry matching the key.
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index 8ced98f0dc..e6176e70d2 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	mlx5_list_create(&priv->hrxqs, "hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#120 lore raw

PATCH v6 13/26 common/mlx5: move list utility to common

In reply to Suanming Mou

Hash list is planned to be implemented with the cache list code.

This commit moves the list utility to common directory.

---
 drivers/common/mlx5/mlx5_common.h       |   2 +
 drivers/common/mlx5/mlx5_common_utils.c | 250 +++++++++++++++++++++++
 drivers/common/mlx5/mlx5_common_utils.h | 205 +++++++++++++++++++
 drivers/common/mlx5/version.map         |   7 +
 drivers/net/mlx5/mlx5_utils.c           | 251 ------------------------
 drivers/net/mlx5/mlx5_utils.h           | 197 -------------------
 6 files changed, 464 insertions(+), 448 deletions(-)
drivers/common/mlx5/mlx5_common.h +2 -0
index 306f2f1ab7..7fb7d40b38 100644
@@ -14,6 +14,8 @@
 #include <rte_kvargs.h>
 #include <rte_devargs.h>
 #include <rte_bitops.h>
+#include <rte_lcore.h>
+#include <rte_spinlock.h>
 #include <rte_os_shim.h>
 
 #include "mlx5_prm.h"
drivers/common/mlx5/mlx5_common_utils.c +250 -0

256 lines not on this page · raw message

drivers/common/mlx5/mlx5_common_utils.h +205 -0

211 lines not on this page · raw message

drivers/common/mlx5/version.map +7 -0
index b8be73a77b..e6586d6f6f 100644
@@ -73,6 +73,13 @@ INTERNAL {
 
 	mlx5_glue;
 
+	mlx5_list_create;
+	mlx5_list_register;
+	mlx5_list_unregister;
+	mlx5_list_lookup;
+	mlx5_list_get_entry_num;
+	mlx5_list_destroy;
+
 	mlx5_hlist_create;
 	mlx5_hlist_lookup;
 	mlx5_hlist_register;
drivers/net/mlx5/mlx5_utils.c +0 -251

257 lines not on this page · raw message

drivers/net/mlx5/mlx5_utils.h +0 -197

203 lines not on this page · raw message

Signature · 1 line
2.25.1
#121 lore raw

PATCH v6 14/26 common/mlx5: add list lcore share

In reply to Suanming Mou

As some actions in SW-steering is only memory and can be allowed to create duplicate objects, for lists which no need to check if there are existing same objects in other sub local lists, search the object only in local list will be more efficient.

This commit adds the lcore share mode to list optimized the list register.

---
 drivers/common/mlx5/mlx5_common_utils.c | 48 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h | 16 ++++++---
 drivers/net/mlx5/linux/mlx5_os.c        | 11 +++---
 drivers/net/mlx5/mlx5_flow_dv.c         |  2 +-
 drivers/net/mlx5/windows/mlx5_os.c      |  2 +-
 5 files changed, 56 insertions(+), 23 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +36 -12
index 8bb8a6016d..bc08f8ba25 100644
@@ -14,7 +14,7 @@
 /********************* mlx5 list ************************/
 
 struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx,
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_create_cb cb_create,
 		 mlx5_list_match_cb cb_match,
 		 mlx5_list_remove_cb cb_remove,
@@ -35,6 +35,7 @@ mlx5_list_create(const char *name, void *ctx,
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
+	list->lcores_share = lcores_share;
 	list->cb_create = cb_create;
 	list->cb_match = cb_match;
 	list->cb_remove = cb_remove;
@@ -119,7 +120,10 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			list->cb_clone_free(list, entry);
+			if (list->lcores_share)
+				list->cb_clone_free(list, entry);
+			else
+				list->cb_remove(list, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -129,7 +133,7 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	struct mlx5_list_entry *entry, *local_entry;
+	struct mlx5_list_entry *entry = NULL, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
@@ -145,25 +149,36 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	local_entry = __list_lookup(list, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	/* 2. Lookup with read lock on global list, reuse if found. */
-	rte_rwlock_read_lock(&list->lock);
-	entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
-	if (likely(entry)) {
+	if (list->lcores_share) {
+		/* 2. Lookup with read lock on global list, reuse if found. */
+		rte_rwlock_read_lock(&list->lock);
+		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		if (likely(entry)) {
+			rte_rwlock_read_unlock(&list->lock);
+			return mlx5_list_cache_insert(list, lcore_index, entry,
+						      ctx);
+		}
+		prev_gen_cnt = list->gen_cnt;
 		rte_rwlock_read_unlock(&list->lock);
-		return mlx5_list_cache_insert(list, lcore_index, entry, ctx);
 	}
-	prev_gen_cnt = list->gen_cnt;
-	rte_rwlock_read_unlock(&list->lock);
 	/* 3. Prepare new entry for global list and for cache. */
 	entry = list->cb_create(list, entry, ctx);
 	if (unlikely(!entry))
 		return NULL;
+	entry->ref_cnt = 1u;
+	if (!list->lcores_share) {
+		entry->lcore_idx = (uint32_t)lcore_index;
+		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
+			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+		return entry;
+	}
 	local_entry = list->cb_clone(list, entry, ctx);
 	if (unlikely(!local_entry)) {
 		list->cb_remove(list, entry);
 		return NULL;
 	}
-	entry->ref_cnt = 1u;
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
@@ -207,13 +222,22 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		list->cb_clone_free(list, entry);
+		if (list->lcores_share)
+			list->cb_clone_free(list, entry);
+		else
+			list->cb_remove(list, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
+	if (!list->lcores_share) {
+		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
+			list->name, (void *)entry);
+		return 0;
+	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
 	rte_rwlock_write_lock(&list->lock);
drivers/common/mlx5/mlx5_common_utils.h +12 -4
index 96add6d003..000279d236 100644
@@ -100,11 +100,8 @@ typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
  */
 struct mlx5_list {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
-	volatile uint32_t gen_cnt;
-	/* List modification will update generation count. */
-	volatile uint32_t count; /* number of entries in list. */
 	void *ctx; /* user objects target to callback. */
-	rte_rwlock_t lock; /* read/write lock. */
+	bool lcores_share; /* Whether to share objects between the lcores. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -112,17 +109,27 @@ struct mlx5_list {
 	mlx5_list_clone_free_cb cb_clone_free;
 	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
+	volatile uint32_t gen_cnt; /* List modification may update it. */
+	volatile uint32_t count; /* number of entries in list. */
+	rte_rwlock_t lock; /* read/write lock. */
 };
 
 /**
  * Create a mlx5 list.
  *
+ * For actions in SW-steering is only memory and  can be allowed
+ * to create duplicate objects, the lists don't need to check if
+ * there are existing same objects in other sub local lists,
+ * search the object only in local list will be more efficient.
+ *
  * @param list
  *   Pointer to the hast list table.
  * @param name
  *   Name of the mlx5 list.
  * @param ctx
  *   Pointer to the list context data.
+ * @param lcores_share
+ *   Whether to share objects between the lcores.
  * @param cb_create
  *   Callback function for entry create.
  * @param cb_match
@@ -134,6 +141,7 @@ struct mlx5_list {
  */
 __rte_internal
 struct mlx5_list *mlx5_list_create(const char *name, void *ctx,
+				   bool lcores_share,
 				   mlx5_list_create_cb cb_create,
 				   mlx5_list_match_cb cb_match,
 				   mlx5_list_remove_cb cb_remove,
drivers/net/mlx5/linux/mlx5_os.c +6 -5
index 2a9a6c3bf8..ce41fb34a0 100644
@@ -274,7 +274,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	/* Init port id action list. */
 	snprintf(s, sizeof(s), "%s_port_id_action_list", sh->ibdev_name);
-	sh->port_id_action_list = mlx5_list_create(s, sh,
+	sh->port_id_action_list = mlx5_list_create(s, sh, true,
 						   flow_dv_port_id_create_cb,
 						   flow_dv_port_id_match_cb,
 						   flow_dv_port_id_remove_cb,
@@ -284,7 +284,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init push vlan action list. */
 	snprintf(s, sizeof(s), "%s_push_vlan_action_list", sh->ibdev_name);
-	sh->push_vlan_action_list = mlx5_list_create(s, sh,
+	sh->push_vlan_action_list = mlx5_list_create(s, sh, true,
 						    flow_dv_push_vlan_create_cb,
 						    flow_dv_push_vlan_match_cb,
 						    flow_dv_push_vlan_remove_cb,
@@ -294,7 +294,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init sample action list. */
 	snprintf(s, sizeof(s), "%s_sample_action_list", sh->ibdev_name);
-	sh->sample_action_list = mlx5_list_create(s, sh,
+	sh->sample_action_list = mlx5_list_create(s, sh, true,
 						  flow_dv_sample_create_cb,
 						  flow_dv_sample_match_cb,
 						  flow_dv_sample_remove_cb,
@@ -304,7 +304,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Init dest array action list. */
 	snprintf(s, sizeof(s), "%s_dest_array_list", sh->ibdev_name);
-	sh->dest_array_list = mlx5_list_create(s, sh,
+	sh->dest_array_list = mlx5_list_create(s, sh, true,
 					       flow_dv_dest_array_create_cb,
 					       flow_dv_dest_array_match_cb,
 					       flow_dv_dest_array_remove_cb,
@@ -1759,7 +1759,8 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, mlx5_hrxq_create_cb,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
+				       mlx5_hrxq_create_cb,
 				       mlx5_hrxq_match_cb,
 				       mlx5_hrxq_remove_cb,
 				       mlx5_hrxq_clone_cb,
drivers/net/mlx5/mlx5_flow_dv.c +1 -1
index 5a536e3dff..4a45172a12 100644
@@ -10054,7 +10054,7 @@ flow_dv_tbl_create_cb(struct mlx5_hlist *list, uint64_t key64, void *cb_ctx)
 	MKSTR(matcher_name, "%s_%s_%u_%u_matcher_list",
 	      key.is_fdb ? "FDB" : "NIC", key.is_egress ? "egress" : "ingress",
 	      key.level, key.id);
-	tbl_data->matchers = mlx5_list_create(matcher_name, sh,
+	tbl_data->matchers = mlx5_list_create(matcher_name, sh, true,
 					      flow_dv_matcher_create_cb,
 					      flow_dv_matcher_match_cb,
 					      flow_dv_matcher_remove_cb,
drivers/net/mlx5/windows/mlx5_os.c +1 -1
index e6176e70d2..a04f93e1d4 100644
@@ -610,7 +610,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 			err = ENOTSUP;
 			goto error;
 	}
-	priv->hrxqs = mlx5_list_create("hrxq", eth_dev,
+	priv->hrxqs = mlx5_list_create("hrxq", eth_dev, true,
 		mlx5_hrxq_create_cb, mlx5_hrxq_match_cb,
 		mlx5_hrxq_remove_cb, mlx5_hrxq_clone_cb,
 		mlx5_hrxq_clone_free_cb);
Signature · 1 line
2.25.1
#122 lore raw

PATCH v6 15/26 common/mlx5: call list callbacks with context

In reply to Suanming Mou

This commit optimizes to call the list callback functions with global context directly.

---
 drivers/common/mlx5/mlx5_common_utils.c |  24 ++---
 drivers/common/mlx5/mlx5_common_utils.h |  36 ++++----
 drivers/net/mlx5/mlx5_flow.h            |  94 +++++++++----------
 drivers/net/mlx5/mlx5_flow_dv.c         | 115 +++++++++++-------------
 drivers/net/mlx5/mlx5_rx.h              |  13 ++-
 drivers/net/mlx5/mlx5_rxq.c             |  53 +++--------
 6 files changed, 132 insertions(+), 203 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +12 -12
index bc08f8ba25..0f400f6353 100644
@@ -55,7 +55,7 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list, entry, ctx) == 0) {
+		if (list->cb_match(list->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
@@ -96,7 +96,7 @@ static struct mlx5_list_entry *
 mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list, gentry, ctx);
+	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
@@ -121,9 +121,9 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
 			if (list->lcores_share)
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			else
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
@@ -162,7 +162,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_rwlock_read_unlock(&list->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list, entry, ctx);
+	entry = list->cb_create(list->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
@@ -174,9 +174,9 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list, entry, ctx);
+	local_entry = list->cb_clone(list->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list, entry);
+		list->cb_remove(list->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
@@ -192,8 +192,8 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
 			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list, entry);
-			list->cb_clone_free(list, local_entry);
+			list->cb_remove(list->ctx, entry);
+			list->cb_clone_free(list->ctx, local_entry);
 			return mlx5_list_cache_insert(list, lcore_index, oentry,
 						      ctx);
 		}
@@ -223,9 +223,9 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (list->lcores_share)
-			list->cb_clone_free(list, entry);
+			list->cb_clone_free(list->ctx, entry);
 		else
-			list->cb_remove(list, entry);
+			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
 		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
 				   __ATOMIC_RELAXED);
@@ -244,7 +244,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
 		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list, gentry);
+		list->cb_remove(list->ctx, gentry);
 		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
 			list->name, (void *)gentry);
drivers/common/mlx5/mlx5_common_utils.h +15 -21
index 000279d236..a691ff8f0d 100644
@@ -33,19 +33,19 @@ struct mlx5_list_cache {
 /**
  * Type of callback function for entry removal.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance user context.
  * @param entry
  *   The entry in the list.
  */
-typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
-				     struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_remove_cb)(void *tool_ctx,
+				    struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined matching.
  *
- * @param list
- *   The mlx5 list.
+ * @param tool_ctx
+ *   The tool instance context.
  * @param entry
  *   The entry in the list.
  * @param ctx
@@ -54,34 +54,28 @@ typedef void (*mlx5_list_remove_cb)(struct mlx5_list *list,
  * @return
  *   0 if matching, non-zero number otherwise.
  */
-typedef int (*mlx5_list_match_cb)(struct mlx5_list *list,
+typedef int (*mlx5_list_match_cb)(void *tool_ctx,
 				   struct mlx5_list_entry *entry, void *ctx);
 
-typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry, void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_clone_cb)(void *tool_ctx,
+				      struct mlx5_list_entry *entry, void *ctx);
 
-typedef void (*mlx5_list_clone_free_cb)(struct mlx5_list *list,
-					 struct mlx5_list_entry *entry);
+typedef void (*mlx5_list_clone_free_cb)(void *tool_ctx,
+					struct mlx5_list_entry *entry);
 
 /**
  * Type of function for user defined mlx5 list entry creation.
  *
- * @param list
- *   The mlx5 list.
- * @param entry
- *   The new allocated entry, NULL if list entry size unspecified,
- *   New entry has to be allocated in callback and return.
+ * @param tool_ctx
+ *   The mlx5 tool instance context.
  * @param ctx
  *   The pointer to new entry context.
  *
  * @return
  *   Pointer of entry on success, NULL otherwise.
  */
-typedef struct mlx5_list_entry *(*mlx5_list_create_cb)
-				 (struct mlx5_list *list,
-				  struct mlx5_list_entry *entry,
-				  void *ctx);
+typedef struct mlx5_list_entry *(*mlx5_list_create_cb)(void *tool_ctx,
+						       void *ctx);
 
 /**
  * Linked mlx5 list structure.
drivers/net/mlx5/mlx5_flow.h +40 -54

104 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +50 -65

294 lines not on this page · raw message

drivers/net/mlx5/mlx5_rx.h +5 -8
index 5450ddd388..3f2b99fb65 100644
@@ -222,17 +222,14 @@ int mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 			      struct mlx5_ind_table_obj *ind_tbl,
 			      uint16_t *queues, const uint32_t queues_n,
 			      bool standalone);
-struct mlx5_list_entry *mlx5_hrxq_create_cb(struct mlx5_list *list,
-		struct mlx5_list_entry *entry __rte_unused, void *cb_ctx);
-int mlx5_hrxq_match_cb(struct mlx5_list *list,
-		       struct mlx5_list_entry *entry,
+struct mlx5_list_entry *mlx5_hrxq_create_cb(void *tool_ctx, void *cb_ctx);
+int mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry,
 		       void *cb_ctx);
-void mlx5_hrxq_remove_cb(struct mlx5_list *list,
-			 struct mlx5_list_entry *entry);
-struct mlx5_list_entry *mlx5_hrxq_clone_cb(struct mlx5_list *list,
+void mlx5_hrxq_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry);
+struct mlx5_list_entry *mlx5_hrxq_clone_cb(void *tool_ctx,
 					   struct mlx5_list_entry *entry,
 					   void *cb_ctx __rte_unused);
-void mlx5_hrxq_clone_free_cb(struct mlx5_list *list,
+void mlx5_hrxq_clone_free_cb(void *tool_ctx __rte_unused,
 			     struct mlx5_list_entry *entry);
 uint32_t mlx5_hrxq_get(struct rte_eth_dev *dev,
 		       struct mlx5_flow_rss_desc *rss_desc);
drivers/net/mlx5/mlx5_rxq.c +10 -43

91 lines not on this page · raw message

Signature · 1 line
2.25.1
#123 lore raw

PATCH v6 16/26 common/mlx5: add per-lcore cache to hash list utility

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Using the mlx5 list utility object in the hlist buckets.

This patch moves the list utility object to the common utility, creates all the clone operations for all the hlist instances in the driver.

Also adjust all the utility callbacks to be generic for both list and hlist.

---
 drivers/common/mlx5/mlx5_common_utils.c | 238 +++++++------------
 drivers/common/mlx5/mlx5_common_utils.h | 141 +++---------
 drivers/net/mlx5/linux/mlx5_os.c        |  35 +--
 drivers/net/mlx5/mlx5.c                 |  10 +-
 drivers/net/mlx5/mlx5.h                 |   1 +
 drivers/net/mlx5/mlx5_flow.c            | 155 +++++++++----
 drivers/net/mlx5/mlx5_flow.h            |  92 ++++----
 drivers/net/mlx5/mlx5_flow_dv.c         | 292 +++++++++++++++---------
 8 files changed, 482 insertions(+), 482 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +77 -161
index 0f400f6353..ab098b13fa 100644
@@ -13,25 +13,21 @@
 
 /********************* mlx5 list ************************/
 
-struct mlx5_list *
-mlx5_list_create(const char *name, void *ctx, bool lcores_share,
-		 mlx5_list_create_cb cb_create,
-		 mlx5_list_match_cb cb_match,
-		 mlx5_list_remove_cb cb_remove,
-		 mlx5_list_clone_cb cb_clone,
-		 mlx5_list_clone_free_cb cb_clone_free)
+static int
+mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
+	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       mlx5_list_match_cb cb_match,
+	       mlx5_list_remove_cb cb_remove,
+	       mlx5_list_clone_cb cb_clone,
+	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	struct mlx5_list *list;
 	int i;
 
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
-		return NULL;
+		return -EINVAL;
 	}
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
-	if (!list)
-		return NULL;
 	if (name)
 		snprintf(list->name, sizeof(list->name), "%s", name);
 	list->ctx = ctx;
@@ -45,6 +41,28 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
 	for (i = 0; i <= RTE_MAX_LCORE; i++)
 		LIST_INIT(&list->cache[i].h);
+	return 0;
+}
+
+struct mlx5_list *
+mlx5_list_create(const char *name, void *ctx, bool lcores_share,
+		 mlx5_list_create_cb cb_create,
+		 mlx5_list_match_cb cb_match,
+		 mlx5_list_remove_cb cb_remove,
+		 mlx5_list_clone_cb cb_clone,
+		 mlx5_list_clone_free_cb cb_clone_free)
+{
+	struct mlx5_list *list;
+
+	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	if (!list)
+		return NULL;
+	if (mlx5_list_init(list, name, ctx, lcores_share,
+			   cb_create, cb_match, cb_remove, cb_clone,
+			   cb_clone_free) != 0) {
+		mlx5_free(list);
+		return NULL;
+	}
 	return list;
 }
 
@@ -254,8 +272,8 @@ mlx5_list_unregister(struct mlx5_list *list,
 	return 1;
 }
 
-void
-mlx5_list_destroy(struct mlx5_list *list)
+static void
+mlx5_list_uninit(struct mlx5_list *list)
 {
 	struct mlx5_list_entry *entry;
 	int i;
@@ -266,15 +284,21 @@ mlx5_list_destroy(struct mlx5_list *list)
 			entry = LIST_FIRST(&list->cache[i].h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list, entry);
+				list->cb_remove(list->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", list->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list, entry);
+				list->cb_clone_free(list->ctx, entry);
 			}
 		}
 	}
+}
+
+void
+mlx5_list_destroy(struct mlx5_list *list)
+{
+	mlx5_list_uninit(list);
 	mlx5_free(list);
 }
 
@@ -287,37 +311,24 @@ mlx5_list_get_entry_num(struct mlx5_list *list)
 
 /********************* Hash List **********************/
 
-static struct mlx5_hlist_entry *
-mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused,
-			     void *ctx __rte_unused)
-{
-	return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY);
-}
-
-static void
-mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused,
-			     struct mlx5_hlist_entry *entry)
-{
-	mlx5_free(entry);
-}
-
 struct mlx5_hlist *
-mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
-		  uint32_t flags, mlx5_hlist_create_cb cb_create,
-		  mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove)
+mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
+		  bool lcores_share, void *ctx, mlx5_list_create_cb cb_create,
+		  mlx5_list_match_cb cb_match,
+		  mlx5_list_remove_cb cb_remove,
+		  mlx5_list_clone_cb cb_clone,
+		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
 
-	if (!size || !cb_match || (!cb_create ^ !cb_remove))
-		return NULL;
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
-		DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, "
-			"will be aligned to 0x%" PRIX32 ".", size, act_size);
+		DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
+			"be aligned to 0x%" PRIX32 ".", size, act_size);
 	} else {
 		act_size = size;
 	}
@@ -331,61 +342,24 @@ mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size,
 			name ? name : "None");
 		return NULL;
 	}
-	if (name)
-		snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
-	h->table_sz = act_size;
 	h->mask = act_size - 1;
-	h->entry_sz = entry_size;
-	h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY);
-	h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST);
-	h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb;
-	h->cb_match = cb_match;
-	h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb;
-	for (i = 0; i < act_size; i++)
-		rte_rwlock_init(&h->buckets[i].lock);
-	DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.",
-		h->name, act_size);
-	return h;
-}
-
-static struct mlx5_hlist_entry *
-__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	       void *ctx, bool reuse)
-{
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	first = &h->buckets[idx].head;
-	LIST_FOREACH(node, first, next) {
-		if (!h->cb_match(h, node, key, ctx)) {
-			if (reuse) {
-				__atomic_add_fetch(&node->ref_cnt, 1,
-						   __ATOMIC_RELAXED);
-				DRV_LOG(DEBUG, "Hash list %s entry %p "
-					"reuse: %u.",
-					h->name, (void *)node, node->ref_cnt);
-			}
-			break;
+	h->lcores_share = lcores_share;
+	h->direct_key = direct_key;
+	for (i = 0; i < act_size; i++) {
+		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   cb_create, cb_match, cb_remove, cb_clone,
+				   cb_clone_free) != 0) {
+			mlx5_free(h);
+			return NULL;
 		}
 	}
-	return node;
+	DRV_LOG(DEBUG, "Hash list %s with size 0x%" PRIX32 " was created.",
+		name, act_size);
+	return h;
 }
 
-static struct mlx5_hlist_entry *
-hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx,
-	     void *ctx, bool reuse)
-{
-	struct mlx5_hlist_entry *node;
-
-	MLX5_ASSERT(h);
-	rte_rwlock_read_lock(&h->buckets[idx].lock);
-	node = __hlist_lookup(h, key, idx, ctx, reuse);
-	rte_rwlock_read_unlock(&h->buckets[idx].lock);
-	return node;
-}
 
-struct mlx5_hlist_entry *
+struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
@@ -394,102 +368,44 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return hlist_lookup(h, key, idx, ctx, false);
+	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
 }
 
-struct mlx5_hlist_entry*
+struct mlx5_list_entry*
 mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
-	struct mlx5_hlist_head *first;
-	struct mlx5_hlist_bucket *b;
-	struct mlx5_hlist_entry *entry;
-	uint32_t prev_gen_cnt = 0;
+	struct mlx5_list_entry *entry;
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	MLX5_ASSERT(h);
-	b = &h->buckets[idx];
-	/* Use write lock directly for write-most list. */
-	if (!h->write_most) {
-		prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE);
-		entry = hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			return entry;
-	}
-	rte_rwlock_write_lock(&b->lock);
-	/* Check if the list changed by other threads. */
-	if (h->write_most ||
-	    prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) {
-		entry = __hlist_lookup(h, key, idx, ctx, true);
-		if (entry)
-			goto done;
-	}
-	first = &b->head;
-	entry = h->cb_create(h, key, ctx);
-	if (!entry) {
-		rte_errno = ENOMEM;
-		DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name);
-		goto done;
+	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	if (likely(entry)) {
+		if (h->lcores_share)
+			entry->gentry->bucket_idx = idx;
+		else
+			entry->bucket_idx = idx;
 	}
-	entry->idx = idx;
-	entry->ref_cnt = 1;
-	LIST_INSERT_HEAD(first, entry, next);
-	__atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL);
-	DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-done:
-	rte_rwlock_write_unlock(&b->lock);
 	return entry;
 }
 
 int
-mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
+mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = entry->idx;
-
-	rte_rwlock_write_lock(&h->buckets[idx].lock);
-	MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev);
-	DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.",
-		h->name, (void *)entry, entry->ref_cnt);
-	if (--entry->ref_cnt) {
-		rte_rwlock_write_unlock(&h->buckets[idx].lock);
-		return 1;
-	}
-	LIST_REMOVE(entry, next);
-	/* Set to NULL to get rid of removing action for more than once. */
-	entry->next.le_prev = NULL;
-	h->cb_remove(h, entry);
-	rte_rwlock_write_unlock(&h->buckets[idx].lock);
-	DRV_LOG(DEBUG, "Hash list %s entry %p removed.",
-		h->name, (void *)entry);
-	return 0;
+	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+							      entry->bucket_idx;
+
+	return mlx5_list_unregister(&h->buckets[idx].l, entry);
 }
 
 void
 mlx5_hlist_destroy(struct mlx5_hlist *h)
 {
-	uint32_t idx;
-	struct mlx5_hlist_entry *entry;
+	uint32_t i;
 
-	MLX5_ASSERT(h);
-	for (idx = 0; idx < h->table_sz; ++idx) {
-		/* No LIST_FOREACH_SAFE, using while instead. */
-		while (!LIST_EMPTY(&h->buckets[idx].head)) {
-			entry = LIST_FIRST(&h->buckets[idx].head);
-			LIST_REMOVE(entry, next);
-			/*
-			 * The owner of whole element which contains data entry
-			 * is the user, so it's the user's duty to do the clean
-			 * up and the free work because someone may not put the
-			 * hlist entry at the beginning(suggested to locate at
-			 * the beginning). Or else the default free function
-			 * will be used.
-			 */
-			h->cb_remove(h, entry);
-		}
-	}
+	for (i = 0; i <= h->mask; i++)
+		mlx5_list_uninit(&h->buckets[i].l);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +35 -106

197 lines not on this page · raw message

drivers/net/mlx5/linux/mlx5_os.c +18 -17
index ce41fb34a0..b04e5f8ba6 100644
@@ -261,7 +261,7 @@ static int
 mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 {
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE] __rte_unused;
+	char s[MLX5_NAME_SIZE] __rte_unused;
 	int err;
 
 	MLX5_ASSERT(sh && sh->refcnt);
@@ -314,44 +314,44 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	/* Create tags hash list table. */
 	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, 0,
-					  MLX5_HLIST_WRITE_MOST,
-					  flow_dv_tag_create_cb,
+	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
+					  false, sh, flow_dv_tag_create_cb,
 					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb);
+					  flow_dv_tag_remove_cb,
+					  flow_dv_tag_clone_cb,
+					  flow_dv_tag_clone_free_cb);
 	if (!sh->tag_table) {
 		DRV_LOG(ERR, "tags with hash creation failed.");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->tag_table->ctx = sh;
 	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
 	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    0, MLX5_HLIST_WRITE_MOST |
-					    MLX5_HLIST_DIRECT_KEY,
+					    true, false, sh,
 					    flow_dv_modify_create_cb,
 					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb);
+					    flow_dv_modify_remove_cb,
+					    flow_dv_modify_clone_cb,
+					    flow_dv_modify_clone_free_cb);
 	if (!sh->modify_cmds) {
 		DRV_LOG(ERR, "hdr modify hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->modify_cmds->ctx = sh;
 	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
 	sh->encaps_decaps = mlx5_hlist_create(s,
 					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      0, MLX5_HLIST_DIRECT_KEY |
-					      MLX5_HLIST_WRITE_MOST,
+					      true, true, sh,
 					      flow_dv_encap_decap_create_cb,
 					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb);
+					      flow_dv_encap_decap_remove_cb,
+					      flow_dv_encap_decap_clone_cb,
+					     flow_dv_encap_decap_clone_free_cb);
 	if (!sh->encaps_decaps) {
 		DRV_LOG(ERR, "encap decap hash creation failed");
 		err = ENOMEM;
 		goto error;
 	}
-	sh->encaps_decaps->ctx = sh;
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -1792,15 +1792,16 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev,
 	    priv->sh->dv_regc0_mask) {
 		priv->mreg_cp_tbl = mlx5_hlist_create(MLX5_FLOW_MREG_HNAME,
 						      MLX5_FLOW_MREG_HTABLE_SZ,
-						      0, 0,
+						      false, true, eth_dev,
 						      flow_dv_mreg_create_cb,
 						      flow_dv_mreg_match_cb,
-						      flow_dv_mreg_remove_cb);
+						      flow_dv_mreg_remove_cb,
+						      flow_dv_mreg_clone_cb,
+						    flow_dv_mreg_clone_free_cb);
 		if (!priv->mreg_cp_tbl) {
 			err = ENOMEM;
 			goto error;
 		}
-		priv->mreg_cp_tbl->ctx = eth_dev;
 	}
 	rte_spinlock_init(&priv->shared_act_sl);
 	mlx5_flow_counter_mode_config(eth_dev);
drivers/net/mlx5/mlx5.c +6 -4
index d04b5d616f..fd675c9262 100644
@@ -1359,20 +1359,22 @@ mlx5_alloc_table_hash_list(struct mlx5_priv *priv __rte_unused)
 	/* Tables are only used in DV and DR modes. */
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	struct mlx5_dev_ctx_shared *sh = priv->sh;
-	char s[MLX5_HLIST_NAMESIZE];
+	char s[MLX5_NAME_SIZE];
 
 	MLX5_ASSERT(sh);
 	snprintf(s, sizeof(s), "%s_flow_table", priv->sh->ibdev_name);
 	sh->flow_tbls = mlx5_hlist_create(s, MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE,
-					  0, 0, flow_dv_tbl_create_cb,
+					  false, true, sh,
+					  flow_dv_tbl_create_cb,
 					  flow_dv_tbl_match_cb,
-					  flow_dv_tbl_remove_cb);
+					  flow_dv_tbl_remove_cb,
+					  flow_dv_tbl_clone_cb,
+					  flow_dv_tbl_clone_free_cb);
 	if (!sh->flow_tbls) {
 		DRV_LOG(ERR, "flow tables with hash creation failed.");
 		err = ENOMEM;
 		return err;
 	}
-	sh->flow_tbls->ctx = sh;
 #ifndef HAVE_MLX5DV_DR
 	struct rte_flow_error error;
 	struct rte_eth_dev *dev = &rte_eth_devices[priv->dev_data->port_id];
drivers/net/mlx5/mlx5.h +1 -0
index 88b84d749a..e9b08094a6 100644
@@ -84,6 +84,7 @@ struct mlx5_flow_cb_ctx {
 	struct rte_eth_dev *dev;
 	struct rte_flow_error *error;
 	void *data;
+	void *data2;
 };
 
 /* Device attributes used in mlx5 PMD */
drivers/net/mlx5/mlx5_flow.c +113 -42

298 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow.h +50 -42

143 lines not on this page · raw message

drivers/net/mlx5/mlx5_flow_dv.c +182 -110

464 lines not on this page · raw message

Signature · 1 line
2.25.1
#124 lore raw

PATCH v6 17/26 common/mlx5: allocate cache list memory individually

In reply to Suanming Mou

Currently, the list's local cache instance memory is allocated with the list. As the local cache instance array size is RTE_MAX_LCORE, most of the cases the system will only have very limited cores. allocate the instance memory individually per core will be more economic to the memory.

This commit changes the instance array to pointer array, allocate the local cache memory only when the core is to be used.

---
 drivers/common/mlx5/mlx5_common_utils.c | 62 ++++++++++++++++++-------
 drivers/common/mlx5/mlx5_common_utils.h |  2 +-
 2 files changed, 45 insertions(+), 19 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +44 -18
index ab098b13fa..18d38355fa 100644
@@ -15,14 +15,13 @@
 
 static int
 mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, mlx5_list_create_cb cb_create,
+	       bool lcores_share, struct mlx5_list_cache *gc,
+	       mlx5_list_create_cb cb_create,
 	       mlx5_list_match_cb cb_match,
 	       mlx5_list_remove_cb cb_remove,
 	       mlx5_list_clone_cb cb_clone,
 	       mlx5_list_clone_free_cb cb_clone_free)
 {
-	int i;
-
 	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
 	    !cb_clone_free) {
 		rte_errno = EINVAL;
@@ -38,9 +37,11 @@ mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
 	list->cb_clone = cb_clone;
 	list->cb_clone_free = cb_clone_free;
 	rte_rwlock_init(&list->lock);
+	if (lcores_share) {
+		list->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
+	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
-	for (i = 0; i <= RTE_MAX_LCORE; i++)
-		LIST_INIT(&list->cache[i].h);
 	return 0;
 }
 
@@ -53,11 +54,16 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 		 mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_list *list;
+	struct mlx5_list_cache *gc = NULL;
 
-	list = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*list), 0, SOCKET_ID_ANY);
+	list = mlx5_malloc(MLX5_MEM_ZERO,
+			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
+			   0, SOCKET_ID_ANY);
 	if (!list)
 		return NULL;
-	if (mlx5_list_init(list, name, ctx, lcores_share,
+	if (lcores_share)
+		gc = (struct mlx5_list_cache *)(list + 1);
+	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
 			   cb_create, cb_match, cb_remove, cb_clone,
 			   cb_clone_free) != 0) {
 		mlx5_free(list);
@@ -69,7 +75,8 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 static struct mlx5_list_entry *
 __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 {
-	struct mlx5_list_entry *entry = LIST_FIRST(&list->cache[lcore_index].h);
+	struct mlx5_list_entry *entry =
+				LIST_FIRST(&list->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
@@ -121,14 +128,14 @@ mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, lentry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
 __list_cache_clean(struct mlx5_list *list, int lcore_index)
 {
-	struct mlx5_list_cache *c = &list->cache[lcore_index];
+	struct mlx5_list_cache *c = list->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -161,6 +168,17 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
+	if (unlikely(!list->cache[lcore_index])) {
+		list->cache[lcore_index] = mlx5_malloc(0,
+					sizeof(struct mlx5_list_cache),
+					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
+		if (!list->cache[lcore_index]) {
+			rte_errno = ENOMEM;
+			return NULL;
+		}
+		list->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&list->cache[lcore_index]->h);
+	}
 	/* 0. Free entries that was invalidated by other lcores. */
 	__list_cache_clean(list, lcore_index);
 	/* 1. Lookup in local cache. */
@@ -186,7 +204,7 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 	entry->ref_cnt = 1u;
 	if (!list->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index].h, entry, next);
+		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
 		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
 			list->name, lcore_index, (void *)entry, entry->ref_cnt);
@@ -217,10 +235,10 @@ mlx5_list_register(struct mlx5_list *list, void *ctx)
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE].h, entry, next);
+	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
 	list->gen_cnt++;
 	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index].h, local_entry, next);
+	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
 	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
 	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
 		(void *)entry, entry->ref_cnt);
@@ -245,7 +263,7 @@ mlx5_list_unregister(struct mlx5_list *list,
 		else
 			list->cb_remove(list->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx].inv_cnt, 1,
+		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
 				   __ATOMIC_RELAXED);
 	} else {
 		return 0;
@@ -280,8 +298,10 @@ mlx5_list_uninit(struct mlx5_list *list)
 
 	MLX5_ASSERT(list);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		while (!LIST_EMPTY(&list->cache[i].h)) {
-			entry = LIST_FIRST(&list->cache[i].h);
+		if (!list->cache[i])
+			continue;
+		while (!LIST_EMPTY(&list->cache[i]->h)) {
+			entry = LIST_FIRST(&list->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
 				list->cb_remove(list->ctx, entry);
@@ -292,6 +312,8 @@ mlx5_list_uninit(struct mlx5_list *list)
 				list->cb_clone_free(list->ctx, entry);
 			}
 		}
+		if (i != RTE_MAX_LCORE)
+			mlx5_free(list->cache[i]);
 	}
 }
 
@@ -320,6 +342,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		  mlx5_list_clone_free_cb cb_clone_free)
 {
 	struct mlx5_hlist *h;
+	struct mlx5_list_cache *gc;
 	uint32_t act_size;
 	uint32_t alloc_size;
 	uint32_t i;
@@ -333,7 +356,9 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket) * act_size;
+		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+	if (lcores_share)
+		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
 	h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE,
 			SOCKET_ID_ANY);
@@ -345,8 +370,10 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->mask = act_size - 1;
 	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
+	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
 		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
+				   lcores_share ? &gc[i] : NULL,
 				   cb_create, cb_match, cb_remove, cb_clone,
 				   cb_clone_free) != 0) {
 			mlx5_free(h);
@@ -358,7 +385,6 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
-
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
drivers/common/mlx5/mlx5_common_utils.h +1 -1
index 4bb974fa3e..d49fb64457 100644
@@ -104,7 +104,7 @@ struct mlx5_list {
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
 	mlx5_list_clone_cb cb_clone; /**< entry clone callback. */
 	mlx5_list_clone_free_cb cb_clone_free;
-	struct mlx5_list_cache cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
 	/* Lcore cache, last index is the global cache. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
Signature · 1 line
2.25.1
#125 lore raw

PATCH v6 18/26 common/mlx5: optimize cache list object memory

In reply to Suanming Mou

Currently, hash list uses the cache list as bucket list. The list in the buckets have the same name, ctx and callbacks. This wastes the memory.

This commit abstracts all the name, ctx and callback members in the list to a constant struct and others to the inconstant struct, uses the wrapper functions to satisfy both hash list and cache list can set the list constant and inconstant struct individually.

---
 drivers/common/mlx5/mlx5_common_utils.c | 295 ++++++++++++++----------
 drivers/common/mlx5/mlx5_common_utils.h |  45 ++--
 2 files changed, 201 insertions(+), 139 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +171 -124
index 18d38355fa..573e40e88c 100644
@@ -14,34 +14,16 @@
 /********************* mlx5 list ************************/
 
 static int
-mlx5_list_init(struct mlx5_list *list, const char *name, void *ctx,
-	       bool lcores_share, struct mlx5_list_cache *gc,
-	       mlx5_list_create_cb cb_create,
-	       mlx5_list_match_cb cb_match,
-	       mlx5_list_remove_cb cb_remove,
-	       mlx5_list_clone_cb cb_clone,
-	       mlx5_list_clone_free_cb cb_clone_free)
+mlx5_list_init(struct mlx5_list_inconst *l_inconst,
+	       struct mlx5_list_const *l_const,
+	       struct mlx5_list_cache *gc)
 {
-	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
-	    !cb_clone_free) {
-		rte_errno = EINVAL;
-		return -EINVAL;
+	rte_rwlock_init(&l_inconst->lock);
+	if (l_const->lcores_share) {
+		l_inconst->cache[RTE_MAX_LCORE] = gc;
+		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
 	}
-	if (name)
-		snprintf(list->name, sizeof(list->name), "%s", name);
-	list->ctx = ctx;
-	list->lcores_share = lcores_share;
-	list->cb_create = cb_create;
-	list->cb_match = cb_match;
-	list->cb_remove = cb_remove;
-	list->cb_clone = cb_clone;
-	list->cb_clone_free = cb_clone_free;
-	rte_rwlock_init(&list->lock);
-	if (lcores_share) {
-		list->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&list->cache[RTE_MAX_LCORE]->h);
-	}
-	DRV_LOG(DEBUG, "mlx5 list %s initialized.", list->name);
+	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
 }
 
@@ -56,16 +38,30 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	struct mlx5_list *list;
 	struct mlx5_list_cache *gc = NULL;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	list = mlx5_malloc(MLX5_MEM_ZERO,
 			   sizeof(*list) + (lcores_share ? sizeof(*gc) : 0),
 			   0, SOCKET_ID_ANY);
+
 	if (!list)
 		return NULL;
+	if (name)
+		snprintf(list->l_const.name,
+			 sizeof(list->l_const.name), "%s", name);
+	list->l_const.ctx = ctx;
+	list->l_const.lcores_share = lcores_share;
+	list->l_const.cb_create = cb_create;
+	list->l_const.cb_match = cb_match;
+	list->l_const.cb_remove = cb_remove;
+	list->l_const.cb_clone = cb_clone;
+	list->l_const.cb_clone_free = cb_clone_free;
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
-	if (mlx5_list_init(list, name, ctx, lcores_share, gc,
-			   cb_create, cb_match, cb_remove, cb_clone,
-			   cb_clone_free) != 0) {
+	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
 		mlx5_free(list);
 		return NULL;
 	}
@@ -73,19 +69,21 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 }
 
 static struct mlx5_list_entry *
-__list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
+__list_lookup(struct mlx5_list_inconst *l_inconst,
+	      struct mlx5_list_const *l_const,
+	      int lcore_index, void *ctx, bool reuse)
 {
 	struct mlx5_list_entry *entry =
-				LIST_FIRST(&list->cache[lcore_index]->h);
+				LIST_FIRST(&l_inconst->cache[lcore_index]->h);
 	uint32_t ret;
 
 	while (entry != NULL) {
-		if (list->cb_match(list->ctx, entry, ctx) == 0) {
+		if (l_const->cb_match(l_const->ctx, entry, ctx) == 0) {
 			if (reuse) {
 				ret = __atomic_add_fetch(&entry->ref_cnt, 1,
 							 __ATOMIC_RELAXED) - 1;
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
-					list->name, (void *)entry,
+					l_const->name, (void *)entry,
 					entry->ref_cnt);
 			} else if (lcore_index < RTE_MAX_LCORE) {
 				ret = __atomic_load_n(&entry->ref_cnt,
@@ -101,41 +99,55 @@ __list_lookup(struct mlx5_list *list, int lcore_index, void *ctx, bool reuse)
 	return NULL;
 }
 
-struct mlx5_list_entry *
-mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
+		  struct mlx5_list_const *l_const, void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL;
 	int i;
 
-	rte_rwlock_read_lock(&list->lock);
+	rte_rwlock_read_lock(&l_inconst->lock);
 	for (i = 0; i < RTE_MAX_LCORE; i++) {
-		entry = __list_lookup(list, i, ctx, false);
+		if (!l_inconst->cache[i])
+			continue;
+		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
 		if (entry)
 			break;
 	}
-	rte_rwlock_read_unlock(&list->lock);
+	rte_rwlock_read_unlock(&l_inconst->lock);
 	return entry;
 }
 
+struct mlx5_list_entry *
+mlx5_list_lookup(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_lookup(&list->l_inconst, &list->l_const, ctx);
+}
+
+
 static struct mlx5_list_entry *
-mlx5_list_cache_insert(struct mlx5_list *list, int lcore_index,
+mlx5_list_cache_insert(struct mlx5_list_inconst *l_inconst,
+		       struct mlx5_list_const *l_const, int lcore_index,
 		       struct mlx5_list_entry *gentry, void *ctx)
 {
-	struct mlx5_list_entry *lentry = list->cb_clone(list->ctx, gentry, ctx);
+	struct mlx5_list_entry *lentry =
+			l_const->cb_clone(l_const->ctx, gentry, ctx);
 
 	if (unlikely(!lentry))
 		return NULL;
 	lentry->ref_cnt = 1u;
 	lentry->gentry = gentry;
 	lentry->lcore_idx = (uint32_t)lcore_index;
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, lentry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, lentry, next);
 	return lentry;
 }
 
 static void
-__list_cache_clean(struct mlx5_list *list, int lcore_index)
+__list_cache_clean(struct mlx5_list_inconst *l_inconst,
+		   struct mlx5_list_const *l_const,
+		   int lcore_index)
 {
-	struct mlx5_list_cache *c = list->cache[lcore_index];
+	struct mlx5_list_cache *c = l_inconst->cache[lcore_index];
 	struct mlx5_list_entry *entry = LIST_FIRST(&c->h);
 	uint32_t inv_cnt = __atomic_exchange_n(&c->inv_cnt, 0,
 					       __ATOMIC_RELAXED);
@@ -145,108 +157,123 @@ __list_cache_clean(struct mlx5_list *list, int lcore_index)
 
 		if (__atomic_load_n(&entry->ref_cnt, __ATOMIC_RELAXED) == 0) {
 			LIST_REMOVE(entry, next);
-			if (list->lcores_share)
-				list->cb_clone_free(list->ctx, entry);
+			if (l_const->lcores_share)
+				l_const->cb_clone_free(l_const->ctx, entry);
 			else
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 			inv_cnt--;
 		}
 		entry = nentry;
 	}
 }
 
-struct mlx5_list_entry *
-mlx5_list_register(struct mlx5_list *list, void *ctx)
+static inline struct mlx5_list_entry *
+_mlx5_list_register(struct mlx5_list_inconst *l_inconst,
+		    struct mlx5_list_const *l_const,
+		    void *ctx)
 {
 	struct mlx5_list_entry *entry = NULL, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
 	int lcore_index = rte_lcore_index(rte_lcore_id());
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
 	if (unlikely(lcore_index == -1)) {
 		rte_errno = ENOTSUP;
 		return NULL;
 	}
-	if (unlikely(!list->cache[lcore_index])) {
-		list->cache[lcore_index] = mlx5_malloc(0,
+	if (unlikely(!l_inconst->cache[lcore_index])) {
+		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
 					RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
-		if (!list->cache[lcore_index]) {
+		if (!l_inconst->cache[lcore_index]) {
 			rte_errno = ENOMEM;
 			return NULL;
 		}
-		list->cache[lcore_index]->inv_cnt = 0;
-		LIST_INIT(&list->cache[lcore_index]->h);
+		l_inconst->cache[lcore_index]->inv_cnt = 0;
+		LIST_INIT(&l_inconst->cache[lcore_index]->h);
 	}
 	/* 0. Free entries that was invalidated by other lcores. */
-	__list_cache_clean(list, lcore_index);
+	__list_cache_clean(l_inconst, l_const, lcore_index);
 	/* 1. Lookup in local cache. */
-	local_entry = __list_lookup(list, lcore_index, ctx, true);
+	local_entry = __list_lookup(l_inconst, l_const, lcore_index, ctx, true);
 	if (local_entry)
 		return local_entry;
-	if (list->lcores_share) {
+	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
-		rte_rwlock_read_lock(&list->lock);
-		entry = __list_lookup(list, RTE_MAX_LCORE, ctx, true);
+		rte_rwlock_read_lock(&l_inconst->lock);
+		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+				      ctx, true);
 		if (likely(entry)) {
-			rte_rwlock_read_unlock(&list->lock);
-			return mlx5_list_cache_insert(list, lcore_index, entry,
-						      ctx);
+			rte_rwlock_read_unlock(&l_inconst->lock);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      entry, ctx);
 		}
-		prev_gen_cnt = list->gen_cnt;
-		rte_rwlock_read_unlock(&list->lock);
+		prev_gen_cnt = l_inconst->gen_cnt;
+		rte_rwlock_read_unlock(&l_inconst->lock);
 	}
 	/* 3. Prepare new entry for global list and for cache. */
-	entry = list->cb_create(list->ctx, ctx);
+	entry = l_const->cb_create(l_const->ctx, ctx);
 	if (unlikely(!entry))
 		return NULL;
 	entry->ref_cnt = 1u;
-	if (!list->lcores_share) {
+	if (!l_const->lcores_share) {
 		entry->lcore_idx = (uint32_t)lcore_index;
-		LIST_INSERT_HEAD(&list->cache[lcore_index]->h, entry, next);
-		__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h,
+				 entry, next);
+		__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "MLX5 list %s c%d entry %p new: %u.",
-			list->name, lcore_index, (void *)entry, entry->ref_cnt);
+			l_const->name, lcore_index,
+			(void *)entry, entry->ref_cnt);
 		return entry;
 	}
-	local_entry = list->cb_clone(list->ctx, entry, ctx);
+	local_entry = l_const->cb_clone(l_const->ctx, entry, ctx);
 	if (unlikely(!local_entry)) {
-		list->cb_remove(list->ctx, entry);
+		l_const->cb_remove(l_const->ctx, entry);
 		return NULL;
 	}
 	local_entry->ref_cnt = 1u;
 	local_entry->gentry = entry;
 	local_entry->lcore_idx = (uint32_t)lcore_index;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	/* 4. Make sure the same entry was not created before the write lock. */
-	if (unlikely(prev_gen_cnt != list->gen_cnt)) {
-		struct mlx5_list_entry *oentry = __list_lookup(list,
+	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
+		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
+							       l_const,
 							       RTE_MAX_LCORE,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
 			/* 4.5. Found real race!!, reuse the old entry. */
-			rte_rwlock_write_unlock(&list->lock);
-			list->cb_remove(list->ctx, entry);
-			list->cb_clone_free(list->ctx, local_entry);
-			return mlx5_list_cache_insert(list, lcore_index, oentry,
-						      ctx);
+			rte_rwlock_write_unlock(&l_inconst->lock);
+			l_const->cb_remove(l_const->ctx, entry);
+			l_const->cb_clone_free(l_const->ctx, local_entry);
+			return mlx5_list_cache_insert(l_inconst, l_const,
+						      lcore_index,
+						      oentry, ctx);
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&list->cache[RTE_MAX_LCORE]->h, entry, next);
-	list->gen_cnt++;
-	rte_rwlock_write_unlock(&list->lock);
-	LIST_INSERT_HEAD(&list->cache[lcore_index]->h, local_entry, next);
-	__atomic_add_fetch(&list->count, 1, __ATOMIC_RELAXED);
-	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", list->name,
+	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	l_inconst->gen_cnt++;
+	rte_rwlock_write_unlock(&l_inconst->lock);
+	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
+	__atomic_add_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
+	DRV_LOG(DEBUG, "mlx5 list %s entry %p new: %u.", l_const->name,
 		(void *)entry, entry->ref_cnt);
 	return local_entry;
 }
 
-int
-mlx5_list_unregister(struct mlx5_list *list,
+struct mlx5_list_entry *
+mlx5_list_register(struct mlx5_list *list, void *ctx)
+{
+	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+}
+
+static inline int
+_mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
+		      struct mlx5_list_const *l_const,
 		      struct mlx5_list_entry *entry)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
@@ -258,69 +285,77 @@ mlx5_list_unregister(struct mlx5_list *list,
 	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
-		if (list->lcores_share)
-			list->cb_clone_free(list->ctx, entry);
+		if (l_const->lcores_share)
+			l_const->cb_clone_free(l_const->ctx, entry);
 		else
-			list->cb_remove(list->ctx, entry);
+			l_const->cb_remove(l_const->ctx, entry);
 	} else if (likely(lcore_idx != -1)) {
-		__atomic_add_fetch(&list->cache[entry->lcore_idx]->inv_cnt, 1,
-				   __ATOMIC_RELAXED);
+		__atomic_add_fetch(&l_inconst->cache[entry->lcore_idx]->inv_cnt,
+				   1, __ATOMIC_RELAXED);
 	} else {
 		return 0;
 	}
-	if (!list->lcores_share) {
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+	if (!l_const->lcores_share) {
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)entry);
+			l_const->name, (void *)entry);
 		return 0;
 	}
 	if (__atomic_sub_fetch(&gentry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	rte_rwlock_write_lock(&list->lock);
+	rte_rwlock_write_lock(&l_inconst->lock);
 	if (likely(gentry->ref_cnt == 0)) {
 		LIST_REMOVE(gentry, next);
-		rte_rwlock_write_unlock(&list->lock);
-		list->cb_remove(list->ctx, gentry);
-		__atomic_sub_fetch(&list->count, 1, __ATOMIC_RELAXED);
+		rte_rwlock_write_unlock(&l_inconst->lock);
+		l_const->cb_remove(l_const->ctx, gentry);
+		__atomic_sub_fetch(&l_inconst->count, 1, __ATOMIC_RELAXED);
 		DRV_LOG(DEBUG, "mlx5 list %s entry %p removed.",
-			list->name, (void *)gentry);
+			l_const->name, (void *)gentry);
 		return 0;
 	}
-	rte_rwlock_write_unlock(&list->lock);
+	rte_rwlock_write_unlock(&l_inconst->lock);
 	return 1;
 }
 
+int
+mlx5_list_unregister(struct mlx5_list *list,
+		      struct mlx5_list_entry *entry)
+{
+	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+}
+
 static void
-mlx5_list_uninit(struct mlx5_list *list)
+mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
+		 struct mlx5_list_const *l_const)
 {
 	struct mlx5_list_entry *entry;
 	int i;
 
-	MLX5_ASSERT(list);
+	MLX5_ASSERT(l_inconst);
 	for (i = 0; i <= RTE_MAX_LCORE; i++) {
-		if (!list->cache[i])
+		if (!l_inconst->cache[i])
 			continue;
-		while (!LIST_EMPTY(&list->cache[i]->h)) {
-			entry = LIST_FIRST(&list->cache[i]->h);
+		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
+			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
 			if (i == RTE_MAX_LCORE) {
-				list->cb_remove(list->ctx, entry);
+				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
-					"destroyed.", list->name,
+					"destroyed.", l_const->name,
 					(void *)entry);
 			} else {
-				list->cb_clone_free(list->ctx, entry);
+				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
 		if (i != RTE_MAX_LCORE)
-			mlx5_free(list->cache[i]);
+			mlx5_free(l_inconst->cache[i]);
 	}
 }
 
 void
 mlx5_list_destroy(struct mlx5_list *list)
 {
-	mlx5_list_uninit(list);
+	mlx5_list_uninit(&list->l_inconst, &list->l_const);
 	mlx5_free(list);
 }
 
@@ -328,7 +363,7 @@ uint32_t
 mlx5_list_get_entry_num(struct mlx5_list *list)
 {
 	MLX5_ASSERT(list);
-	return __atomic_load_n(&list->count, __ATOMIC_RELAXED);
+	return __atomic_load_n(&list->l_inconst.count, __ATOMIC_RELAXED);
 }
 
 /********************* Hash List **********************/
@@ -347,6 +382,11 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	uint32_t alloc_size;
 	uint32_t i;
 
+	if (!cb_match || !cb_create || !cb_remove || !cb_clone ||
+	    !cb_clone_free) {
+		rte_errno = EINVAL;
+		return NULL;
+	}
 	/* Align to the next power of 2, 32bits integer is enough now. */
 	if (!rte_is_power_of_2(size)) {
 		act_size = rte_align32pow2(size);
@@ -356,7 +396,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 		act_size = size;
 	}
 	alloc_size = sizeof(struct mlx5_hlist) +
-		     sizeof(struct mlx5_hlist_bucket)  * act_size;
+		     sizeof(struct mlx5_hlist_bucket) * act_size;
 	if (lcores_share)
 		alloc_size += sizeof(struct mlx5_list_cache)  * act_size;
 	/* Using zmalloc, then no need to initialize the heads. */
@@ -367,15 +407,21 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 			name ? name : "None");
 		return NULL;
 	}
+	if (name)
+		snprintf(h->l_const.name, sizeof(h->l_const.name), "%s", name);
+	h->l_const.ctx = ctx;
+	h->l_const.lcores_share = lcores_share;
+	h->l_const.cb_create = cb_create;
+	h->l_const.cb_match = cb_match;
+	h->l_const.cb_remove = cb_remove;
+	h->l_const.cb_clone = cb_clone;
+	h->l_const.cb_clone_free = cb_clone_free;
 	h->mask = act_size - 1;
-	h->lcores_share = lcores_share;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
 	for (i = 0; i < act_size; i++) {
-		if (mlx5_list_init(&h->buckets[i].l, name, ctx, lcores_share,
-				   lcores_share ? &gc[i] : NULL,
-				   cb_create, cb_match, cb_remove, cb_clone,
-				   cb_clone_free) != 0) {
+		if (mlx5_list_init(&h->buckets[i].l, &h->l_const,
+		    lcores_share ? &gc[i] : NULL) != 0) {
 			mlx5_free(h);
 			return NULL;
 		}
@@ -385,6 +431,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	return h;
 }
 
+
 struct mlx5_list_entry *
 mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
@@ -394,7 +441,7 @@ mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	return mlx5_list_lookup(&h->buckets[idx].l, ctx);
+	return _mlx5_list_lookup(&h->buckets[idx].l, &h->l_const, ctx);
 }
 
 struct mlx5_list_entry*
@@ -407,9 +454,9 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = mlx5_list_register(&h->buckets[idx].l, ctx);
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
 	if (likely(entry)) {
-		if (h->lcores_share)
+		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
@@ -420,10 +467,10 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
-	uint32_t idx = h->lcores_share ? entry->gentry->bucket_idx :
+	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
 
-	return mlx5_list_unregister(&h->buckets[idx].l, entry);
+	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
 }
 
 void
@@ -432,6 +479,6 @@ mlx5_hlist_destroy(struct mlx5_hlist *h)
 	uint32_t i;
 
 	for (i = 0; i <= h->mask; i++)
-		mlx5_list_uninit(&h->buckets[i].l);
+		mlx5_list_uninit(&h->buckets[i].l, &h->l_const);
 	mlx5_free(h);
 }
drivers/common/mlx5/mlx5_common_utils.h +30 -15

69 lines not on this page · raw message

Signature · 1 line
2.25.1
#126 lore raw

PATCH v6 19/26 common/mlx5: support list non-lcore operations

In reply to Suanming Mou

This commit supports the list non-lcore operations with an extra sub-list and lock.

---
 drivers/common/mlx5/mlx5_common_utils.c | 92 +++++++++++++++++--------
 drivers/common/mlx5/mlx5_common_utils.h |  9 ++-
 2 files changed, 71 insertions(+), 30 deletions(-)
drivers/common/mlx5/mlx5_common_utils.c +63 -29
index 573e40e88c..fcc81f05c6 100644
@@ -20,8 +20,8 @@ mlx5_list_init(struct mlx5_list_inconst *l_inconst,
 {
 	rte_rwlock_init(&l_inconst->lock);
 	if (l_const->lcores_share) {
-		l_inconst->cache[RTE_MAX_LCORE] = gc;
-		LIST_INIT(&l_inconst->cache[RTE_MAX_LCORE]->h);
+		l_inconst->cache[MLX5_LIST_GLOBAL] = gc;
+		LIST_INIT(&l_inconst->cache[MLX5_LIST_GLOBAL]->h);
 	}
 	DRV_LOG(DEBUG, "mlx5 list %s initialized.", l_const->name);
 	return 0;
@@ -59,6 +59,7 @@ mlx5_list_create(const char *name, void *ctx, bool lcores_share,
 	list->l_const.cb_remove = cb_remove;
 	list->l_const.cb_clone = cb_clone;
 	list->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&list->l_const.lcore_lock);
 	if (lcores_share)
 		gc = (struct mlx5_list_cache *)(list + 1);
 	if (mlx5_list_init(&list->l_inconst, &list->l_const, gc) != 0) {
@@ -85,11 +86,11 @@ __list_lookup(struct mlx5_list_inconst *l_inconst,
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p ref: %u.",
 					l_const->name, (void *)entry,
 					entry->ref_cnt);
-			} else if (lcore_index < RTE_MAX_LCORE) {
+			} else if (lcore_index < MLX5_LIST_GLOBAL) {
 				ret = __atomic_load_n(&entry->ref_cnt,
 						      __ATOMIC_RELAXED);
 			}
-			if (likely(ret != 0 || lcore_index == RTE_MAX_LCORE))
+			if (likely(ret != 0 || lcore_index == MLX5_LIST_GLOBAL))
 				return entry;
 			if (reuse && ret == 0)
 				entry->ref_cnt--; /* Invalid entry. */
@@ -107,10 +108,11 @@ _mlx5_list_lookup(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	rte_rwlock_read_lock(&l_inconst->lock);
-	for (i = 0; i < RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_GLOBAL; i++) {
 		if (!l_inconst->cache[i])
 			continue;
-		entry = __list_lookup(l_inconst, l_const, i, ctx, false);
+		entry = __list_lookup(l_inconst, l_const, i,
+			      ctx, false);
 		if (entry)
 			break;
 	}
@@ -170,18 +172,11 @@ __list_cache_clean(struct mlx5_list_inconst *l_inconst,
 static inline struct mlx5_list_entry *
 _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		    struct mlx5_list_const *l_const,
-		    void *ctx)
+		    void *ctx, int lcore_index)
 {
 	struct mlx5_list_entry *entry = NULL, *local_entry;
 	volatile uint32_t prev_gen_cnt = 0;
-	int lcore_index = rte_lcore_index(rte_lcore_id());
-
 	MLX5_ASSERT(l_inconst);
-	MLX5_ASSERT(lcore_index < RTE_MAX_LCORE);
-	if (unlikely(lcore_index == -1)) {
-		rte_errno = ENOTSUP;
-		return NULL;
-	}
 	if (unlikely(!l_inconst->cache[lcore_index])) {
 		l_inconst->cache[lcore_index] = mlx5_malloc(0,
 					sizeof(struct mlx5_list_cache),
@@ -202,7 +197,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (l_const->lcores_share) {
 		/* 2. Lookup with read lock on global list, reuse if found. */
 		rte_rwlock_read_lock(&l_inconst->lock);
-		entry = __list_lookup(l_inconst, l_const, RTE_MAX_LCORE,
+		entry = __list_lookup(l_inconst, l_const, MLX5_LIST_GLOBAL,
 				      ctx, true);
 		if (likely(entry)) {
 			rte_rwlock_read_unlock(&l_inconst->lock);
@@ -241,7 +236,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 	if (unlikely(prev_gen_cnt != l_inconst->gen_cnt)) {
 		struct mlx5_list_entry *oentry = __list_lookup(l_inconst,
 							       l_const,
-							       RTE_MAX_LCORE,
+							       MLX5_LIST_GLOBAL,
 							       ctx, true);
 
 		if (unlikely(oentry)) {
@@ -255,7 +250,7 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 		}
 	}
 	/* 5. Update lists. */
-	LIST_INSERT_HEAD(&l_inconst->cache[RTE_MAX_LCORE]->h, entry, next);
+	LIST_INSERT_HEAD(&l_inconst->cache[MLX5_LIST_GLOBAL]->h, entry, next);
 	l_inconst->gen_cnt++;
 	rte_rwlock_write_unlock(&l_inconst->lock);
 	LIST_INSERT_HEAD(&l_inconst->cache[lcore_index]->h, local_entry, next);
@@ -268,21 +263,30 @@ _mlx5_list_register(struct mlx5_list_inconst *l_inconst,
 struct mlx5_list_entry *
 mlx5_list_register(struct mlx5_list *list, void *ctx)
 {
-	return _mlx5_list_register(&list->l_inconst, &list->l_const, ctx);
+	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	entry =  _mlx5_list_register(&list->l_inconst, &list->l_const, ctx,
+				     lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return entry;
 }
 
 static inline int
 _mlx5_list_unregister(struct mlx5_list_inconst *l_inconst,
 		      struct mlx5_list_const *l_const,
-		      struct mlx5_list_entry *entry)
+		      struct mlx5_list_entry *entry,
+		      int lcore_idx)
 {
 	struct mlx5_list_entry *gentry = entry->gentry;
-	int lcore_idx;
 
 	if (__atomic_sub_fetch(&entry->ref_cnt, 1, __ATOMIC_RELAXED) != 0)
 		return 1;
-	lcore_idx = rte_lcore_index(rte_lcore_id());
-	MLX5_ASSERT(lcore_idx < RTE_MAX_LCORE);
 	if (entry->lcore_idx == (uint32_t)lcore_idx) {
 		LIST_REMOVE(entry, next);
 		if (l_const->lcores_share)
@@ -321,7 +325,19 @@ int
 mlx5_list_unregister(struct mlx5_list *list,
 		      struct mlx5_list_entry *entry)
 {
-	return _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry);
+	int ret;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&list->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&list->l_inconst, &list->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&list->l_const.lcore_lock);
+	return ret;
+
 }
 
 static void
@@ -332,13 +348,13 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 	int i;
 
 	MLX5_ASSERT(l_inconst);
-	for (i = 0; i <= RTE_MAX_LCORE; i++) {
+	for (i = 0; i < MLX5_LIST_MAX; i++) {
 		if (!l_inconst->cache[i])
 			continue;
 		while (!LIST_EMPTY(&l_inconst->cache[i]->h)) {
 			entry = LIST_FIRST(&l_inconst->cache[i]->h);
 			LIST_REMOVE(entry, next);
-			if (i == RTE_MAX_LCORE) {
+			if (i == MLX5_LIST_GLOBAL) {
 				l_const->cb_remove(l_const->ctx, entry);
 				DRV_LOG(DEBUG, "mlx5 list %s entry %p "
 					"destroyed.", l_const->name,
@@ -347,7 +363,7 @@ mlx5_list_uninit(struct mlx5_list_inconst *l_inconst,
 				l_const->cb_clone_free(l_const->ctx, entry);
 			}
 		}
-		if (i != RTE_MAX_LCORE)
+		if (i != MLX5_LIST_GLOBAL)
 			mlx5_free(l_inconst->cache[i]);
 	}
 }
@@ -416,6 +432,7 @@ mlx5_hlist_create(const char *name, uint32_t size, bool direct_key,
 	h->l_const.cb_remove = cb_remove;
 	h->l_const.cb_clone = cb_clone;
 	h->l_const.cb_clone_free = cb_clone_free;
+	rte_spinlock_init(&h->l_const.lcore_lock);
 	h->mask = act_size - 1;
 	h->direct_key = direct_key;
 	gc = (struct mlx5_list_cache *)&h->buckets[act_size];
@@ -449,28 +466,45 @@ mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx)
 {
 	uint32_t idx;
 	struct mlx5_list_entry *entry;
+	int lcore_index = rte_lcore_index(rte_lcore_id());
 
 	if (h->direct_key)
 		idx = (uint32_t)(key & h->mask);
 	else
 		idx = rte_hash_crc_8byte(key, 0) & h->mask;
-	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	entry = _mlx5_list_register(&h->buckets[idx].l, &h->l_const, ctx,
+				    lcore_index);
 	if (likely(entry)) {
 		if (h->l_const.lcores_share)
 			entry->gentry->bucket_idx = idx;
 		else
 			entry->bucket_idx = idx;
 	}
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
 	return entry;
 }
 
 int
 mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_list_entry *entry)
 {
+	int lcore_index = rte_lcore_index(rte_lcore_id());
+	int ret;
 	uint32_t idx = h->l_const.lcores_share ? entry->gentry->bucket_idx :
 							      entry->bucket_idx;
-
-	return _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry);
+	if (unlikely(lcore_index == -1)) {
+		lcore_index = MLX5_LIST_NLCORE;
+		rte_spinlock_lock(&h->l_const.lcore_lock);
+	}
+	ret = _mlx5_list_unregister(&h->buckets[idx].l, &h->l_const, entry,
+				    lcore_index);
+	if (unlikely(lcore_index == MLX5_LIST_NLCORE))
+		rte_spinlock_unlock(&h->l_const.lcore_lock);
+	return ret;
 }
 
 void
drivers/common/mlx5/mlx5_common_utils.h +8 -1
index 5718a21be0..613d29de0c 100644
@@ -11,6 +11,12 @@
 
 /** Maximum size of string for naming. */
 #define MLX5_NAME_SIZE			32
+/** Maximum size of list. */
+#define MLX5_LIST_MAX			(RTE_MAX_LCORE + 2)
+/** Global list index. */
+#define MLX5_LIST_GLOBAL		((MLX5_LIST_MAX) - 1)
+/** None rte core list index. */
+#define MLX5_LIST_NLCORE		((MLX5_LIST_MAX) - 2)
 
 struct mlx5_list;
 
@@ -87,6 +93,7 @@ struct mlx5_list_const {
 	char name[MLX5_NAME_SIZE]; /**< Name of the mlx5 list. */
 	void *ctx; /* user objects target to callback. */
 	bool lcores_share; /* Whether to share objects between the lcores. */
+	rte_spinlock_t lcore_lock; /* Lock for non-lcore list. */
 	mlx5_list_create_cb cb_create; /**< entry create callback. */
 	mlx5_list_match_cb cb_match; /**< entry match callback. */
 	mlx5_list_remove_cb cb_remove; /**< entry remove callback. */
@@ -102,7 +109,7 @@ struct mlx5_list_inconst {
 	rte_rwlock_t lock; /* read/write lock. */
 	volatile uint32_t gen_cnt; /* List modification may update it. */
 	volatile uint32_t count; /* number of entries in list. */
-	struct mlx5_list_cache *cache[RTE_MAX_LCORE + 1];
+	struct mlx5_list_cache *cache[MLX5_LIST_MAX];
 	/* Lcore cache, last index is the global cache. */
 };
 
Signature · 1 line
2.25.1
#127 lore raw

PATCH v6 20/26 net/mlx5: move modify header allocator to ipool

In reply to Suanming Mou

From: Matan Azrad <[email protected]>

Modify header actions are allocated by mlx5_malloc which has a big overhead of memory and allocation time.

One of the action types under the modify header object is SET_TAG,

The SET_TAG action is commonly not reused by the flows and each flow has its own value.

Hence, the mlx5_malloc becomes a bottleneck in flow insertion rate in the common cases of SET_TAG.

Use ipool allocator for SET_TAG action.

Ipool allocator has less overhead of memory and insertion rate and has better synchronization mechanism in multithread cases.

Different ipool is created for each optional size of modify header handler.

---
 drivers/net/mlx5/mlx5.c         |  4 ++
 drivers/net/mlx5/mlx5.h         | 14 ++++++
 drivers/net/mlx5/mlx5_flow.h    | 14 +-----
 drivers/net/mlx5/mlx5_flow_dv.c | 79 ++++++++++++++++++++++++++++-----
 4 files changed, 86 insertions(+), 25 deletions(-)
drivers/net/mlx5/mlx5.c +4 -0
index fd675c9262..640d36c6be 100644
@@ -802,6 +802,7 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 	}
 }
 
+
 /**
  * Release the flow resources' indexed mempool.
  *
@@ -815,6 +816,9 @@ mlx5_flow_ipool_destroy(struct mlx5_dev_ctx_shared *sh)
 
 	for (i = 0; i < MLX5_IPOOL_MAX; ++i)
 		mlx5_ipool_destroy(sh->ipool[i]);
+	for (i = 0; i < MLX5_MAX_MODIFY_NUM; ++i)
+		if (sh->mdh_ipools[i])
+			mlx5_ipool_destroy(sh->mdh_ipools[i]);
 }
 
 /*
drivers/net/mlx5/mlx5.h +14 -0
index e9b08094a6..e7e4749824 100644
@@ -36,6 +36,19 @@
 
 #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh)
 
+/*
+ * Number of modification commands.
+ * The maximal actions amount in FW is some constant, and it is 16 in the
+ * latest releases. In some old releases, it will be limited to 8.
+ * Since there is no interface to query the capacity, the maximal value should
+ * be used to allow PMD to create the flow. The validation will be done in the
+ * lower driver layer or FW. A failure will be returned if exceeds the maximal
+ * supported actions number on the root table.
+ * On non-root tables, there is no limitation, but 32 is enough right now.
+ */
+#define MLX5_MAX_MODIFY_NUM			32
+#define MLX5_ROOT_TBL_MODIFY_NUM		16
+
 enum mlx5_ipool_index {
 #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
 	MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
@@ -1146,6 +1159,7 @@ struct mlx5_dev_ctx_shared {
 	struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
 	void *default_miss_action; /* Default miss action. */
 	struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+	struct mlx5_indexed_pool *mdh_ipools[MLX5_MAX_MODIFY_NUM];
 	/* Memory Pool for mlx5 flow resources. */
 	struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
 	/* Shared interrupt handler section. */
drivers/net/mlx5/mlx5_flow.h +1 -13
index e3a29297ba..7027012220 100644
@@ -505,23 +505,11 @@ struct mlx5_flow_dv_tag_resource {
 	uint32_t tag_id; /**< Tag ID. */
 };
 
-/*
- * Number of modification commands.
- * The maximal actions amount in FW is some constant, and it is 16 in the
- * latest releases. In some old releases, it will be limited to 8.
- * Since there is no interface to query the capacity, the maximal value should
- * be used to allow PMD to create the flow. The validation will be done in the
- * lower driver layer or FW. A failure will be returned if exceeds the maximal
- * supported actions number on the root table.
- * On non-root tables, there is no limitation, but 32 is enough right now.
- */
-#define MLX5_MAX_MODIFY_NUM			32
-#define MLX5_ROOT_TBL_MODIFY_NUM		16
-
 /* Modify resource structure */
 struct mlx5_flow_dv_modify_hdr_resource {
 	struct mlx5_list_entry entry;
 	void *action; /**< Modify header action object. */
+	uint32_t idx;
 	/* Key area for hash list matching: */
 	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
 	uint8_t actions_num; /**< Number of modification actions. */
drivers/net/mlx5/mlx5_flow_dv.c +67 -12

138 lines not on this page · raw message

Signature · 1 line
2.25.1
#128 lore raw

PATCH v6 21/26 net/mlx5: adjust the hash bucket size

In reply to Suanming Mou

With the new per core optimization to the list, the hash bucket size can be tuned to a more accurate number.

This commit adjusts the hash bucket size.

---
 drivers/net/mlx5/linux/mlx5_os.c | 2 +-
 drivers/net/mlx5/mlx5.c          | 2 +-
 drivers/net/mlx5/mlx5_defs.h     | 6 +++---
 drivers/net/mlx5/mlx5_flow.c     | 5 ++---
 4 files changed, 7 insertions(+), 8 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -1
index b04e5f8ba6..969d265350 100644
@@ -50,7 +50,7 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
 
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
drivers/net/mlx5/mlx5.c +1 -1
index 640d36c6be..3b5c7840a8 100644
@@ -376,7 +376,7 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 #define MLX5_FLOW_MIN_ID_POOL_SIZE 512
 #define MLX5_ID_GENERATION_ARRAY_FACTOR 16
 
-#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 4096
+#define MLX5_FLOW_TABLE_HLIST_ARRAY_SIZE 1024
 
 /**
  * Decide whether representor ID is a HPF(host PF) port on BF2.
drivers/net/mlx5/mlx5_defs.h +3 -3
index 906aa43c5a..ca67ce8213 100644
@@ -178,15 +178,15 @@
 				 sizeof(struct rte_ipv4_hdr))
 
 /* Size of the simple hash table for metadata register table. */
-#define MLX5_FLOW_MREG_HTABLE_SZ 4096
+#define MLX5_FLOW_MREG_HTABLE_SZ 64
 #define MLX5_FLOW_MREG_HNAME "MARK_COPY_TABLE"
 #define MLX5_DEFAULT_COPY_ID UINT32_MAX
 
 /* Size of the simple hash table for header modify table. */
-#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_HDR_MODIFY_HTABLE_SZ (1 << 15)
 
 /* Size of the simple hash table for encap decap table. */
-#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 16)
+#define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
drivers/net/mlx5/mlx5_flow.c +2 -3
index 45e90bb245..1c5d4c283a 100644
@@ -8881,7 +8881,7 @@ mlx5_flow_tunnel_allocate(struct rte_eth_dev *dev,
 		DRV_LOG(ERR, "Tunnel ID %d exceed max limit.", id);
 		return NULL;
 	}
-	tunnel->groups = mlx5_hlist_create("tunnel groups", 1024, false, true,
+	tunnel->groups = mlx5_hlist_create("tunnel groups", 64, false, true,
 					   priv->sh,
 					   mlx5_flow_tunnel_grp2tbl_create_cb,
 					   mlx5_flow_tunnel_grp2tbl_match_cb,
@@ -8990,8 +8990,7 @@ int mlx5_alloc_tunnel_hub(struct mlx5_dev_ctx_shared *sh)
 		return -ENOMEM;
 	LIST_INIT(&thub->tunnels);
 	rte_spinlock_init(&thub->sl);
-	thub->groups = mlx5_hlist_create("flow groups",
-					 rte_align32pow2(MLX5_MAX_TABLES),
+	thub->groups = mlx5_hlist_create("flow groups", 64,
 					 false, true, sh,
 					 mlx5_flow_tunnel_grp2tbl_create_cb,
 					 mlx5_flow_tunnel_grp2tbl_match_cb,
Signature · 1 line
2.25.1
#129 lore raw

PATCH v6 22/26 net/mlx5: enable index pool per-core cache

In reply to Suanming Mou

This commit enables the tag and header modify action index pool per-core cache in non-reclaim memory mode.

---
 drivers/net/mlx5/mlx5.c         | 4 +++-
 drivers/net/mlx5/mlx5.h         | 1 +
 drivers/net/mlx5/mlx5_flow_dv.c | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)
drivers/net/mlx5/mlx5.c +3 -1
index 3b5c7840a8..f79a094a35 100644
@@ -217,7 +217,8 @@ static const struct mlx5_indexed_pool_config mlx5_ipool_cfg[] = {
 		.grow_trunk = 3,
 		.grow_shift = 2,
 		.need_lock = 1,
-		.release_mem_en = 1,
+		.release_mem_en = 0,
+		.per_core_cache = (1 << 16),
 		.malloc = mlx5_malloc,
 		.free = mlx5_free,
 		.type = "mlx5_tag_ipool",
@@ -1129,6 +1130,7 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 	}
 	sh->refcnt = 1;
 	sh->max_port = spawn->max_port;
+	sh->reclaim_mode = config->reclaim_mode;
 	strncpy(sh->ibdev_name, mlx5_os_get_ctx_device_name(sh->ctx),
 		sizeof(sh->ibdev_name) - 1);
 	strncpy(sh->ibdev_path, mlx5_os_get_ctx_device_path(sh->ctx),
drivers/net/mlx5/mlx5.h +1 -0
index e7e4749824..30893c7f38 100644
@@ -1116,6 +1116,7 @@ struct mlx5_dev_ctx_shared {
 	uint32_t qp_ts_format:2; /* QP timestamp formats supported. */
 	uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */
 	uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */
+	uint32_t reclaim_mode:1; /* Reclaim memory. */
 	uint32_t max_port; /* Maximal IB device port index. */
 	struct mlx5_bond_info bond; /* Bonding information. */
 	void *ctx; /* Verbs/DV/DevX context. */
drivers/net/mlx5/mlx5_flow_dv.c +2 -1
index e39fe43854..739c387c0a 100644
@@ -5361,7 +5361,8 @@ flow_dv_modify_ipool_get(struct mlx5_dev_ctx_shared *sh, uint8_t index)
 		       .grow_trunk = 3,
 		       .grow_shift = 2,
 		       .need_lock = 1,
-		       .release_mem_en = 1,
+		       .release_mem_en = !!sh->reclaim_mode,
+		       .per_core_cache = sh->reclaim_mode ? 0 : (1 << 16),
 		       .malloc = mlx5_malloc,
 		       .free = mlx5_free,
 		       .type = "mlx5_modify_action_resource",
Signature · 1 line
2.25.1
#130 lore raw

PATCH v6 23/26 net/mlx5: optimize hash list table allocate on demand

In reply to Suanming Mou

Currently, all the hash list tables are allocated during start up. Since different applications may only use dedicated limited actions, optimized the hash list table allocate on demand will save initial memory.

This commit optimizes hash list table allocate on demand.

---
 drivers/net/mlx5/linux/mlx5_os.c   | 44 +----------------
 drivers/net/mlx5/mlx5_defs.h       |  6 +++
 drivers/net/mlx5/mlx5_flow_dv.c    | 79 ++++++++++++++++++++++++++++--
 drivers/net/mlx5/windows/mlx5_os.c |  2 -
 4 files changed, 82 insertions(+), 49 deletions(-)
drivers/net/mlx5/linux/mlx5_os.c +1 -43
index 969d265350..ec2f8a4aed 100644
@@ -50,8 +50,6 @@
 #include "mlx5_nl.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
-
 #ifndef HAVE_IBV_MLX5_MOD_MPW
 #define MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED (1 << 2)
 #define MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW (1 << 3)
@@ -312,46 +310,6 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 					      flow_dv_dest_array_clone_free_cb);
 	if (!sh->dest_array_list)
 		goto error;
-	/* Create tags hash list table. */
-	snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name);
-	sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE, false,
-					  false, sh, flow_dv_tag_create_cb,
-					  flow_dv_tag_match_cb,
-					  flow_dv_tag_remove_cb,
-					  flow_dv_tag_clone_cb,
-					  flow_dv_tag_clone_free_cb);
-	if (!sh->tag_table) {
-		DRV_LOG(ERR, "tags with hash creation failed.");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_hdr_modify", sh->ibdev_name);
-	sh->modify_cmds = mlx5_hlist_create(s, MLX5_FLOW_HDR_MODIFY_HTABLE_SZ,
-					    true, false, sh,
-					    flow_dv_modify_create_cb,
-					    flow_dv_modify_match_cb,
-					    flow_dv_modify_remove_cb,
-					    flow_dv_modify_clone_cb,
-					    flow_dv_modify_clone_free_cb);
-	if (!sh->modify_cmds) {
-		DRV_LOG(ERR, "hdr modify hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
-	snprintf(s, sizeof(s), "%s_encaps_decaps", sh->ibdev_name);
-	sh->encaps_decaps = mlx5_hlist_create(s,
-					      MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ,
-					      true, true, sh,
-					      flow_dv_encap_decap_create_cb,
-					      flow_dv_encap_decap_match_cb,
-					      flow_dv_encap_decap_remove_cb,
-					      flow_dv_encap_decap_clone_cb,
-					     flow_dv_encap_decap_clone_free_cb);
-	if (!sh->encaps_decaps) {
-		DRV_LOG(ERR, "encap decap hash creation failed");
-		err = ENOMEM;
-		goto error;
-	}
 #endif
 #ifdef HAVE_MLX5DV_DR
 	void *domain;
@@ -396,7 +354,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv)
 		goto error;
 	}
 #endif
-	if (!sh->tunnel_hub)
+	if (!sh->tunnel_hub && priv->config.dv_miss_info)
 		err = mlx5_alloc_tunnel_hub(sh);
 	if (err) {
 		DRV_LOG(ERR, "mlx5_alloc_tunnel_hub failed err=%d", err);
drivers/net/mlx5/mlx5_defs.h +6 -0
index ca67ce8213..fe86bb40d3 100644
@@ -188,6 +188,12 @@
 /* Size of the simple hash table for encap decap table. */
 #define MLX5_FLOW_ENCAP_DECAP_HTABLE_SZ (1 << 12)
 
+/* Size of the hash table for tag table. */
+#define MLX5_TAGS_HLIST_ARRAY_SIZE	(1 << 15)
+
+/* Size fo the hash table for SFT table. */
+#define MLX5_FLOW_SFT_HLIST_ARRAY_SIZE	4096
+
 /* Hairpin TX/RX queue configuration parameters. */
 #define MLX5_HAIRPIN_QUEUE_STRIDE 6
 #define MLX5_HAIRPIN_JUMBO_LOG_SIZE (14 + 2)
drivers/net/mlx5/mlx5_flow_dv.c +75 -4

119 lines not on this page · raw message

drivers/net/mlx5/windows/mlx5_os.c +0 -2
index a04f93e1d4..5da362a9d5 100644
@@ -30,8 +30,6 @@
 #include "mlx5_flow.h"
 #include "mlx5_devx.h"
 
-#define MLX5_TAGS_HLIST_ARRAY_SIZE 8192
-
 static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
 
 /* Spinlock for mlx5_shared_data allocation. */
Signature · 1 line
2.25.1
#131 lore raw

PATCH v6 24/26 net/mlx5: change memory release configuration

In reply to Suanming Mou

This commit changes the index pool memory release configuration to 0 when memory reclaim mode is not required.

---
 drivers/net/mlx5/mlx5.c | 2 ++
 1 file changed, 2 insertions(+)
drivers/net/mlx5/mlx5.c +2 -0
index f79a094a35..021a34dd4d 100644
@@ -798,6 +798,8 @@ mlx5_flow_ipool_create(struct mlx5_dev_ctx_shared *sh,
 		if (config->reclaim_mode) {
 			cfg.release_mem_en = 1;
 			cfg.per_core_cache = 0;
+		} else {
+			cfg.release_mem_en = 0;
 		}
 		sh->ipool[i] = mlx5_ipool_create(&cfg);
 	}
Signature · 1 line
2.25.1
#132 lore raw

PATCH v6 25/26 net/mlx5: optimize Rx queue match

In reply to Suanming Mou

As hrxq struct has the indirect table pointer, while matching the hrxq, better to use the hrxq indirect table instead of searching from the list.

This commit optimizes the hrxq indirect table matching.

---
 drivers/net/mlx5/mlx5_rxq.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)
drivers/net/mlx5/mlx5_rxq.c +7 -11
index 7893b3edd4..23685d7654 100644
@@ -2094,23 +2094,19 @@ mlx5_ind_table_obj_modify(struct rte_eth_dev *dev,
 }
 
 int
-mlx5_hrxq_match_cb(void *tool_ctx, struct mlx5_list_entry *entry, void *cb_ctx)
+mlx5_hrxq_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
+		   void *cb_ctx)
 {
-	struct rte_eth_dev *dev = tool_ctx;
 	struct mlx5_flow_cb_ctx *ctx = cb_ctx;
 	struct mlx5_flow_rss_desc *rss_desc = ctx->data;
 	struct mlx5_hrxq *hrxq = container_of(entry, typeof(*hrxq), entry);
-	struct mlx5_ind_table_obj *ind_tbl;
 
-	if (hrxq->rss_key_len != rss_desc->key_len ||
+	return (hrxq->rss_key_len != rss_desc->key_len ||
 	    memcmp(hrxq->rss_key, rss_desc->key, rss_desc->key_len) ||
-	    hrxq->hash_fields != rss_desc->hash_fields)
-		return 1;
-	ind_tbl = mlx5_ind_table_obj_get(dev, rss_desc->queue,
-					 rss_desc->queue_num);
-	if (ind_tbl)
-		mlx5_ind_table_obj_release(dev, ind_tbl, hrxq->standalone);
-	return ind_tbl != hrxq->ind_table;
+	    hrxq->hash_fields != rss_desc->hash_fields ||
+	    hrxq->ind_table->queues_n != rss_desc->queue_num ||
+	    memcmp(hrxq->ind_table->queues, rss_desc->queue,
+	    rss_desc->queue_num * sizeof(rss_desc->queue[0])));
 }
 
 /**
Signature · 1 line
2.25.1
#133 lore raw

PATCH v6 26/26 doc: add mlx5 multiple-thread flow insertion optimization

In reply to Suanming Mou

This commit adds the multiple-thread flow insertion optimization description.

---
 doc/guides/nics/mlx5.rst               | 5 +++++
 doc/guides/rel_notes/release_21_08.rst | 1 +
 2 files changed, 6 insertions(+)
doc/guides/nics/mlx5.rst +5 -0
index 5a3c4f198a..199beb2549 100644
@@ -448,6 +448,11 @@ Limitations
   - 256 ports maximum.
   - 4M connections maximum.
 
+- Multiple-thread flow insertion:
+
+  - In order to achieve best insertion rate, application should manage the flows on the rte-lcore.
+  - Better to configure ``reclaim_mem_mode`` as 0 to accelerate the flow object allocate and release with cache.
+
 Statistics
 ----------
 
doc/guides/rel_notes/release_21_08.rst +1 -0
index 6a902ef9ac..68496496e8 100644
@@ -88,6 +88,7 @@ New Features
 
   * Added support for meter hierarchy.
   * Added devargs options ``allow_duplicate_pattern``.
+  * Optimize multiple-thread flow insertion rate.
 
 * **Added support for Marvell CNXK crypto driver.**
 
Signature · 1 line
2.25.1
#134 lore raw

Re: [dpdk-dev] [PATCH v6 00/26] net/mlx5: insertion rate optimization

In reply to Suanming Mou

Hi,

Quoted 98 lines

-----Original Message-----
From: Suanming Mou <[email protected]>
Sent: Tuesday, July 13, 2021 11:45 AM
To: Slava Ovsiienko <[email protected]>; Matan Azrad <[email protected]>
Cc: Raslan Darawsheh <[email protected]>; Ori Kam <[email protected]>; [email protected]
Subject: [PATCH v6 00/26] net/mlx5: insertion rate optimization

This patch series optimize the flow insertion rate with adding local cache to index pool and list.

For object which wants efficient index allocate and free, local cache will be very helpful.

For index pool, two level cache is added, one as local and another as global. The global cache is able to save all the allocated index. That means all the allocated index will not be freed. Once the local cache is full, the extra index will be flushed to the global cache. Once local cache is empty, first try to fetch more index from global, if global is still empty, allocate new trunk and more index.

For list, sub local core list is introduced. The allocated objects will be added and released only from local list without any locks.
Only the objects need to be shared will be synced from global list.

---

v6: fix compilation issue in "common/mlx5: add list lcore share" patch.

v5: code rebase to the latest version.

v4:
- split the list utility to commom patch
- fix one ipool crash issue in mlx5_ipool_get_cache() function.
- reorganize the common and mlx5 patches.
- split the doc update to an individual patch.
- update and rebase to the latest version.

v3: fix Windows compilation issue

v2: add the list per-lcore cache optimization

---

Matan Azrad (9):
net/mlx5: optimize modify header action memory net/mlx5: remove cache term from the list utility net/mlx5: add per lcore cache to the list utility net/mlx5: minimize list critical sections net/mlx5: manage list cache entries release net/mlx5: relax the list utility atomic operations net/mlx5: allocate list memory by the create API common/mlx5: add per-lcore cache to hash list utility net/mlx5: move modify header allocator to ipool

Suanming Mou (17):
net/mlx5: allow limiting the index pool maximum index net/mlx5: add indexed pool local cache
net/mlx5: add index pool foreach define net/mlx5: support index pool non-lcore operations net/mlx5: replace flow list with index pool common/mlx5: move list utility to common common/mlx5: add list lcore share
common/mlx5: call list callbacks with context common/mlx5: allocate cache list memory individually common/mlx5: optimize cache list object memory common/mlx5: support list non-lcore operations net/mlx5: adjust the hash bucket size
net/mlx5: enable index pool per-core cache net/mlx5: optimize hash list table allocate on demand net/mlx5: change memory release configuration net/mlx5: optimize Rx queue match
doc: add mlx5 multiple-thread flow insertion optimization

 doc/guides/nics/mlx5.rst                |    5 +
 doc/guides/rel_notes/release_21_08.rst  |    1 +
 drivers/common/mlx5/linux/mlx5_glue.h   |    1 +
 drivers/common/mlx5/mlx5_common.h       |    2 +
 drivers/common/mlx5/mlx5_common_utils.c |  569 ++++++++---
drivers/common/mlx5/mlx5_common_utils.h |  294 ++++--
 drivers/common/mlx5/version.map         |    7 +
 drivers/net/mlx5/linux/mlx5_flow_os.h   |    3 +-
 drivers/net/mlx5/linux/mlx5_os.c        |  209 ++--
 drivers/net/mlx5/mlx5.c                 |   34 +-
 drivers/net/mlx5/mlx5.h                 |   46 +-
 drivers/net/mlx5/mlx5_defs.h            |   12 +-
 drivers/net/mlx5/mlx5_flow.c            |  308 +++---
 drivers/net/mlx5/mlx5_flow.h            |  209 ++--
 drivers/net/mlx5/mlx5_flow_dv.c         | 1206 +++++++++++++++--------
 drivers/net/mlx5/mlx5_rx.h              |   14 +-
 drivers/net/mlx5/mlx5_rxq.c             |  136 ++-
 drivers/net/mlx5/mlx5_trigger.c         |    8 +-
 drivers/net/mlx5/mlx5_utils.c           |  627 ++++++++----
 drivers/net/mlx5/mlx5_utils.h           |  255 ++---
 drivers/net/mlx5/windows/mlx5_os.c      |   11 +-
 21 files changed, 2543 insertions(+), 1414 deletions(-)
Quoted 2 lines

--
2.25.1

Series applied to next-net-mlx,

Kindest regards,
Raslan Darawsheh