mbuf: add fast free bulk function
PATCH 1/1 mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in rte_mbuf_raw_free() were moved to __rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.
The description of RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE was updated to include the remaining requirements, which were missing here.
And finally:
A new rte_mbuf_fast_free_bulk() inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then calls rte_mempool_put_bulk().
--- lib/ethdev/rte_ethdev.h | 6 ++++-- lib/mbuf/rte_mbuf.h | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-)
lib/ethdev/rte_ethdev.h +4 -2
@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)
lib/mbuf/rte_mbuf.h +37 -2
@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }
@@ -623,12 +627,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) ||RTE_MBUF_HAS_PINNED_EXTBUF (m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* Put a bulk of mbufs allocated from the same mempool back into the mempool. +* +* The caller must ensure that the mbufs come from the specified mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf **mbufs,unsigned int count) +{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + } + +rte_mempool_put_bulk (mp, (void **)mbufs, count); +} +/** * The packet mbuf constructor. *
Signature · 1 line
2.43.0
PATCH v2 1/1 mbuf: add fast free bulk function
mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in rte_mbuf_raw_free() were moved to __rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.
The description of RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE was updated to include the remaining requirements, which were missing here.
And finally:
A new rte_mbuf_fast_free_bulk() inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then calls rte_mempool_put_bulk().
---
v2:
* Fixed missing inline.
--- lib/ethdev/rte_ethdev.h | 6 ++++-- lib/mbuf/rte_mbuf.h | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-)
lib/ethdev/rte_ethdev.h +4 -2
@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)
lib/mbuf/rte_mbuf.h +37 -2
@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }
@@ -623,12 +627,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) ||RTE_MBUF_HAS_PINNED_EXTBUF (m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* Put a bulk of mbufs allocated from the same mempool back into the mempool. +* +* The caller must ensure that the mbufs come from the specified mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +static __rte_always_inline void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf **mbufs,unsigned int count) +{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + } + +rte_mempool_put_bulk (mp, (void **)mbufs, count); +} +/** * The packet mbuf constructor. *
Signature · 1 line
2.43.0
Quoted 16 lines · On Tue, 14 Jan 2025 16:39:51 +0000 Morten Brørup <[email protected]>…
mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.
Quoted 7 lines
And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().Signed-off-by Morten Brørup <[email protected]>
Since it is a new function it should be marked experimental for now.
Quoted 92 lines · Morten Brørup
mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().Signed-off-by Morten Brørup <[email protected]>---
v2:
* Fixed missing inline.--- lib/ethdev/rte_ethdev.h | 6 ++++-- lib/mbuf/rte_mbuf.h | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h +4
-2@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)lib/mbuf/rte_mbuf.h +23
-2@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }@@ -623,12 +627,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) ||RTE_MBUF_HAS_PINNED_EXTBUF (m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* Put a bulk of mbufs allocated from the same mempool back into the mempool. +* +* The caller must ensure that the mbufs come from the specified mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size.
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 662
Quoted 10 lines
lib/mbuf/rte_mbuf.h +10
-0+*/ +static __rte_always_inline void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf **mbufs,unsigned int count) +{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + }
Is there some way to avoid executing a loop in non-debug mode? Like the following or other better way
#ifdef RTE_LIBRTE_MBUF_DEBUG
{
for (unsigned int idx = 0; idx < count; idx++) {
const struct rte_mbuf *m = mbufs[idx];
RTE_ASSERT(m != NULL);
RTE_ASSERT(m->pool == mp);
__rte_mbuf_raw_sanity_check(m);
}
}#endif
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 664
Quoted 2 lines
lib/mbuf/rte_mbuf.h +2
-0+ +rte_mempool_put_bulk (mp, (void **)mbufs, count);
Can the mp be obtained from the mbuf?
Quoted 5 lines
lib/mbuf/rte_mbuf.h +2
-0+} +/** * The packet mbuf constructor. *
Quoted 134 lines
From: huangdengdui [mailto:[email protected]]
Sent: Wednesday, 15 January 2025 07.52Quoted 3 lines · Morten Brørup
mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain
requirements
Quoted 5 lines
to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were
Quoted 2 lines
moved to
__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer wasenhanced;
Quoted 8 lines
it now also asserts that the referenced pinned external buffer has refcnt == 1.
The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for thebenefit
Quoted 2 lines
of ethdev drivers supporting fast release of mbufs. It asserts these requirements and that the mbufs belong to the
specified
Quoted 16 lines
mempool, and then calls
rte_mempool_put_bulk().Signed-off-by Morten Brørup <[email protected]>---
v2:
* Fixed missing inline.--- lib/ethdev/rte_ethdev.h | 6 ++++-- lib/mbuf/rte_mbuf.h | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h +0
-0lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1615
Quoted 4 lines
lib/ethdev/rte_ethdev.h +0
-1@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from
lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1615
Quoted 2 lines
lib/ethdev/rte_ethdev.h +1
-1-* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool,
lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1616
Quoted 1 line
lib/ethdev/rte_ethdev.h +1
-0+* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by
rte_pktmbuf_prefree_seg().lib/mbuf/rte_mbuf.h __rte_mbuf_raw_sanity_check(__rte_unused const
Quoted 10 lines
lib/ethdev/rte_ethdev.h +2
-0+* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)lib/mbuf/rte_mbuf.h +0
-0@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused conststruct
rte_mbuf*m)lib/mbuf/rte_mbuf.h static inline struct rte_mbuf
Quoted 11 lines
lib/mbuf/rte_mbuf.h +4
-0RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }@@ -623,12 +627,43 @@ static inline struct rte_mbuf*rte_mbuf_raw_alloc(struct
rte_mempool*mp)lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 5 lines
lib/mbuf/rte_mbuf.h +0
-2static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { - RTE_ASSERT(!RTE_MBUF_CLONED(m) && - (!RTE_MBUF_HAS_EXTBUF(m) ||
RTE_MBUF_HAS_PINNED_EXTBUF(m)));lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 6 lines
lib/mbuf/rte_mbuf.h +2
-0__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* Put a bulk of mbufs allocated from the same mempool back into the mempool.
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf · line 637
Quoted 2 lines
lib/mbuf/rte_mbuf.h +2
-0+* +* The caller must ensure that the mbufs come from the specified mempool,
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf · line 638
Quoted 1 line
lib/mbuf/rte_mbuf.h +1
-0+* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by
Quoted 7 lines
lib/mbuf/rte_mbuf.h +7
-0+* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 10 lines
lib/mbuf/rte_mbuf.h +10
-0+* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +static __rte_always_inline void +rte_mbuf_fast_free_bulk(struct rte_mempool *mp, struct rte_mbuf**mbufs, unsigned int count)
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 662
Quoted 7 lines
lib/mbuf/rte_mbuf.h +7
-0+{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + }Is there some way to avoid executing a loop in non-debug mode? Like the following or other better way
#ifdef RTE_LIBRTE_MBUF_DEBUG { for (unsigned int idx = 0; idx < count; idx++) { const struct rte_mbuf *m = mbufs[idx]; RTE_ASSERT(m != NULL); RTE_ASSERT(m->pool == mp); __rte_mbuf_raw_sanity_check(m); } }
Quoted 1 line
#endif
The loop is already omitted in non-debug mode:RTE_ASSERT() [1] is omitted unless RTE_ENABLE_ASSERT is set. __rte_mbuf_raw_sanity_check() [2] consist of some RTE_ASSERTs and __rte_mbuf_sanity_check(). __rte_mbuf_sanity_check() [3] is omitted unless RTE_LIBRTE_MBUF_DEBUG is set.
So the compiler will detect that the loop has no effect, and optimize away the loop.
[1]: https://elixir.bootlin.com/dpdk/v24.11.1/source/lib/eal/include/rte_debug.h#L46
[2]: https://elixir.bootlin.com/dpdk/v24.11.1/source/lib/mbuf/rte_mbuf.h#L566
[3]: https://elixir.bootlin.com/dpdk/v24.11.1/source/lib/mbuf/rte_mbuf.h#L348
Quoted 5 lines
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 664
Quoted 3 lines
lib/mbuf/rte_mbuf.h +2
-0+ +rte_mempool_put_bulk (mp, (void **)mbufs, count);Can the mp be obtained from the mbuf?
Passing "mp" as a parameter avoids a potential CPU cache miss by not dereferencing the first mbuf, if the driver/application already has the mempool pointer readily available (and hot in the CPU cache) from somewhere else. If the driver/or application doesn't have the mempool pointer readily available, it can obtain it from the mbuf when calling this function.
And as a bonus side effect, passing "mp" as a parameter allows calling the function with count=0 without special handling inside the function.
Obviously, if the driver/application gets "mp" from mbuf[0]->pool, it needs to first check that count>0; but that would be the situation for the driver/application whenever it accesses an mbuf array, regardless what it is doing with that array.
huangdengdui
Acked-byQuoted 136 lines · Morten Brørup
Quoted 134 lines
From: huangdengdui [mailto:[email protected]]
Sent: Wednesday, 15 January 2025 07.52Quoted 3 lines · Morten Brørup
mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain
requirements
Quoted 5 lines
to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were
Quoted 2 lines
moved to
__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer wasenhanced;
Quoted 8 lines
it now also asserts that the referenced pinned external buffer has refcnt == 1.
The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for thebenefit
Quoted 2 lines
of ethdev drivers supporting fast release of mbufs. It asserts these requirements and that the mbufs belong to the
specified
Quoted 16 lines
mempool, and then calls
rte_mempool_put_bulk().Signed-off-by Morten Brørup <[email protected]>---
v2:
* Fixed missing inline.--- lib/ethdev/rte_ethdev.h | 6 ++++-- lib/mbuf/rte_mbuf.h | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h +0
-0lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1615
Quoted 4 lines
lib/ethdev/rte_ethdev.h +0
-1@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from
lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1615
Quoted 2 lines
lib/ethdev/rte_ethdev.h +1
-1-* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool,
lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1616
Quoted 1 line
lib/ethdev/rte_ethdev.h +1
-0+* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by
rte_pktmbuf_prefree_seg().lib/mbuf/rte_mbuf.h __rte_mbuf_raw_sanity_check(__rte_unused const
Quoted 10 lines
lib/ethdev/rte_ethdev.h +2
-0+* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)lib/mbuf/rte_mbuf.h +0
-0@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused conststruct
rte_mbuf*m)lib/mbuf/rte_mbuf.h static inline struct rte_mbuf
Quoted 11 lines
lib/mbuf/rte_mbuf.h +4
-0RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }@@ -623,12 +627,43 @@ static inline struct rte_mbuf*rte_mbuf_raw_alloc(struct
rte_mempool*mp)lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 5 lines
lib/mbuf/rte_mbuf.h +0
-2static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { - RTE_ASSERT(!RTE_MBUF_CLONED(m) && - (!RTE_MBUF_HAS_EXTBUF(m) ||
RTE_MBUF_HAS_PINNED_EXTBUF(m)));lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 6 lines
lib/mbuf/rte_mbuf.h +2
-0__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* Put a bulk of mbufs allocated from the same mempool back into the mempool.
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf · line 637
Quoted 2 lines
lib/mbuf/rte_mbuf.h +2
-0+* +* The caller must ensure that the mbufs come from the specified mempool,
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf · line 638
Quoted 1 line
lib/mbuf/rte_mbuf.h +1
-0+* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by
Quoted 7 lines
lib/mbuf/rte_mbuf.h +7
-0+* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 10 lines
lib/mbuf/rte_mbuf.h +10
-0+* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +static __rte_always_inline void +rte_mbuf_fast_free_bulk(struct rte_mempool *mp, struct rte_mbuf**mbufs, unsigned int count)
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 662
Quoted 7 lines
lib/mbuf/rte_mbuf.h +7
-0+{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + }Is there some way to avoid executing a loop in non-debug mode? Like the following or other better way
#ifdef RTE_LIBRTE_MBUF_DEBUG { for (unsigned int idx = 0; idx < count; idx++) { const struct rte_mbuf *m = mbufs[idx]; RTE_ASSERT(m != NULL); RTE_ASSERT(m->pool == mp); __rte_mbuf_raw_sanity_check(m); } }Quoted 1 line
#endif
Quoted 6 lines
The loop is already omitted in non-debug mode:
RTE_ASSERT()[1] is omitted unlessRTE_ENABLE_ASSERTis set.__rte_mbuf_raw_sanity_check()[2] consist of someRTE_ASSERTsand__rte_mbuf_sanity_check().__rte_mbuf_sanity_check()[3] is omitted unlessRTE_LIBRTE_MBUF_DEBUGis set.So the compiler will detect that the loop has no effect, and optimize away the loop.
Okay, you're right, I compiled the code and this loop code didn't generate any instructions after the compiler optimization was enabled.
Quoted 11 lines
[1]: https://elixir.bootlin.com/dpdk/v24.11.1/source/lib/eal/include/rte_debug.h#L46
[2]: https://elixir.bootlin.com/dpdk/v24.11.1/source/lib/mbuf/rte_mbuf.h#L566
[3]: https://elixir.bootlin.com/dpdk/v24.11.1/source/lib/mbuf/rte_mbuf.h#L348Quoted 5 lines
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 664
Quoted 3 lines
lib/mbuf/rte_mbuf.h +2
-0+ +rte_mempool_put_bulk (mp, (void **)mbufs, count);Can the mp be obtained from the mbuf?
Quoted 7 lines
Passing "mp" as a parameter avoids a potential CPU cache miss by not dereferencing the first mbuf, if the driver/application already has the mempool pointer readily available (and hot in the CPU cache) from somewhere else. If the driver/or application doesn't have the mempool pointer readily available, it can obtain it from the mbuf when calling this function.
And as a bonus side effect, passing "mp" as a parameter allows calling the function with count=0 without special handling inside the function.
Obviously, if the driver/application gets "mp" from mbuf[0]->pool, it needs to first check that count>0; but that would be the situation for the driver/application whenever it accesses an mbuf array, regardless what it is doing with that array.
Okay, It's better this way.
Quoted 115 lines · On Wed, 15 Jan 2025 14:52:16 +0800 huangdengdui <[email protected]> wrote:
Quoted 92 lines · Morten Brørup
mbuf: add fast free bulk function
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().Signed-off-by Morten Brørup <[email protected]>---
v2:
* Fixed missing inline.--- lib/ethdev/rte_ethdev.h | 6 ++++-- lib/mbuf/rte_mbuf.h | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h +4
-2@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)lib/mbuf/rte_mbuf.h +23
-2@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }@@ -623,12 +627,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) ||RTE_MBUF_HAS_PINNED_EXTBUF (m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* Put a bulk of mbufs allocated from the same mempool back into the mempool. +* +* The caller must ensure that the mbufs come from the specified mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 662
Quoted 10 lines
lib/mbuf/rte_mbuf.h +10
-0+*/ +static __rte_always_inline void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf **mbufs,unsigned int count) +{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + }Is there some way to avoid executing a loop in non-debug mode? Like the following or other better way
#ifdef RTE_LIBRTE_MBUF_DEBUG { for (unsigned int idx = 0; idx < count; idx++) { const struct rte_mbuf *m = mbufs[idx]; RTE_ASSERT(m != NULL); RTE_ASSERT(m->pool == mp); __rte_mbuf_raw_sanity_check(m); } }
Quoted 1 line
#endif
I suspect compiler will optimize it way to nothing and drop the loop. Use godbolt to see.
Quoted 4 lines
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) · line 664
Quoted 2 lines
lib/mbuf/rte_mbuf.h +2
-0+ +rte_mempool_put_bulk (mp, (void **)mbufs, count);Can the mp be obtained from the mbuf?
Yes, see rte_pktmbuf_free
Quoted 6 lines
Quoted 6 lines
lib/mbuf/rte_mbuf.h +2
-0+} +/** * The packet mbuf constructor. *
PATCH v3 1/1 mbuf: add fast free bulk and raw alloc bulk functions
In reply to Morten BrørupWhen putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in rte_mbuf_raw_free() were moved to __rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.
The description of RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE was updated to include the remaining requirements, which were missing here.
And finally:
A new rte_mbuf_fast_free_bulk() inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then calls rte_mempool_put_bulk().
For symmetry, a new rte_mbuf_raw_alloc_bulk() inline function was also added.
---
v2:
* Fixed missing inline.
v3:
* Fixed missing experimental warning. (Stephen)
* Added raw alloc bulk function.
--- lib/ethdev/rte_ethdev.h | 6 ++-- lib/mbuf/rte_mbuf.h | 80 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 4 deletions(-)
lib/ethdev/rte_ethdev.h +4 -2
@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)
lib/mbuf/rte_mbuf.h +78 -2
102 lines not on this page · raw message
Signature · 1 line
2.43.0
Re: [PATCH v3] mbuf: add fast free bulk and raw alloc bulk functions
LGTM
Quoted 25 lines · Morten Brørup
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().For symmetry, a new
rte_mbuf_raw_alloc_bulk()inline function was also added.Signed-off-by Morten Brørup <[email protected]>Acked-by Dengdui Huang <[email protected]>
RE: [PATCH v3] mbuf: add fast free bulk and raw alloc bulk functions
In reply to Morten BrørupPS. It seems we should consider replacing the rte_mempool_put_bulk() with the new wrapper rte_mbuf_fast_free_bulk() in the drivers.
Quoted 185 lines
-----Original Message-----
From: Morten Brørup <[email protected]>
Sent: Tuesday, January 21, 2025 3:40 PM
To: NBU-Contact-Thomas Monjalon (EXTERNAL) <[email protected]>; Ferruh Yigit <[email protected]>; Andrew Rybchenko <[email protected]>; Slava Ovsiienko <[email protected]>; Shahaf Shuler <[email protected]>; Olivier Matz <[email protected]>
Cc: [email protected]; Morten Brørup <[email protected]>; Dengdui Huang <[email protected]>
Subject: [PATCH v3] mbuf: add fast free bulk and raw alloc bulk functionsWhen putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().For symmetry, a new
rte_mbuf_raw_alloc_bulk()inline function was also added.Signed-off-by Morten Brørup <[email protected]>Acked-by Dengdui Huang <[email protected]>---
v2:
* Fixed missing inline.
v3:
* Fixed missing experimental warning. (Stephen)
* Added raw alloc bulk function.--- lib/ethdev/rte_ethdev.h | 6 ++-- lib/mbuf/rte_mbuf.h | 80 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h index renamed from lib/ethdev/rte_ethdev.h +0
-0
1f71cad244..e9267fca79100644lib/ethdev/rte_ethdev.h +0
-1@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from
- * the same mempool and has refcnt = 1.
+ * When set application must guarantee that per-queue all mbufs come
+ from the same mempool,
+ * are direct, have refcnt=1, next=NULL and nb_segs=1, as done byrte_pktmbuf_prefree_seg().
+ *
+ * @seerte_mbuf_fast_free_bulk()
*/
#defineRTE_ETH_TX_OFFLOAD_MBUF_FAST_FREERTE_BIT64(16)
#defineRTE_ETH_TX_OFFLOAD_SECURITYRTE_BIT64(17)lib/mbuf/rte_mbuf.h index renamed from lib/mbuf/rte_mbuf.h +0
-0
0d2e0e64b3..1e40e7fcf7100644lib/mbuf/rte_mbuf.h +7
-0@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m) RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }@@ -606,6 +610,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)return ret.m; } +/** +* @warning +* @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
+ *
+ * Allocate a bulk of uninitialized mbufs from mempool *mp*.
+ *
+ * This function can be used by PMDs (especially in RX functions) to
+ * allocate a bulk of uninitialized mbufs. The driver is responsible of
+ * initializing all the required fields. Seerte_pktmbuf_reset().
+ * For standard needs, preferrte_pktmbuf_alloc_bulk().
+ *
+ * The caller can expect that the following fields of the mbuflib/mbuf/rte_mbuf.h +29
-2+structure +* are initialized: buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1, +* next=NULL, pool, priv_size. The other fields must be initialized +* by the caller. +* +* @param mp +* The mempool from which mbufs are allocated. +* @param mbufs +* Array of pointers to mbufs. +* @param count +* Array size. +* @return +* - 0: Success. +* - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved. +*/ +__rte_experimental +static __rte_always_inline int +rte_mbuf_raw_alloc_bulk (struct rte_mempool *mp,struct rte_mbuf +**mbufs,unsigned int count) { +int rc =rte_mempool_get_bulk (mp, (void **)mbufs, count); +if (likely (rc ==0 )) +for (unsigned int idx =0 ; idx < count; idx++) +__rte_mbuf_raw_sanity_check (mbufs[idx]); +return rc; +} +/** * Put mbuf back into its original mempool. * @@ -623,12 +664,47 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline void rte_mbuf_raw_free(struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) || RTE_MBUF_HAS_PINNED_EXTBUF(m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* @warning +* @b EXPERIMENTAL: This API may change, or be removed, without prior notice.
+ *
+ * Put a bulk of mbufs allocated from the same mempool back into the mempool.
+ *
+ * The caller must ensure that the mbufs come from the specifiedlib/mbuf/rte_mbuf.h +31
-1+mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, +nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +__rte_experimental +static __rte_always_inline void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf +**mbufs,unsigned int count) { +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + } + +rte_mempool_put_bulk (mp, (void **)mbufs, count); } +/** * The packet mbuf constructor. * -- 2.43.0
RE: [PATCH v3] mbuf: add fast free bulk and raw alloc bulk functions
Quoted 7 lines
From: Slava Ovsiienko [mailto:[email protected]]
Sent: Friday, 24 January 2025 11.23PS. It seems we should consider replacing the
rte_mempool_put_bulk()with the new wrapperrte_mbuf_fast_free_bulk()in the drivers.
Agree. And the individual drivers should be carefully tested - with RTE_ENABLE_ASSERT and RTE_LIBRTE_MBUF_DEBUG enabled - by the developers updating their drivers. I think it will be easier for the driver developers if we merge this patch stand-alone first.
Quoted 192 lines
Quoted 5 lines
From: Morten Brørup <[email protected]>
Sent: Tuesday, January 21, 2025 3:40 PMWhen putting an mbuf back into its mempool, there are certain
requirements
Quoted 5 lines
to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were
Quoted 2 lines
moved to
__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer wasenhanced; it
Quoted 1 line
now also asserts that the referenced pinned external buffer has
refcnt == 1.
Quoted 6 lines
The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for thebenefit of
Quoted 2 lines
ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to thespecified
Quoted 3 lines
mempool, and then calls
rte_mempool_put_bulk().For symmetry, a new
rte_mbuf_raw_alloc_bulk()inline function wasalso
Quoted 10 lines
added.
Signed-off-by Morten Brørup <[email protected]>Acked-by Dengdui Huang <[email protected]>---
v2:
* Fixed missing inline.
v3:
* Fixed missing experimental warning. (Stephen)
* Added raw alloc bulk function.Quoted 3 lines
--- lib/ethdev/rte_ethdev.h | 6 ++-- lib/mbuf/rte_mbuf.h | 80+++++++++++++++++++++++++++++++++++++++--
lib/ethdev/rte_ethdev.h struct rte_eth_conf { · line 1615
Quoted 11 lines
2 files changed, 82 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h index renamed from lib/ethdev/rte_ethdev.h +0
-0
1f71cad244..e9267fca79100644lib/ethdev/rte_ethdev.h +0
-1@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes
Quoted 28 lines
from
- * the same mempool and has refcnt = 1.
+ * When set application must guarantee that per-queue all mbufs come
+ from the same mempool,
+ * are direct, have refcnt=1, next=NULL and nb_segs=1, as done byrte_pktmbuf_prefree_seg().
+ *
+ * @seerte_mbuf_fast_free_bulk()
*/
#defineRTE_ETH_TX_OFFLOAD_MBUF_FAST_FREERTE_BIT64(16)
#defineRTE_ETH_TX_OFFLOAD_SECURITYRTE_BIT64(17)lib/mbuf/rte_mbuf.h index renamed from lib/mbuf/rte_mbuf.h +0
-0
0d2e0e64b3..1e40e7fcf7100644lib/mbuf/rte_mbuf.h +4
-0@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m) RTE_ASSERT(rte_mbuf_refcnt_read(m) == 1);RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }lib/mbuf/rte_mbuf.h static inline struct rte_mbuf · line 616
Quoted 7 lines
lib/mbuf/rte_mbuf.h +3
-0@@ -606,6 +610,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)return ret.m; } +/** +* @warning +* @b EXPERIMENTAL: This API may change, or be removed, without prior
Quoted 6 lines
notice.
+ *
+ * Allocate a bulk of uninitialized mbufs from mempool *mp*.
+ *
+ * This function can be used by PMDs (especially in RX functions) to
+ * allocate a bulk of uninitialized mbufs. The driver is responsibleof
Quoted 6 lines
+ * initializing all the required fields. See
rte_pktmbuf_reset().
+ * For standard needs, preferrte_pktmbuf_alloc_bulk().
+ *
+ * The caller can expect that the following fields of the mbuf +structure
+ * are initialized:buf_addr,buf_iova,buf_len, refcnt=1,nb_segs=1,
Quoted 4 lines
+ * next=NULL, pool,
priv_size. The other fields must be initialized
+ * by the caller.
+ *
+ * @param mpQuoted 8 lines
+ * The mempool from which mbufs are allocated.
+ * @param mbufs
+ * Array of pointers to mbufs.
+ * @param count
+ * Array size.
+ * @return
+ * - 0: Success.
+ * --ENOENT: Not enough entries in the mempool; no mbufs areretrieved.
Quoted 16 lines
+ */
lib/mbuf/rte_mbuf.h +11
-0+__rte_experimental +static __rte_always_inline int +rte_mbuf_raw_alloc_bulk (struct rte_mempool *mp,struct rte_mbuf +**mbufs,unsigned int count) { +int rc =rte_mempool_get_bulk (mp, (void **)mbufs, count); +if (likely (rc ==0 )) +for (unsigned int idx =0 ; idx < count; idx++) +__rte_mbuf_raw_sanity_check (mbufs[idx]); +return rc; +} +/** * Put mbuf back into its original mempool. * lib/mbuf/rte_mbuf.h static inline struct rte_mbuf · line 664
Quoted 1 line
lib/mbuf/rte_mbuf.h +0
-0@@ -623,12 +664,47 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static
__rte_always_inlinelib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 11 lines
lib/mbuf/rte_mbuf.h +3
-2void rte_mbuf_raw_free (struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) || RTE_MBUF_HAS_PINNED_EXTBUF(m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* @warning +* @b EXPERIMENTAL: This API may change, or be removed, without prior
Quoted 42 lines
notice.
+ *
+ * Put a bulk of mbufs allocated from the same mempool back into the mempool.
+ *
+ * The caller must ensure that the mbufs come from the specifiedlib/mbuf/rte_mbuf.h +31
-1+mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, +nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +__rte_experimental +static __rte_always_inline void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf +**mbufs,unsigned int count) { +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp); +__rte_mbuf_raw_sanity_check (m); + } + +rte_mempool_put_bulk (mp, (void **)mbufs, count); } +/** * The packet mbuf constructor. * -- 2.43.0
RE: [PATCH v3] mbuf: add fast free bulk and raw alloc bulk functions
In reply to Morten BrørupQuoted 154 lines
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_fast_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs.
It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().For symmetry, a new
rte_mbuf_raw_alloc_bulk()inline function was also added.Signed-off-by Morten Brørup <[email protected]>Acked-by Dengdui Huang <[email protected]>---
v2:
* Fixed missing inline.
v3:
* Fixed missing experimental warning. (Stephen)
* Added raw alloc bulk function.--- lib/ethdev/rte_ethdev.h | 6 ++-- lib/mbuf/rte_mbuf.h | 80 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 4 deletions(-)lib/ethdev/rte_ethdev.h +4
-2@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_fast_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)lib/mbuf/rte_mbuf.h +72
-2@@ -568,6 +568,10 @@ __rte_mbuf_raw_sanity_check(__rte_unused const struct rte_mbuf *m)RTE_ASSERT (rte_mbuf_refcnt_read (m) ==1 );RTE_ASSERT (m->next ==NULL );RTE_ASSERT (m->nb_segs ==1 ); +RTE_ASSERT (!RTE_MBUF_CLONED (m)); +RTE_ASSERT (!RTE_MBUF_HAS_EXTBUF (m) || + (RTE_MBUF_HAS_PINNED_EXTBUF (m) && +rte_mbuf_ext_refcnt_read (m->shinfo) ==1 ));__rte_mbuf_sanity_check (m,0 ); }@@ -606,6 +610,43 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)return ret.m; } +/** +* @warning +* @b EXPERIMENTAL: This API may change, or be removed, without prior notice. +* +* Allocate a bulk of uninitialized mbufs from mempool *mp*. +* +* This function can be used by PMDs (especially in RX functions) to +* allocate a bulk of uninitialized mbufs. The driver is responsible of +* initializing all the required fields. See rte_pktmbuf_reset(). +* For standard needs, prefer rte_pktmbuf_alloc_bulk(). +* +* The caller can expect that the following fields of the mbuf structure +* are initialized: buf_addr, buf_iova, buf_len, refcnt=1, nb_segs=1, +* next=NULL, pool, priv_size. The other fields must be initialized +* by the caller. +* +* @param mp +* The mempool from which mbufs are allocated. +* @param mbufs +* Array of pointers to mbufs. +* @param count +* Array size. +* @return +* - 0: Success. +* - -ENOENT: Not enough entries in the mempool; no mbufs are retrieved. +*/ +__rte_experimental +static __rte_always_inline int +rte_mbuf_raw_alloc_bulk (struct rte_mempool *mp,struct rte_mbuf **mbufs,unsigned int count) +{ +int rc =rte_mempool_get_bulk (mp, (void **)mbufs, count); +if (likely (rc ==0 )) +for (unsigned int idx =0 ; idx < count; idx++) +__rte_mbuf_raw_sanity_check (mbufs[idx]); +return rc; +} +/** * Put mbuf back into its original mempool. * @@ -623,12 +664,47 @@ static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp) static __rte_always_inline voidrte_mbuf_raw_free (struct rte_mbuf *m) { -RTE_ASSERT (!RTE_MBUF_CLONED (m) && - (!RTE_MBUF_HAS_EXTBUF (m) ||RTE_MBUF_HAS_PINNED_EXTBUF (m)));__rte_mbuf_raw_sanity_check (m);rte_mempool_put (m->pool, m); } +/** +* @warning +* @b EXPERIMENTAL: This API may change, or be removed, without prior notice. +* +* Put a bulk of mbufs allocated from the same mempool back into the mempool. +* +* The caller must ensure that the mbufs come from the specified mempool, +* are direct and properly reinitialized (refcnt=1, next=NULL, nb_segs=1), as done by +* rte_pktmbuf_prefree_seg(). +* +* This function should be used with care, when optimization is +* required. For standard needs, prefer rte_pktmbuf_free_bulk(). +* +* @see RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE +* +* @param mp +* The mempool to which the mbufs belong. +* @param mbufs +* Array of pointers to packet mbufs. +* The array must not contain NULL pointers. +* @param count +* Array size. +*/ +__rte_experimental +static __rte_always_inline void +rte_mbuf_fast_free_bulk (struct rte_mempool *mp,struct rte_mbuf **mbufs,unsigned int count) +{ +for (unsigned int idx =0 ; idx < count; idx++) { +const struct rte_mbuf *m = mbufs[idx]; +RTE_ASSERT (m !=NULL ); +RTE_ASSERT (m->pool == mp);
lib/mbuf/rte_mbuf.h static inline struct rte_mbuf *rte_mbuf_raw_alloc(struct rte_mempool *mp)
Quoted 10 lines
lib/mbuf/rte_mbuf.h +6
-1+__rte_mbuf_raw_sanity_check (m); + } + +rte_mempool_put_bulk (mp, (void **)mbufs, count); +} +/** * The packet mbuf constructor. * --
Quoted 1 line
2.43.0
PATCH v4 1/1 mbuf: add raw free and alloc bulk functions
In reply to Morten BrørupWhen putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in rte_mbuf_raw_free() were moved to __rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.
The description of RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE was updated to include the remaining requirements, which were missing here.
And finally:
A new rte_mbuf_raw_free_bulk() inline function was added for the benefit of ethdev drivers supporting fast release of mbufs. It asserts these requirements and that the mbufs belong to the specified mempool, and then calls rte_mempool_put_bulk().
For symmetry, a new rte_mbuf_raw_alloc_bulk() inline function was also added.
---
v4:
* Renamed rte_mbuf_fast_free_bulk() to rte_mbuf_raw_free_bulk().
* Updated patch title to reflect behavior of new functions instead of intended use case.
v3:
* Fixed missing experimental warning. (Stephen)
* Added raw alloc bulk function.
v2:
* Fixed missing inline.
--- lib/ethdev/rte_ethdev.h | 6 ++-- lib/mbuf/rte_mbuf.h | 80 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 4 deletions(-)
lib/ethdev/rte_ethdev.h +4 -2
@@ -1612,8 +1612,10 @@ struct rte_eth_conf {#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS RTE_BIT64(15)/** * Device supports optimization for fast release of mbufs. -* When set application must guarantee that per-queue all mbufs comes from -* the same mempool and has refcnt = 1. +* When set application must guarantee that per-queue all mbufs come from the same mempool, +* are direct, have refcnt=1, next=NULL and nb_segs=1, as done by rte_pktmbuf_prefree_seg(). +* +* @see rte_mbuf_raw_free_bulk() */ #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE RTE_BIT64(16)#define RTE_ETH_TX_OFFLOAD_SECURITY RTE_BIT64(17)
lib/mbuf/rte_mbuf.h +78 -2
102 lines not on this page · raw message
Signature · 1 line
2.43.0
Thomas Monjalon
AppliedRe: [PATCH v4] mbuf: add raw free and alloc bulk functions
30/01/2025 15:49, Morten Brørup:
Quoted 20 lines
When putting an mbuf back into its mempool, there are certain requirements to the mbuf. Specifically, some of its fields must be initialized.
These requirements are in fact invariants about free mbufs, held in mempools, and thus also apply when allocating an mbuf from a mempool. With this in mind, the additional assertions in
rte_mbuf_raw_free()were moved to__rte_mbuf_raw_sanity_check().
Furthermore, the assertion regarding pinned external buffer was enhanced; it now also asserts that the referenced pinned external buffer has refcnt == 1.The description of
RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREEwas updated to include the remaining requirements, which were missing here.And finally:
A newrte_mbuf_raw_free_bulk()inline function was added for the benefit of ethdev drivers supporting fast release of mbufs. It asserts these requirements and that the mbufs belong to the specified mempool, and then callsrte_mempool_put_bulk().
Quoted 8 lines
For symmetry, a new
rte_mbuf_raw_alloc_bulk()inline function was also added.Signed-off-by Morten Brørup <[email protected]>Acked-by Dengdui Huang <[email protected]>Acked-by Chengwen Feng <[email protected]>
Applied, thanks.