RCU integration with LPM library
Subsystems (2)
Re: [dpdk-dev] [PATCH v3 2/3] lib/rcu: add resource reclamation APIs
In reply to Honnappa NagarahalliHi Honnappa,
Quoted 489 lines · Honnappa Nagarahalli
Hi Vladimir,
Apologies for the delayed response, I had to run few experiments.<snip>
Quoted 484 lines
Hi Honnappa,
Quoted 46 lines · Honnappa Nagarahalli
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- app/test/test_rcu_qsbr.c | 291 ++++++++++++++++++++++++++++- lib/librte_rcu/meson.build | 2 + lib/librte_rcu/rte_rcu_qsbr.c | 185 ++++++++++++++++++ lib/librte_rcu/rte_rcu_qsbr.h | 169 +++++++++++++++++ lib/librte_rcu/rte_rcu_qsbr_pvt.h | 46 +++++ lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 700 insertions(+), 3 deletions(-)create mode 100644
lib/librte_rcu/rte_rcu_qsbr_pvt.happ/test/test_rcu_qsbr.c index renamed from app/test/test_rcu_qsbr.c +0
-0
d1b9e46a2..3a6815243100644app/test/test_rcu_qsbr.c +5
-1@@ -1,8 +1,9 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2019 Arm Limited */ #include <stdio.h> +#include <string.h> #include <rte_pause.h> #include <rte_rcu_qsbr.h> #include <rte_hash.h> @@ -33,6 +34,7 @@ static uint32_t *keys;#define COUNTER_VALUE 4096static uint32_t *hash_data[RTE_MAX_LCORE ][TOTAL_ENTRY ];static uint8_t writer_done; +static uint8_t cb_failed;static struct rte_rcu_qsbr *t[RTE_MAX_LCORE ];struct rte_hash *h[RTE_MAX_LCORE ];@@ -582,6 +584,269 @@ test_rcu_qsbr_thread_offline(void)return 0 ; } +static void +rte_rcu_qsbr_test_free_resource (void *p,void *e) {app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void) · line 598
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+if (p !=NULL && e !=NULL ) { +printf ("%s: Test failed\n" , __func__); + cb_failed =1 ; + } +} + +/* +* rte_rcu_qsbr_dq_create: create a queue used to store the data +structure +* elements that can be freed later. This queue is referred to as 'defer queue'.
Quoted 41 lines
app/test/test_rcu_qsbr.c +41
-0+*/ +static int +test_rcu_qsbr_dq_create (void ) +{ +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_create()\n" ); + +/* Pass invalid parameters */ + dq =rte_rcu_qsbr_dq_create (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + + params.f = rte_rcu_qsbr_test_free_resource; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + + params.size =1 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params");app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ + params.esize =3 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +/* Pass all valid parameters */ + params.esize =16 ; + dq =rte_rcu_qsbr_dq_create (¶ms); + TEST_RCU_QSBR_RETURN_IF_ERROR((dq == NULL), "dq create validparams");
Quoted 38 lines
app/test/test_rcu_qsbr.c +38
-0+rte_rcu_qsbr_dq_delete (dq); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_enqueue (void ) +{ +int ret; +uint64_t r; +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_enqueue()\n" ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.f = rte_rcu_qsbr_test_free_resource; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid +params"); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_enqueue (NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid +params"); +app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ ret =rte_rcu_qsbr_dq_enqueue (dq,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid +params"); + + ret =rte_rcu_qsbr_dq_enqueue (NULL , &r); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid +params"); + + ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 1), "dq delete validparams");
Quoted 40 lines
app/test/test_rcu_qsbr.c +40
-0+ +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_reclaim: Reclaim resources from the defer queue. +*/ +static int +test_rcu_qsbr_dq_reclaim (void ) +{ +int ret; + +printf ("\nTest rte_rcu_qsbr_dq_reclaim()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_reclaim (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq reclaim invalid +params"); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_delete: Delete a defer queue. +*/ +static int +test_rcu_qsbr_dq_delete (void ) +{ +int ret; +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_delete()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_delete (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq delete invalid +params"); +app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.f = rte_rcu_qsbr_test_free_resource; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + dq =rte_rcu_qsbr_dq_create (¶ms); + TEST_RCU_QSBR_RETURN_IF_ERROR((dq == NULL), "dq create validparams");
app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void) · line 749
Quoted 2 lines
app/test/test_rcu_qsbr.c +2
-0+ ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret != 0), "dq delete validparams");
Quoted 70 lines
app/test/test_rcu_qsbr.c +70
-0+ +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_functional (int32_t size,int32_t esize) { +int i, j, ret; +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; +uint64_t *e; +uint64_t sc =200 ; +int max_entries; + +printf ("\nTest rte_rcu_qsbr_dq_xxx functional tests()\n" ); +printf ("Size = %d, esize = %d\n" , size, esize); + + e = (uint64_t *)rte_zmalloc (NULL , esize,RTE_CACHE_LINE_SIZE ); +if (e ==NULL ) +return 0 ; + cb_failed =0 ; + +/* Initialize the RCU variable. No threads are registered */ +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.f = rte_rcu_qsbr_test_free_resource; + params.v = t[0 ]; + params.size = size; + params.esize = esize; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid +params"); + +/* Given the size and esize, calculate the maximum number of entries +* that can be stored on the defer queue (look at the logic used +* in capacity calculation of rte_ring). +*/ + max_entries =rte_align32pow2 (((esize/8 +1 ) * size) +1 ); + max_entries = (max_entries -1 )/(esize/8 +1 ); + +/* Enqueue few counters starting with the value 'sc' */ +/* The queue size will be rounded up to 2. The enqueue API also +* reclaims if the queue size is above certain limit. Since, there +* are no threads registered, reclamation succedes. Hence, it should +* be possible to enqueue more than the provided queue size. +*/ +for (i =0 ; i <10 ; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Register a thread on the RCU QSBR variable. Reclamation will not +* succeed. It should not be possible to enqueue more than the size +* number of resources. +*/ +rte_rcu_qsbr_thread_register (t[0 ],1 ); +rte_rcu_qsbr_thread_online (t[0 ],1 ); + +for (i =0 ; i < max_entries; i++) {app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Enqueue fails as queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 0), "dq enqueuefunctional");
app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 6 lines
app/test/test_rcu_qsbr.c +6
-0+ +/* Delete should fail as there are elements in defer queue which +* cannot be reclaimed. +*/ + ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 0), "dq delete validparams");
Quoted 4 lines
app/test/test_rcu_qsbr.c +4
-0+ +/* Report quiescent state, enqueue should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); +for (i =0 ; i < max_entries; i++) {app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 0), "dq enqueuefunctional");
app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 5 lines
app/test/test_rcu_qsbr.c +5
-0+ +/* Report quiescent state, delete should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); + ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret != 0), "dq delete validparams");
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr
Quoted 72 lines
app/test/test_rcu_qsbr.c +32
-0+ +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_RETURN_IF_ERROR ((cb_failed ==1 ),"CB failed" ); + +rte_free (e); +return 0 ; +} +/* * rte_rcu_qsbr_dump: Dump status of a single QS variable to a file */ @@ -1025,6 +1290,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_thread_offline () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_create () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_reclaim () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_delete () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_enqueue () <0 ) + goto test_fail; +printf ("\nFunctional tests\n" );if (test_rcu_qsbr_sw_sv_3qs () <0 )@@ -1033,6 +1310,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_mw_mv_mqs () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_functional (1 ,8 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (2 ,8 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (303 ,16 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (7 ,128 ) <0 ) + goto test_fail; +free_rcu ();printf ("\n" );lib/librte_rcu/meson.build +2
-0@@ -10,3 +10,5 @@ headers = files('rte_rcu_qsbr.h') if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false ext_deps += cc.find_library('atomic') endif + +deps += ['ring'](quoted diff) +0
-0
b/lib/librte_rcu/rte_rcu_qsbr.cindexce7f93dd3..76814f50b100644lib/librte_rcu/rte_rcu_qsbr.c +1
-0@@ -21,6 +21,7 @@#include <rte_errno.h> #include "rte_rcu_qsbr.h" +#include "rte_rcu_qsbr_pvt.h" /* Get the memory size of QSBR variable */ size_t @@ -267,6 +268,190 @@ rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr*v)
Quoted 75 lines
lib/librte_rcu/rte_rcu_qsbr.c +72
-0return 0 ; } +/* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +*/ +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters +*params) { +struct rte_rcu_qsbr_dq *dq; +uint32_t qs_fifo_size; + +if (params ==NULL || params->f ==NULL || + params->v ==NULL || params->name ==NULL || + params->size ==0 || params->esize ==0 || + (params->esize %8 !=0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return NULL ; + } + + dq =rte_zmalloc (NULL , + (sizeof (struct rte_rcu_qsbr_dq ) + params->esize), +RTE_CACHE_LINE_SIZE ); +if (dq ==NULL ) { + rte_errno =ENOMEM ; + +return NULL ; + } + +/* round up qs_fifo_size to next power of two that is not less than +* max_size. +*/ + qs_fifo_size =rte_align32pow2 ((((params->esize/8 ) +1 ) + * params->size) +1 ); + dq->r =rte_ring_create (params->name, qs_fifo_size, +SOCKET_ID_ANY ,0 ); +if (dq->r ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): defer queue create failed\n" , __func__); +rte_free (dq); +return NULL ; + } + + dq->v = params->v; + dq->size = params->size; + dq->esize = params->esize; + dq->f = params->f; + dq->p = params->p; + +return dq; +} + +/* Enqueue one resource to the defer queue to free after the grace +* period is over. +*/ +int rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e) { +uint64_t token; +uint64_t *tmp; +uint32_t i; +uint32_t cur_size, free_size; + +if (dq ==NULL || e ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Start the grace period */ + token =rte_rcu_qsbr_start (dq->v); +lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 353
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+/* Reclaim resources if the queue is 1/8th full. This helps +* the queue from growing too large and allows time for reader +* threads to report their quiescent state. +*/ + cur_size =rte_ring_count (dq->r) / (dq->esize/8 +1 ); +if (cur_size > (dq->size >>RTE_RCU_QSBR_AUTO_RECLAIM_LIMIT )) { +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Triggering reclamation\n" , __func__); +rte_rcu_qsbr_dq_reclaim (dq); + }There are two problems I see:
Quoted 4 lines
Quoted 3 lines
1.
rte_rcu_qsbr_dq_reclaim()reclaims only 1/16 of the defer queue while it triggers on 1/8. This means that there will always be 1/16 of non reclaimed entries in the queue.There will be 'at least' 1/16 non-reclaimed entries.
Correct, that's what I meant :)
Quoted 1 line
It could be more depending on the length of the grace period and the rate of deletion.
Right, the number of entries to reclaim depends on:
- grace period which is application specific
- cost of delete operation which is library (algorithm) specific
- rate of deletion which depends on runtime.
So it is very hard to predict how big should be threshold to trigger reclamation and how many entries should it reclaim.
Quoted 1 line
The trigger of 1/8 is used to give sufficient time for the readers to report their quiescent state. 1/16 is used to spread the load of reclamation across multiple calls and provide a upper bound on the cycles consumed.
1/16 of max entries to reclaim within single call can cost a lot. Moreover, it could have an impact on the readers through massive cache evictions.
Consider a set of routes from test_lpm_perf.c. To install all routes you need to have at least 65k tbl8 entries (now it has 2k). So when reclaiming, besides the costs of rte_rcu_qsbr_check(), you'll need to rewrite 4k cache lines.
So 1/16 of max entries is relatively big and it's better to spread this load across multiple calls.
Quoted 9 lines
Quoted 3 lines
2. Number of entries to reclaim depend on
dq->size. So,rte_rcu_qsbr_dq_reclaim()could take a lot of cycles. For LPM library thisThat is true. It depends on
dq->size(number of tbl8 groups). However, note that there is patch [1] which provides batch reclamation kind of behavior which reduces the cycles consumed by reclamation significantly.[1] https://patches.dpdk.org/patch/58960/
Quoted 1 line
means that
rte_lpm_delete()sometimes takes a long time.Agree, sometimes takes additional time. It is good to spread it over multiple calls.
Right, with batch reclamation we have here classic throughput vs latency problem. Either reclaiming big number of entries relatively infrequently spreading the cost of readers quiescent state check or reclaiming small amount of entries more often spending more cycles in average. I'd prefer latency here because as I mentioned earlier huge batches could have an impact on readers and lead to big difference in cost of delete().
Quoted 4 lines
Quoted 4 lines
So, my suggestions here would be
- trigger
rte_rcu_qsbr_dq_reclaim()with every enqueue
Quoted 8 lines
Given that the LPM APIs are mainly for control plane, I would think that, the next time LPM API is called, the readers have completed the grace period. But if there are frequent updates, we might end up with empty reclaims which will waste cycles. IMO, this trigger should happen after at least few entries are in the queue.
Quoted 1 line
- reclaim small amount of entries (could be configurable of creation time)
Agree. I would keep it a smaller than the trigger amount knowing that the elements added right before the trigger might not have completed the grace period.
Quoted 1 line
- provide API to trigger reclaim from the application manually.
IMO, this will add additional complexity to the application. I agree that there will be special needs for some applications. I think those applications might have to implement their own methods using the base RCU APIs.
Instead, as agreed in other threads, I suggest we expose the parameters (when to trigger and how much to reclaim) to the application as optional configurable parameters. i.e. if the application does not provide we can use default values. I think this should provide enough flexibility to the application.
Agree.
Regarding default values, one strategy could be:
- if reported threshold isn't set (i.e. is equal 0) then call reclaim with every enqueue (i.e. threshold == 1)
- if max_entries_to_reclaim isn't set then reclaim as much as we can
Quoted 393 lines
Quoted 393 lines
Quoted 252 lines
lib/librte_rcu/rte_rcu_qsbr.c +98
-0+ +/* Check if there is space for atleast for 1 resource */ + free_size =rte_ring_free_count (dq->r) / (dq->esize/8 +1 ); +if (!free_size) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Defer queue is full\n" , __func__); + rte_errno =ENOSPC ; +return 1 ; + } + +/* Enqueue the resource */ +rte_ring_sp_enqueue (dq->r, (void *)(uintptr_t )token); + +/* The resource to enqueue needs to be a multiple of 64b +* due to the limitation of the rte_ring implementation. +*/ +for (i =0 , tmp = (uint64_t *)e; i < dq->esize/8 ; i++, tmp++) +rte_ring_sp_enqueue (dq->r, (void *)(uintptr_t )*tmp); + +return 0 ; +} + +/* Reclaim resources from the defer queue. */ int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq) { +uint32_t max_cnt; +uint32_t cnt; +void *token; +uint64_t *tmp; +uint32_t i; + +if (dq ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Anything to reclaim? */ +if (rte_ring_count (dq->r) ==0 ) +return 0 ; + +/* Reclaim at the max 1/16th the total number of entries. */ + max_cnt = dq->size >>RTE_RCU_QSBR_MAX_RECLAIM_LIMIT ; + max_cnt = (max_cnt ==0 ) ? dq->size : max_cnt; + cnt =0 ; + +/* Check reader threads quiescent state and reclaim resources */ +while ((cnt < max_cnt) && (rte_ring_peek (dq->r, &token) ==0 ) && + (rte_rcu_qsbr_check (dq->v, (uint64_t )((uintptr_t )token), false) + ==1 )) { + (void )rte_ring_sc_dequeue (dq->r, &token); +/* The resource to dequeue needs to be a multiple of 64b +* due to the limitation of the rte_ring implementation. +*/ +for (i =0 , tmp = (uint64_t *)dq->e; i < dq->esize/8 ; + i++, tmp++) + (void )rte_ring_sc_dequeue (dq->r, + (void *)(uintptr_t )tmp); + dq->f (dq->p, dq->e); + + cnt++; + } + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed %u resources\n" , __func__, cnt); + +if (cnt ==0 ) { +/* No resources were reclaimed */ + rte_errno =EAGAIN ; +return 1 ; + } + +return 0 ; +} + +/* Delete a defer queue. */ +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq) { +if (dq ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Reclaim all the resources */ +if (rte_rcu_qsbr_dq_reclaim (dq) !=0 ) +/* Error number is already set by the reclaim API */ +return 1 ; + +rte_ring_free (dq->r); +rte_free (dq); + +return 0 ; +} +int rte_rcu_log_type;RTE_INIT (rte_rcu_register)(quoted diff) +0
-0
b/lib/librte_rcu/rte_rcu_qsbr.hindexc80f15c00..185d4b50a100644lib/librte_rcu/rte_rcu_qsbr.h +129
-1@@ -34,6 +34,7 @@ extern "C" {#include <rte_lcore.h> #include <rte_debug.h> #include <rte_atomic.h> +#include <rte_ring.h> extern int rte_rcu_log_type;@@ -109,6 +110,67 @@ struct rte_rcu_qsbr {*/ } __rte_cache_aligned; +/** +* Call back function called to free the resources. +* +* @param p +* Pointer provided while creating the defer queue +* @param e +* Pointer to the resource data stored on the defer queue +* +* @return +* None +*/ +typedef void (*rte_rcu_qsbr_free_resource )(void *p,void *e); + +#define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE + +/** +* Trigger automatic reclamation after 1/8th the defer queue is full. +*/ +#define RTE_RCU_QSBR_AUTO_RECLAIM_LIMIT 3 + +/** +* Reclaim at the max 1/16th the total number of resources. +*/ +#define RTE_RCU_QSBR_MAX_RECLAIM_LIMIT 4 + +/** +* Parameters used when creating the defer queue. +*/ +struct rte_rcu_qsbr_dq_parameters { +const char *name; +/**< Name of the queue. */ +uint32_t size; +/**< Number of entries in queue. Typically, this will be +* the same as the maximum number of entries supported in the +* lock free data structure. +* Data structures with unbounded number of entries is not +* supported currently. +*/ +uint32_t esize; +/**< Size (in bytes) of each element in the defer queue. +* This has to be multiple of 8B as the rte_ring APIs +* support 8B element sizes only. +*/ +rte_rcu_qsbr_free_resource f; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. This can be NULL. +*/ +struct rte_rcu_qsbr *v; +/**< RCU QSBR variable to use for this defer queue */ }; + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq ; +/** * @warning * @b EXPERIMENTAL: this API may change without prior notice @@ -648,6 +710,113 @@ __rte_experimental int rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +* +* @param params +* Parameters to create a defer queue. +* @return +* On success - Valid pointer to defer queue +* On error - NULL +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - ENOMEM - Not enough memory +*/ +__rte_experimental +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters +*params); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Enqueue one resource to the defer queue and start the grace period. +* The resource will be freed later after at least one grace period +* is over. +* +* If the defer queue is full, it will attempt to reclaim resources. +* It will also reclaim resources at regular intervals to avoid +* the defer queue from growing too big. +* +* This API is not multi-thread safe. It is expected that the caller +* provides multi-thread safety by locking a mutex or some other means. +* +* A lock free multi-thread writer algorithm could achieve +multi-thread +* safety by creating and using one defer queue per thread. +* +* @param dq +* Defer queue to allocate an entry from. +* @param e +* Pointer to resource data to copy to the defer queue. The size of +* the data to copy is equal to the element size provided when the +* defer queue was created. +* @return +* On success - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +* - ENOSPC - Defer queue is full. This condition can not happen +* if the defer queue size is equal (or larger) than the +* number of elements in the data structure. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Reclaim resources from the defer queue. +* +* This API is not multi-thread safe. It is expected that the caller +* provides multi-thread safety by locking a mutex or some other means. +* +* A lock free multi-thread writer algorithm could achieve lib/librte_rcu/rte_rcu_qsbr.h struct rte_rcu_qsbr {
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.h +10
-0+multi-thread + * safety by creating and using one defer queue per thread. + * + * @param dq + * Defer queue to reclaim an entry from. + * @return + * On successful reclamation of at least 1 resource - 0 + * On error - 1 with rte_errno set to + * - EINVAL - NULL parameters are passed + * - EAGAIN - None of the resources have completed at least 1 graceperiod,
Quoted 119 lines
+ * try again.
+ */lib/librte_rcu/rte_rcu_qsbr.h +30
-0+__rte_experimental +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Delete a defer queue. +* +* It tries to reclaim all the resources on the defer queue. +* If any of the resources have not completed the grace period +* the reclamation stops and returns immediately. The rest of +* the resources are not reclaimed and the defer queue is not +* freed. +* +* @param dq +* Defer queue to delete. +* @return +* On success - 0 +* On error - 1 +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - EAGAIN - Some of the resources have not completed at least 1 grace +* period, try again. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq); + #ifdef __cplusplus }#endif lib/librte_rcu/rte_rcu_qsbr_pvt.h new file +45
-0@@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2019 Arm Limited +*/ + +#ifndef _RTE_RCU_QSBR_PVT_H_ +#define _RTE_RCU_QSBR_PVT_H_ + +/** +* This file is private to the RCU library. It should not be included +* by the user of this library. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "rte_rcu_qsbr.h" + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq { +struct rte_rcu_qsbr *v;/**< RCU QSBR variable used by this queue.*/ +struct rte_ring *r;/**< RCU QSBR defer queue. */ +uint32_t size; +/**< Number of elements in the defer queue */ +uint32_t esize; +/**< Size (in bytes) of data stored on the defer queue */ +rte_rcu_qsbr_free_resource f; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. +*/ +char e[0 ]; +/**< Temporary storage to copy the defer queue element. */ }; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_RCU_QSBR_PVT_H_ */ lib/librte_rcu/rte_rcu_version.map +4
-0@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };lib/meson.build index renamed from lib/meson.build +0
-0
e5ff83893..0e1be8407100644lib/meson.build +3
-1@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on thislib/meson.build libraries = [ · line 30
Quoted 8 lines
lib/meson.build +1
-1@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev', - 'rcu', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', # add pkt framework libs which use other libs from above--
Regards,
Vladimir
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v3 2/3] lib/rcu: add resource reclamation APIs
<snip>
Quoted 577 lines
Hi Honnappa,
Quoted 399 lines · Honnappa Nagarahalli
Hi Vladimir,
Apologies for the delayed response, I had to run few experiments.<snip>
Quoted 394 lines
Hi Honnappa,
Quoted 48 lines · Honnappa Nagarahalli
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- app/test/test_rcu_qsbr.c | 291 ++++++++++++++++++++++++++++- lib/librte_rcu/meson.build | 2 + lib/librte_rcu/rte_rcu_qsbr.c | 185 ++++++++++++++++++ lib/librte_rcu/rte_rcu_qsbr.h | 169 +++++++++++++++++ lib/librte_rcu/rte_rcu_qsbr_pvt.h | 46 +++++ lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 700 insertions(+), 3 deletions(-)create mode 100644
lib/librte_rcu/rte_rcu_qsbr_pvt.happ/test/test_rcu_qsbr.c +0
-0index
d1b9e46a2..3a6815243100644app/test/test_rcu_qsbr.c +3
-1@@ -1,8 +1,9 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2019 Arm Limited */ #include <stdio.h> +#include <string.h> #include <rte_pause.h> #include <rte_rcu_qsbr.h> #include <rte_hash.h> @@ -33,6 +34,7 @@ static uint32_t *keys;#define COUNTER_VALUE 4096static uint32_t *hash_data[RTE_MAX_LCORE ][TOTAL_ENTRY ];static uint8_t writer_done; +static uint8_t cb_failed;static struct rte_rcu_qsbr *t[RTE_MAX_LCORE ]; struct rte_hash *h[RTE_MAX_LCORE]; @@ -582,6 +584,269 @@
test_rcu_qsbr_thread_offline(void)return 0; }app/test/test_rcu_qsbr.c +3
-0+static void +rte_rcu_qsbr_test_free_resource (void *p,void *e) { +if (p !=NULL && e !=NULL ) {app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+printf ("%s: Test failed\n" , __func__); + cb_failed =1 ; + } +} + +/* +* rte_rcu_qsbr_dq_create: create a queue used to store the data +structure +* elements that can be freed later. This queue is referred to as +'defer queue'.
Quoted 41 lines
+ */
app/test/test_rcu_qsbr.c +40
-0+static int +test_rcu_qsbr_dq_create (void ) +{ +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_create()\n" ); + +/* Pass invalid parameters */ + dq =rte_rcu_qsbr_dq_create (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + + params.f = rte_rcu_qsbr_test_free_resource; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + + params.size =1 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params");app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ + params.esize =3 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid +params"); + +/* Pass all valid parameters */ + params.esize =16 ; + dq =rte_rcu_qsbr_dq_create (¶ms); + TEST_RCU_QSBR_RETURN_IF_ERROR((dq == NULL), "dq create validparams");
Quoted 39 lines
app/test/test_rcu_qsbr.c +39
-0+rte_rcu_qsbr_dq_delete (dq); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer +queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_enqueue (void ) +{ +int ret; +uint64_t r; +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_enqueue()\n" ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.f = rte_rcu_qsbr_test_free_resource; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid +params"); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_enqueue (NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid +params"); +app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ ret =rte_rcu_qsbr_dq_enqueue (dq,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid +params"); + + ret =rte_rcu_qsbr_dq_enqueue (NULL , &r); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid +params"); + + ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 1), "dq delete validparams");
Quoted 40 lines
app/test/test_rcu_qsbr.c +40
-0+ +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_reclaim: Reclaim resources from the defer queue. +*/ +static int +test_rcu_qsbr_dq_reclaim (void ) +{ +int ret; + +printf ("\nTest rte_rcu_qsbr_dq_reclaim()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_reclaim (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq reclaim invalid +params"); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_delete: Delete a defer queue. +*/ +static int +test_rcu_qsbr_dq_delete (void ) +{ +int ret; +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_delete()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_delete (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq delete invalid +params"); +app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.f = rte_rcu_qsbr_test_free_resource; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + dq =rte_rcu_qsbr_dq_create (¶ms); + TEST_RCU_QSBR_RETURN_IF_ERROR((dq == NULL), "dq create validparams");
app/test/test_rcu_qsbr.c
Quoted 2 lines
app/test/test_rcu_qsbr.c +2
-0+ ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret != 0), "dq delete validparams");
Quoted 71 lines
app/test/test_rcu_qsbr.c +71
-0+ +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer +queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_functional (int32_t size,int32_t esize) { +int i, j, ret; +char rcu_dq_name[RTE_RING_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; +uint64_t *e; +uint64_t sc =200 ; +int max_entries; + +printf ("\nTest rte_rcu_qsbr_dq_xxx functional tests()\n" ); +printf ("Size = %d, esize = %d\n" , size, esize); + + e = (uint64_t *)rte_zmalloc (NULL , esize,RTE_CACHE_LINE_SIZE ); +if (e ==NULL ) +return 0 ; + cb_failed =0 ; + +/* Initialize the RCU variable. No threads are registered */ +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.f = rte_rcu_qsbr_test_free_resource; + params.v = t[0 ]; + params.size = size; + params.esize = esize; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid +params"); + +/* Given the size and esize, calculate the maximum number of entries +* that can be stored on the defer queue (look at the logic used +* in capacity calculation of rte_ring). +*/ + max_entries =rte_align32pow2 (((esize/8 +1 ) * size) +1 ); + max_entries = (max_entries -1 )/(esize/8 +1 ); + +/* Enqueue few counters starting with the value 'sc' */ +/* The queue size will be rounded up to 2. The enqueue API also +* reclaims if the queue size is above certain limit. Since, there +* are no threads registered, reclamation succedes. Hence, it should +* be possible to enqueue more than the provided queue size. +*/ +for (i =0 ; i <10 ; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Register a thread on the RCU QSBR variable. Reclamation will not +* succeed. It should not be possible to enqueue more than the size +* number of resources. +*/ +rte_rcu_qsbr_thread_register (t[0 ],1 ); +rte_rcu_qsbr_thread_online (t[0 ],1 ); + +for (i =0 ; i < max_entries; i++) {app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Enqueue fails as queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 0), "dq enqueuefunctional");
app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 6 lines
app/test/test_rcu_qsbr.c +6
-0+ +/* Delete should fail as there are elements in defer queue which +* cannot be reclaimed. +*/ + ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 0), "dq delete validparams");
Quoted 4 lines
app/test/test_rcu_qsbr.c +4
-0+ +/* Report quiescent state, enqueue should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); +for (i =0 ; i < max_entries; i++) {app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 10 lines
app/test/test_rcu_qsbr.c +10
-0+ ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret == 0), "dq enqueuefunctional");
app/test/test_rcu_qsbr.c test_rcu_qsbr_thread_offline(void)
Quoted 5 lines
app/test/test_rcu_qsbr.c +5
-0+ +/* Report quiescent state, delete should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); + ret =rte_rcu_qsbr_dq_delete (dq); + TEST_RCU_QSBR_RETURN_IF_ERROR((ret != 0), "dq delete validparams");
Quoted 66 lines
app/test/test_rcu_qsbr.c +20
-0+ +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_RETURN_IF_ERROR ((cb_failed ==1 ),"CB failed" ); + +rte_free (e); +return 0 ; +} +/* * rte_rcu_qsbr_dump: Dump status of a single QS variable to a file */ @@ -1025,6 +1290,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_thread_offline () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_create () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_reclaim () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_delete () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_enqueue () <0 ) + goto test_fail; +printf ("\nFunctional tests\n" ); if (test_rcu_qsbr_sw_sv_3qs() < 0) @@ -1033,6 +1310,18 @@
test_rcu_qsbr_main(void)
if (test_rcu_qsbr_mw_mv_mqs()< 0)
gototest_fail;app/test/test_rcu_qsbr.c +12
-0+if (test_rcu_qsbr_dq_functional (1 ,8 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (2 ,8 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (303 ,16 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (7 ,128 ) <0 ) + goto test_fail; +free_rcu ();printf ("\n" );lib/librte_rcu/meson.build +2
-0@@ -10,3 +10,5 @@ headers = files('rte_rcu_qsbr.h') if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false ext_deps += cc.find_library('atomic') endif + +deps += ['ring'](quoted diff) +0
-0
b/lib/librte_rcu/rte_rcu_qsbr.cindexce7f93dd3..76814f50b100644lib/librte_rcu/rte_rcu_qsbr.c +0
-0@@ -21,6 +21,7 @@#include <rte_errno.h> Quoted 6 lines
Quoted 6 lines
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct
Quoted 6 lines
lib/librte_rcu/rte_rcu_qsbr.c +1
-0#include "rte_rcu_qsbr.h" +#include "rte_rcu_qsbr_pvt.h" /* Get the memory size of QSBR variable */ size_t @@ -267,6 +268,190 @@ rte_rcu_qsbr_dump(FILE *f, struct
rte_rcu_qsbrQuoted 89 lines
Quoted 89 lines
*v)
Quoted 76 lines
lib/librte_rcu/rte_rcu_qsbr.c +73
-0return 0 ; } +/* Create a queue used to store the data structure elements that +can +* be freed later. This queue is referred to as 'defer queue'. +*/ +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters +*params) { +struct rte_rcu_qsbr_dq *dq; +uint32_t qs_fifo_size; + +if (params ==NULL || params->f ==NULL || + params->v ==NULL || params->name ==NULL || + params->size ==0 || params->esize ==0 || + (params->esize %8 !=0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return NULL ; + } + + dq =rte_zmalloc (NULL , + (sizeof (struct rte_rcu_qsbr_dq ) + params->esize), +RTE_CACHE_LINE_SIZE ); +if (dq ==NULL ) { + rte_errno =ENOMEM ; + +return NULL ; + } + +/* round up qs_fifo_size to next power of two that is not less than +* max_size. +*/ + qs_fifo_size =rte_align32pow2 ((((params->esize/8 ) +1 ) + * params->size) +1 ); + dq->r =rte_ring_create (params->name, qs_fifo_size, +SOCKET_ID_ANY ,0 ); +if (dq->r ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): defer queue create failed\n" , __func__); +rte_free (dq); +return NULL ; + } + + dq->v = params->v; + dq->size = params->size; + dq->esize = params->esize; + dq->f = params->f; + dq->p = params->p; + +return dq; +} + +/* Enqueue one resource to the defer queue to free after the grace +* period is over. +*/ +int rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e) { +uint64_t token; +uint64_t *tmp; +uint32_t i; +uint32_t cur_size, free_size; + +if (dq ==NULL || e ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Start the grace period */ + token =rte_rcu_qsbr_start (dq->v); +lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 353
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+/* Reclaim resources if the queue is 1/8th full. This helps +* the queue from growing too large and allows time for reader +* threads to report their quiescent state. +*/ + cur_size =rte_ring_count (dq->r) / (dq->esize/8 +1 ); +if (cur_size > (dq->size >>RTE_RCU_QSBR_AUTO_RECLAIM_LIMIT )) { +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Triggering reclamation\n" , __func__); +rte_rcu_qsbr_dq_reclaim (dq); + }There are two problems I see:
Quoted 4 lines
Quoted 3 lines
1.
rte_rcu_qsbr_dq_reclaim()reclaims only 1/16 of the defer queue while it triggers on 1/8. This means that there will always be 1/16 of non reclaimed entries in the queue.There will be 'at least' 1/16 non-reclaimed entries.
Correct, that's what I meant :)
Quoted 1 line
It could be more depending on the length of the grace period and the rate
of deletion.
Right, the number of entries to reclaim depends on:
- grace period which is application specific
- cost of delete operation which is library (algorithm) specific
- rate of deletion which depends on runtime.
So it is very hard to predict how big should be threshold to trigger reclamation and how many entries should it reclaim.
Quoted 1 line
The trigger of 1/8 is used to give sufficient time for the readers to report
their quiescent state. 1/16 is used to spread the load of reclamation across multiple calls and provide a upper bound on the cycles consumed.
1/16 of max entries to reclaim within single call can cost a lot. Moreover, it could have an impact on the readers through massive cache evictions.
Consider a set of routes from
test_lpm_perf.c. To install all routes you need to have at least 65k tbl8 entries (now it has 2k). So when reclaiming, besides the costs ofrte_rcu_qsbr_check(), you'll need to rewrite 4k cache lines.So 1/16 of max entries is relatively big and it's better to spread this load across multiple calls.
Quoted 5 lines
Quoted 4 lines
2. Number of entries to reclaim depend on
dq->size. So,rte_rcu_qsbr_dq_reclaim()could take a lot of cycles. For LPM library thisThat is true. It depends on
dq->size(number of tbl8 groups). However, notethat there is patch [1] which provides batch reclamation kind of behavior which reduces the cycles consumed by reclamation significantly.
Quoted 5 lines
[1] https://patches.dpdk.org/patch/58960/
Quoted 1 line
means that
rte_lpm_delete()sometimes takes a long time.Agree, sometimes takes additional time. It is good to spread it over multiple
calls.
Right, with batch reclamation we have here classic throughput vs latency problem. Either reclaiming big number of entries relatively infrequently spreading the cost of readers quiescent state check or reclaiming small amount of entries more often spending more cycles in average. I'd prefer latency here because as I mentioned earlier huge batches could have an impact on readers and lead to big difference in cost ofdelete().Quoted 5 lines
Quoted 4 lines
So, my suggestions here would be
- trigger
rte_rcu_qsbr_dq_reclaim()with every enqueueGiven that the LPM APIs are mainly for control plane, I would think that,
the next time LPM API is called, the readers have completed the grace period. But if there are frequent updates, we might end up with empty reclaims which will waste cycles. IMO, this trigger should happen after at least few entries are in the queue.
Quoted 4 lines
Quoted 3 lines
- reclaim small amount of entries (could be configurable of creation time)
Agree. I would keep it a smaller than the trigger amount knowing that the
elements added right before the trigger might not have completed the grace period.
Quoted 3 lines
Quoted 2 lines
- provide API to trigger reclaim from the application manually.
IMO, this will add additional complexity to the application. I agree that
there will be special needs for some applications. I think those applications might have to implement their own methods using the base RCU APIs.
Quoted 1 line
Instead, as agreed in other threads, I suggest we expose the parameters
(when to trigger and how much to reclaim) to the application as optional configurable parameters. i.e. if the application does not provide we can use default values. I think this should provide enough flexibility to the application.
Quoted 9 lines
Agree.
Regarding default values, one strategy could be:
- if reported threshold isn't set (i.e. is equal 0) then call reclaim with every enqueue (i.e. threshold == 1)
- if
max_entries_to_reclaimisn't set then reclaim as much as we can
Ok, sounds good.
Quoted 403 lines
Quoted 285 lines
Quoted 285 lines
Quoted 256 lines
lib/librte_rcu/rte_rcu_qsbr.c +98
-0+ +/* Check if there is space for atleast for 1 resource */ + free_size =rte_ring_free_count (dq->r) / (dq->esize/8 +1 ); +if (!free_size) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Defer queue is full\n" , __func__); + rte_errno =ENOSPC ; +return 1 ; + } + +/* Enqueue the resource */ +rte_ring_sp_enqueue (dq->r, (void *)(uintptr_t )token); + +/* The resource to enqueue needs to be a multiple of 64b +* due to the limitation of the rte_ring implementation. +*/ +for (i =0 , tmp = (uint64_t *)e; i < dq->esize/8 ; i++, tmp++) +rte_ring_sp_enqueue (dq->r, (void *)(uintptr_t )*tmp); + +return 0 ; +} + +/* Reclaim resources from the defer queue. */ int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq) { +uint32_t max_cnt; +uint32_t cnt; +void *token; +uint64_t *tmp; +uint32_t i; + +if (dq ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Anything to reclaim? */ +if (rte_ring_count (dq->r) ==0 ) +return 0 ; + +/* Reclaim at the max 1/16th the total number of entries. */ + max_cnt = dq->size >>RTE_RCU_QSBR_MAX_RECLAIM_LIMIT ; + max_cnt = (max_cnt ==0 ) ? dq->size : max_cnt; + cnt =0 ; + +/* Check reader threads quiescent state and reclaim resources */ +while ((cnt < max_cnt) && (rte_ring_peek (dq->r, &token) ==0 ) && + (rte_rcu_qsbr_check (dq->v, (uint64_t )((uintptr_t )token), false) + ==1 )) { + (void )rte_ring_sc_dequeue (dq->r, &token); +/* The resource to dequeue needs to be a multiple of 64b +* due to the limitation of the rte_ring implementation. +*/ +for (i =0 , tmp = (uint64_t *)dq->e; i < dq->esize/8 ; + i++, tmp++) + (void )rte_ring_sc_dequeue (dq->r, + (void *)(uintptr_t )tmp); + dq->f (dq->p, dq->e); + + cnt++; + } + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed %u resources\n" , __func__, cnt); + +if (cnt ==0 ) { +/* No resources were reclaimed */ + rte_errno =EAGAIN ; +return 1 ; + } + +return 0 ; +} + +/* Delete a defer queue. */ +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq) { +if (dq ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Reclaim all the resources */ +if (rte_rcu_qsbr_dq_reclaim (dq) !=0 ) +/* Error number is already set by the reclaim API */ +return 1 ; + +rte_ring_free (dq->r); +rte_free (dq); + +return 0 ; +} +int rte_rcu_log_type;RTE_INIT (rte_rcu_register)(quoted diff) +0
-0
b/lib/librte_rcu/rte_rcu_qsbr.hindexc80f15c00..185d4b50a100644lib/librte_rcu/rte_rcu_qsbr.h +132
-1@@ -34,6 +34,7 @@ extern "C" {#include <rte_lcore.h> #include <rte_debug.h> #include <rte_atomic.h> +#include <rte_ring.h> extern int rte_rcu_log_type;@@ -109,6 +110,67 @@ struct rte_rcu_qsbr {*/ } __rte_cache_aligned; +/** +* Call back function called to free the resources. +* +* @param p +* Pointer provided while creating the defer queue +* @param e +* Pointer to the resource data stored on the defer queue +* +* @return +* None +*/ +typedef void (*rte_rcu_qsbr_free_resource )(void *p,void *e); + +#define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE + +/** +* Trigger automatic reclamation after 1/8th the defer queue is full. +*/ +#define RTE_RCU_QSBR_AUTO_RECLAIM_LIMIT 3 + +/** +* Reclaim at the max 1/16th the total number of resources. +*/ +#define RTE_RCU_QSBR_MAX_RECLAIM_LIMIT 4 + +/** +* Parameters used when creating the defer queue. +*/ +struct rte_rcu_qsbr_dq_parameters { +const char *name; +/**< Name of the queue. */ +uint32_t size; +/**< Number of entries in queue. Typically, this will be +* the same as the maximum number of entries supported in the +* lock free data structure. +* Data structures with unbounded number of entries is not +* supported currently. +*/ +uint32_t esize; +/**< Size (in bytes) of each element in the defer queue. +* This has to be multiple of 8B as the rte_ring APIs +* support 8B element sizes only. +*/ +rte_rcu_qsbr_free_resource f; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. This can be NULL. +*/ +struct rte_rcu_qsbr *v; +/**< RCU QSBR variable to use for this defer queue */ }; + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq ; +/** * @warning * @b EXPERIMENTAL: this API may change without prior notice @@ -648,6 +710,113 @@ __rte_experimental int rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Create a queue used to store the data structure elements that +can +* be freed later. This queue is referred to as 'defer queue'. +* +* @param params +* Parameters to create a defer queue. +* @return +* On success - Valid pointer to defer queue +* On error - NULL +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - ENOMEM - Not enough memory +*/ +__rte_experimental +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters +*params); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Enqueue one resource to the defer queue and start the grace period. +* The resource will be freed later after at least one grace period +* is over. +* +* If the defer queue is full, it will attempt to reclaim resources. +* It will also reclaim resources at regular intervals to avoid +* the defer queue from growing too big. +* +* This API is not multi-thread safe. It is expected that the +caller +* provides multi-thread safety by locking a mutex or some other means. +* +* A lock free multi-thread writer algorithm could achieve +multi-thread +* safety by creating and using one defer queue per thread. +* +* @param dq +* Defer queue to allocate an entry from. +* @param e +* Pointer to resource data to copy to the defer queue. The size of +* the data to copy is equal to the element size provided when the +* defer queue was created. +* @return +* On success - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +* - ENOSPC - Defer queue is full. This condition can not happen +* if the defer queue size is equal (or larger) than the +* number of elements in the data structure. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Reclaim resources from the defer queue. +* +* This API is not multi-thread safe. It is expected that the +caller +* provides multi-thread safety by locking a mutex or some other means. +* +* A lock free multi-thread writer algorithm could achieve lib/librte_rcu/rte_rcu_qsbr.h struct rte_rcu_qsbr {
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.h +10
-0+multi-thread + * safety by creating and using one defer queue per thread. + * + * @param dq + * Defer queue to reclaim an entry from. + * @return + * On successful reclamation of at least 1 resource - 0 + * On error - 1 with rte_errno set to + * - EINVAL - NULL parameters are passed + * - EAGAIN - None of the resources have completed at least 1 graceperiod,
Quoted 18 lines
+ * try again.
+ */lib/librte_rcu/rte_rcu_qsbr.h +16
-0+__rte_experimental +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Delete a defer queue. +* +* It tries to reclaim all the resources on the defer queue. +* If any of the resources have not completed the grace period +* the reclamation stops and returns immediately. The rest of +* the resources are not reclaimed and the defer queue is not +* freed. +* Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
+ * @param dq
+ * Defer queue to delete.
+ * @return+ * On success - 0 + * On error - 1 + * Possible rte_errno codes are: + * - EINVAL - NULL parameters are passed + * - EAGAIN - Some of the resources have not completed at least 1grace
Quoted 97 lines
Quoted 97 lines
Quoted 97 lines
+ * period, try again.
+ */lib/librte_rcu/rte_rcu_qsbr.h +4
-0+__rte_experimental +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq); + #ifdef __cplusplus }#endif lib/librte_rcu/rte_rcu_qsbr_pvt.h new file +46
-0@@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2019 Arm Limited +*/ + +#ifndef _RTE_RCU_QSBR_PVT_H_ +#define _RTE_RCU_QSBR_PVT_H_ + +/** +* This file is private to the RCU library. It should not be +included +* by the user of this library. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "rte_rcu_qsbr.h" + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq { +struct rte_rcu_qsbr *v;/**< RCU QSBR variable used by this queue.*/ +struct rte_ring *r;/**< RCU QSBR defer queue. */ +uint32_t size; +/**< Number of elements in the defer queue */ +uint32_t esize; +/**< Size (in bytes) of data stored on the defer queue */ +rte_rcu_qsbr_free_resource f; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. +*/ +char e[0 ]; +/**< Temporary storage to copy the defer queue element. */ }; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_RCU_QSBR_PVT_H_ */ lib/librte_rcu/rte_rcu_version.map +4
-0@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };lib/meson.build index renamed from lib/meson.build +0
-0
e5ff83893..0e1be8407100644lib/meson.build +3
-1@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on this@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev',Quoted 8 lines
Quoted 8 lines
lib/meson.build libraries = [ · line 30
Quoted 5 lines
lib/meson.build +1
-1- 'rcu', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', # add pkt framework libs which use other libs from above--
Regards,
Vladimir--
Regards,
Vladimir
PATCH v3 3/3 doc/rcu: add RCU integration design details
In reply to Honnappa NagarahalliFrom: Ruifeng Wang <[email protected]>
Add a section to describe a design to integrate QSBR RCU library with other libraries in DPDK.
--- doc/guides/prog_guide/rcu_lib.rst | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)
doc/guides/prog_guide/rcu_lib.rst +59 -0
@@ -186,3 +186,62 @@ However, when ``CONFIG_RTE_LIBRTE_RCU_DEBUG`` is enabled, these APIs aid in debugging issues. One can mark the access to shared data structures on the reader side using these APIs. The ``rte_rcu_qsbr_quiescent()`` will check if all the locks are unlocked. + +Resource reclamation framework for DPDK +--------------------------------------- + +Lock-free algorithms place additional burden of resource reclamation on +the application. When a writer deletes an entry from a data structure, the writer: + +#. Has to start the grace period +#. Has to store a reference to the deleted resources in a FIFO +#. Should check if the readers have completed a grace period and free the resources. This can also be done when the writer runs out of free resources. + +There are several APIs provided to help with this process. The writer +can create a FIFO to store the references to deleted resources using ``rte_rcu_qsbr_dq_create()``. +The resources can be enqueued to this FIFO using ``rte_rcu_qsbr_dq_enqueue()``. +If the FIFO is full, ``rte_rcu_qsbr_dq_enqueue`` will reclaim the resources before enqueuing. It will also reclaim resources on regular basis to keep the FIFO from growing too large. If the writer runs out of resources, the writer can call ``rte_rcu_qsbr_dq_reclaim`` API to reclaim resources. ``rte_rcu_qsbr_dq_delete`` is provided to reclaim any remaining resources and free the FIFO while shutting down. + +However, if this resource reclamation process were to be integrated in lock-free data structure libraries, it +hides this complexity from the application and makes it easier for the application to adopt lock-free algorithms. The following paragraphs discuss how the reclamation process can be integrated in DPDK libraries. + +In any DPDK application, the resource reclamation process using QSBR can be split into 4 parts: + +#. Initialization +#. Quiescent State Reporting +#. Reclaiming Resources +#. Shutdown + +The design proposed here assigns different parts of this process to client libraries and applications. The term 'client library' refers to lock-free data structure libraries such at rte_hash, rte_lpm etc. in DPDK or similar libraries outside of DPDK. The term 'application' refers to the packet processing application that makes use of DPDK such as L3 Forwarding example application, OVS, VPP etc.. + +The application has to handle 'Initialization' and 'Quiescent State Reporting'. So, + +* the application has to create the RCU variable and register the reader threads to report their quiescent state. +* the application has to register the same RCU variable with the client library. +* reader threads in the application have to report the quiescent state. This allows for the application to control the length of the critical section/how frequently the application wants to report the quiescent state. + +The client library will handle 'Reclaiming Resources' part of the process. The +client libraries will make use of the writer thread context to execute the memory +reclamation algorithm. So, + +* client library should provide an API to register a RCU variable that it will use. It should call ``rte_rcu_qsbr_dq_create()`` to create the FIFO to store the references to deleted entries. +* client library should use ``rte_rcu_qsbr_dq_enqueue`` to enqueue the deleted resources on the FIFO and start the grace period. +* if the library runs out of resources while adding entries, it should call ``rte_rcu_qsbr_dq_reclaim`` to reclaim the resources and try the resource allocation again. + +The 'Shutdown' process needs to be shared between the application and the +client library. + +* the application should make sure that the reader threads are not using the shared data structure, unregister the reader threads from the QSBR variable before calling the client library's shutdown function. + +* client library should call ``rte_rcu_qsbr_dq_delete`` to reclaim any remaining resources and free the FIFO. + +Integrating the resource reclamation with client libraries removes the burden from +the application and makes it easy to use lock-free algorithms. + +This design has several advantages over currently known methods. + +#. Application does not need a dedicated thread to reclaim resources. Memory + reclamation happens as part of the writer thread with little impact on + performance. +#. The client library has better control over the resources. For ex: the client + library can attempt to reclaim when it has run out of resources.
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v3 0/3] Add RCU reclamation APIs
In reply to Honnappa Nagarahalli01/10/2019 08:29, Honnappa Nagarahalli:
Quoted 8 lines
This is not a new patch. This patch set is separated from the LPM changes as the size of the changes in RCU library has grown due to comments from community. These APIs will help reduce the changes in LPM and hash libraries that are getting integrated with RCU library.
This adds 4 new APIs to RCU library to create a defer queue, enqueue deleted resources, reclaim resources and delete the defer queue.
It is in the roadmap for 20.05.
What is the status of this patchset?
Quoted 2 lines
The patches to LPM and HASH libraries to integrate RCU will depend on this patch.
I guess lpm and hash integrations are planned for 20.08?
Re: [dpdk-dev] [PATCH v3 0/3] Add RCU reclamation APIs
<snip>
Quoted 7 lines
01/10/2019 08:29, Honnappa Nagarahalli:
Quoted 5 lines
This is not a new patch. This patch set is separated from the LPM changes as the size of the changes in RCU library has grown due to comments from community. These APIs will help reduce the changes in LPM and hash libraries that are getting integrated with RCU library.
Quoted 5 lines
Quoted 2 lines
This adds 4 new APIs to RCU library to create a defer queue, enqueue deleted resources, reclaim resources and delete the defer queue.
It is in the roadmap for 20.05.
What is the status of this patchset?
It has a dependency on changes to rte_ring APIs, which have a clash from Konstantin's [1] and my patch [2]. Konstantin is working through his patch to address comments.
I am currently incorporating the review comments I received on the RCU defer APIs. We should see the next version soon.
[1] http://mails.dpdk.org/archives/dev/2020-March/160828.html
[2] http://mails.dpdk.org/archives/dev/2020-March/160787.html
Quoted 5 lines
Quoted 3 lines
The patches to LPM and HASH libraries to integrate RCU will depend on this patch.
I guess lpm and hash integrations are planned for 20.08?
Yes, this is pushed to 20.08.
Quoted 1 line
v4 cover Add RCU reclamation APIs
In reply to Honnappa Nagarahalliv4
1) RCU reclamation APIs changed to provide lot more flexibility
a) The rte_rcu_qsbr_dq_enqueue and rte_rcu_qsbr_dq_reclaim APIs can be configured to be MT safe
b) The auto reclamation limit and how much to reclaim
can be configured
c) rte_rcu_qsbr_dq_reclaim API returns the number of resources reclaimed and the number of pending resources on the defer queue
d) rte_rcu_qsbr_dq_reclaim API takes maximum number of resources to reclaim as a parameter
2) Multiple minor fixes
a) Private header file and test function names changed to remove 'rte_'
b) Compilation for shared library
c) Split the test cases into a separate commit
d) Uses rte_ring_xxx_elem APIs to support flexible ring element size
v3
1) Separated from the original series
(https://patches.dpdk.org/cover/58811/)
2) Added reclamation APIs and test cases (Stephen, Yipeng)
Depends on: https://patchwork.dpdk.org/cover/67696/. However it needed changes (will send the comments). So, cannot be tested right now. But, please look at the changes in V4.
This is not a new patch. This patch set is separated from the LPM changes as the size of the changes in RCU library has grown due to comments from community. These APIs will help reduce the changes in LPM and hash libraries that are getting integrated with RCU library.
This adds 4 new APIs to RCU library to create a defer queue, enqueue deleted resources, reclaim resources and delete the defer queue.
The rationale for the APIs is documented in 3/4.
The patches to LPM and HASH libraries to integrate RCU will depend on this patch.
Honnappa Nagarahalli (3):
lib/rcu: add resource reclamation APIs
test/rcu: test cases for RCU defer queue APIs
lib/rcu: add additional debug logs
Ruifeng Wang (1):
doc/rcu: add RCU integration design details
app/test/test_rcu_qsbr.c | 365 ++++++++++++++++++++++++++++- doc/guides/prog_guide/rcu_lib.rst | 59 +++++ lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 2 + lib/librte_rcu/rcu_qsbr_pvt.h | 57 +++++ lib/librte_rcu/rte_rcu_qsbr.c | 243 ++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 197 +++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 9 files changed, 928 insertions(+), 7 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.h
Signature · 1 line
2.17.1
PATCH v4 1/4 lib/rcu: add resource reclamation APIs
Add resource reclamation APIs to make it simple for applications and libraries to integrate rte_rcu library.
--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 2 + lib/librte_rcu/rcu_qsbr_pvt.h | 57 +++++++ lib/librte_rcu/rte_rcu_qsbr.c | 243 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 188 ++++++++++++++++++++++ lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 498 insertions(+), 4 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.h
lib/librte_rcu/Makefile +1 -1
@@ -8,7 +8,7 @@ LIB = librte_rcu.a CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.map
lib/librte_rcu/meson.build +2 -0
@@ -10,3 +10,5 @@ headers = files('rte_rcu_qsbr.h') if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false ext_deps += cc.find_library('atomic') endif + +deps += ['ring']
lib/librte_rcu/rcu_qsbr_pvt.h new file +57 -0
57 lines not on this page · raw message
lib/librte_rcu/rte_rcu_qsbr.c +242 -1
262 lines not on this page · raw message
lib/librte_rcu/rte_rcu_qsbr.h +188 -0
212 lines not on this page · raw message
lib/librte_rcu/rte_rcu_version.map +4 -0
@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };
lib/meson.build +4 -2
@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on this
@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev', - 'rcu', 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', #fib lib depends on rib
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 1/4] lib/rcu: add resource reclamation APIs
Quoted 2 lines
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.
Few nits, thoughts, please see below.
Apart from that - LGTM.
Quoted 215 lines
Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 2 + lib/librte_rcu/rcu_qsbr_pvt.h | 57 +++++++ lib/librte_rcu/rte_rcu_qsbr.c | 243 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 188 ++++++++++++++++++++++ lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 498 insertions(+), 4 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.hlib/librte_rcu/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_rcu.a CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.maplib/librte_rcu/meson.build +2
-0@@ -10,3 +10,5 @@ headers = files('rte_rcu_qsbr.h') if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false ext_deps += cc.find_library('atomic') endif + +deps += ['ring']lib/librte_rcu/rcu_qsbr_pvt.h new file +57
-0@@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2019 Arm Limited +*/ + +#ifndef _RTE_RCU_QSBR_PVT_H_ +#define _RTE_RCU_QSBR_PVT_H_ + +/** +* This file is private to the RCU library. It should not be included +* by the user of this library. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_ring.h> +#include <rte_ring_elem.h> + +#include "rte_rcu_qsbr.h" + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq { +struct rte_rcu_qsbr *v;/**< RCU QSBR variable used by this queue.*/ +struct rte_ring *r;/**< RCU QSBR defer queue. */ +uint32_t size; +/**< Number of elements in the defer queue */ +uint32_t esize; +/**< Size (in bytes) of data, including the token, stored on the +* defer queue. +*/ +uint32_t trigger_reclaim_limit; +/**< Trigger automatic reclamation after the defer queue +* has atleast these many resources waiting. +*/ +uint32_t max_reclaim_size; +/**< Reclaim at the max these many resources during auto +* reclamation. +*/ +rte_rcu_qsbr_free_resource_t free_fn; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. +*/ +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_RCU_QSBR_PVT_H_ */ lib/librte_rcu/rte_rcu_qsbr.c +90
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2018-2019 Arm Limited */ #include <stdio.h> @@ -18,8 +18,10 @@#include <rte_per_lcore.h> #include <rte_lcore.h> #include <rte_errno.h> +#include <rte_ring_elem.h> #include "rte_rcu_qsbr.h" +#include "rcu_qsbr_pvt.h" /* Get the memory size of QSBR variable */ size_t @@ -270,6 +272,245 @@ rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)return 0 ; } +/* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +*/ +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters *params) +{ +struct rte_rcu_qsbr_dq *dq; +uint32_t qs_fifo_size; +unsigned int flags; + +if (params ==NULL || params->free_fn ==NULL || + params->v ==NULL || params->name ==NULL || + params->size ==0 || params->esize ==0 || + (params->esize %4 !=0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return NULL ; + } +/* If auto reclamation is configured, reclaim limit +* should be a valid value. +*/ +if ((params->trigger_reclaim_limit <= params->size) && + (params->max_reclaim_size ==0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter, size = %u, trigger_reclaim_limit = %u, max_reclaim_size = %u\n" , + __func__, params->size, params->trigger_reclaim_limit, + params->max_reclaim_size); + rte_errno =EINVAL ; + +return NULL ; + } + + dq =rte_zmalloc (NULL ,sizeof (struct rte_rcu_qsbr_dq ), +RTE_CACHE_LINE_SIZE ); +if (dq ==NULL ) { + rte_errno =ENOMEM ; + +return NULL ; + } + +/* Decide the flags for the ring. +* If MT safety is requested, use RTS for ring enqueue as most +* use cases involve dq-enqueue happening on the control plane. +* Ring dequeue is always HTS due to the possibility of revert. +*/ + flags =RING_F_MP_RTS_ENQ ; +if (params->flags &RTE_RCU_QSBR_DQ_MT_UNSAFE ) + flags =RING_F_SP_ENQ ; + flags |=RING_F_MC_HTS_DEQ ; +/* round up qs_fifo_size to next power of two that is not less than +* max_size. +*/ + qs_fifo_size =rte_align32pow2 (params->size +1 ); +/* Add token size to ring element size */ + dq->r =rte_ring_create_elem (params->name, + __RTE_QSBR_TOKEN_SIZE + params->esize, + qs_fifo_size,SOCKET_ID_ANY , flags); +if (dq->r ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): defer queue create failed\n" , __func__); +rte_free (dq); +return NULL ; + } + + dq->v = params->v; + dq->size = params->size; + dq->esize = __RTE_QSBR_TOKEN_SIZE + params->esize; + dq->trigger_reclaim_limit = params->trigger_reclaim_limit; + dq->max_reclaim_size = params->max_reclaim_size; + dq->free_fn = params->free_fn; + dq->p = params->p; + +return dq; +} + +/* Enqueue one resource to the defer queue to free after the grace +* period is over. +*/ +int rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e) +{ +uint64_t token; +uint32_t cur_size, free_size; + +if (dq ==NULL || e ==NULL ) { + rte_log(RTE_LOG_ERR, rte_rcu_log_type,
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 371
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+ "%s(): Invalid input parameter\n", __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Start the grace period */ + token =rte_rcu_qsbr_start (dq->v); + +/* Reclaim resources if the queue is 1/8th full. This helps
Comment about 1/8 is probably left from older version? As I understand now it is configurable parameter.
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 384
Quoted 13 lines
lib/librte_rcu/rte_rcu_qsbr.c +13
-0+* the queue from growing too large and allows time for reader +* threads to report their quiescent state. +*/ + cur_size =rte_ring_count (dq->r); +if (cur_size > dq->trigger_reclaim_limit) { +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Triggering reclamation\n" , __func__); +rte_rcu_qsbr_dq_reclaim (dq, dq->max_reclaim_size,NULL ,NULL ); + } + +/* Check if there is space for atleast 1 resource */ + free_size =rte_ring_free_count (dq->r); +if (!free_size) {
Is there any point to do this check at all? You are doing enqueue below and handle situation with not enough space in the ring anyway.
Quoted 32 lines
lib/librte_rcu/rte_rcu_qsbr.c +32
-0+rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Defer queue is full\n" , __func__); +/* Note that the token generated above is not used. +* Other than wasting tokens, it should not cause any +* other issues. +*/ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Skipped enqueuing token = %" PRIu64"\n" , + __func__, token); + + rte_errno =ENOSPC ; +return 1 ; + } + +/* Enqueue the token and resource. Generating the token +* and enqueuing (token + resource) on the queue is not an +* atomic operation. This might result in tokens enqueued +* out of order on the queue. So, some tokens might wait +* longer than they are required to be reclaimed. +*/ +char data[dq->esize]; +memcpy (data, &token, __RTE_QSBR_TOKEN_SIZE); +memcpy (data + __RTE_QSBR_TOKEN_SIZE, e, + dq->esize - __RTE_QSBR_TOKEN_SIZE); +/* Check the status as enqueue might fail since the other thread +* might have used up the freed space. +* Enqueue uses the configured flags when the DQ was created. +*/ +if (rte_ring_enqueue_elem (dq->r, data, dq->esize) !=0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Enqueue failed\n" , __func__); +/* Note that the token generated above is not used.
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 426
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+* Other than wasting tokens, it should not cause any +* other issues. +*/ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Skipped enqueuing token = %" PRIu64"\n" , + __func__, token); + + rte_errno =ENOSPC ; +return 1 ; + }
Just as a thought: in theory if we'll use MP_HTS(/SP) ring we can avoid wasting RCU tokens:
if (rte_ring_enqueue_elem_bulk_start(dq->r, 1, NULL) != 0) {
token = rte_rcu_qsbr_start(dq->v);
memcpy(data, &token, __RTE_QSBR_TOKEN_SIZE);
rte_ring_enqueue_elem_finish(dq->r, data, dq->esize, 1);
}Though it might slowdown things if we'll have a lot of parallel dq_enqueue.
So not sure is it worth it or not.
Quoted 20 lines
lib/librte_rcu/rte_rcu_qsbr.c +20
-0+ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Enqueued token = %" PRIu64"\n" , __func__, token); + +return 0 ; +} + +/* Reclaim resources from the defer queue. */ +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending) +{ +uint32_t cnt; +uint64_t token; + +if (dq ==NULL || n ==0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; +
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 456
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+return 1 ; + } + + cnt =0 ; + +char e[dq->esize]; +/* Check reader threads quiescent state and reclaim resources */ +while ((cnt < n) && + (rte_ring_dequeue_bulk_elem_start (dq->r, e, + dq->esize,1 ,NULL ) !=0 )) {
Another thought - any point to use burst_elem_start() here to retrieve more then 1 elem in one go? Something like:
char e[32][dq->size];
while ((cnt < n) {
k = RTE_MAX(32, cnt - n);
k = rte_ring_dequeue_burst_elem_start(dq->r, e, dq->esize, k, NULL);
if (k = 0)
break;
for (i = 0; i != k; i++) {
memcpy(&token, e[i], sizeof(uint64_t));
if (rte_rcu_qsbr_check(dq->v, token, false) != 1)
break;
}
k = i;
rte_ring_dequeue_elem_finish(dq->r, k);
for (i = 0; i != k; i++)
dq->free_fn(dq->p, e[i] + __RTE_QSBR_TOKEN_SIZE);
n += k;
if (k == 0)
break;?
Also if at enqueue we guarantee strict ordrer (via enqueue_start/enqueue_finish), then here we probably can do _check_ from the last retrieved token here? In theory that might help to minimize number of checks.
I.E. do:
for (i = k; i-- !=0; ) {
memcpy(&token, e[i], sizeof(uint64_t));
if (rte_rcu_qsbr_check(dq->v, token, false) != 1)
break;
}k = i + 1;
...
Quoted 321 lines
lib/librte_rcu/rte_rcu_qsbr.c +57
-0+memcpy (&token, e,sizeof (uint64_t )); + +/* Reclaim the resource */ +if (rte_rcu_qsbr_check (dq->v, token, false) !=1 ) { +rte_ring_dequeue_finish (dq->r,0 ); +break ; + } +rte_ring_dequeue_finish (dq->r,1 ); + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed token = %" PRIu64"\n" , + __func__, *(uint64_t *)e); + + dq->free_fn (dq->p, e + __RTE_QSBR_TOKEN_SIZE); + + cnt++; + } + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed %u resources\n" , __func__, cnt); + +if (freed !=NULL ) + *freed = cnt; +if (pending !=NULL ) + *pending =rte_ring_count (dq->r); + +return 0 ; +} + +/* Delete a defer queue. */ +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq) +{ +unsigned int pending; + +if (dq ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Reclaim all the resources */ +rte_rcu_qsbr_dq_reclaim (dq, ~0 ,NULL , &pending); +if (pending !=0 ) { + rte_errno =EAGAIN ; + +return 1 ; + } + +rte_ring_free (dq->r); +rte_free (dq); + +return 0 ; +} +int rte_rcu_log_type;RTE_INIT (rte_rcu_register)lib/librte_rcu/rte_rcu_qsbr.h +188
-0@@ -34,6 +34,7 @@ extern "C" {#include <rte_lcore.h> #include <rte_debug.h> #include <rte_atomic.h> +#include <rte_ring.h> extern int rte_rcu_log_type;@@ -84,6 +85,7 @@ struct rte_rcu_qsbr_cnt {#define __RTE_QSBR_CNT_THR_OFFLINE 0#define __RTE_QSBR_CNT_INIT 1#define __RTE_QSBR_CNT_MAX ((uint64_t)~0) +#define __RTE_QSBR_TOKEN_SIZE sizeof(uint64_t)/* RTE Quiescent State variable structure. * This structure has two elements that vary in size based on the @@ -114,6 +116,84 @@ struct rte_rcu_qsbr {*/ } __rte_cache_aligned; +/** +* Call back function called to free the resources. +* +* @param p +* Pointer provided while creating the defer queue +* @param e +* Pointer to the resource data stored on the defer queue +* +* @return +* None +*/ +typedef void (*rte_rcu_qsbr_free_resource_t )(void *p,void *e); + +#define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE + +/** +* Various flags supported. +*/ +/**< Enqueue and reclaim operations are multi-thread safe by default. +* The call back functions registered to free the resources are +* assumed to be multi-thread safe. +* Set this flag is multi-thread safety is not required. +*/ +#define RTE_RCU_QSBR_DQ_MT_UNSAFE 1 + +/** +* Parameters used when creating the defer queue. +*/ +struct rte_rcu_qsbr_dq_parameters { +const char *name; +/**< Name of the queue. */ +uint32_t flags; +/**< Flags to control API behaviors */ +uint32_t size; +/**< Number of entries in queue. Typically, this will be +* the same as the maximum number of entries supported in the +* lock free data structure. +* Data structures with unbounded number of entries is not +* supported currently. +*/ +uint32_t esize; +/**< Size (in bytes) of each element in the defer queue. +* This has to be multiple of 4B. +*/ +uint32_t trigger_reclaim_limit; +/**< Trigger automatic reclamation after the defer queue +* has atleast these many resources waiting. This auto +* reclamation is triggered in rte_rcu_qsbr_dq_enqueue API +* call. +* If this is greater than 'size', auto reclamation is +* not triggered. +* If this is set to 0, auto reclamation is triggered +* in every call to rte_rcu_qsbr_dq_enqueue API. +*/ +uint32_t max_reclaim_size; +/**< When automatic reclamation is enabled, reclaim at the max +* these many resources. This should contain a valid value, if +* auto reclamation is on. Setting this to 'size' or greater will +* reclaim all possible resources currently on the defer queue. +*/ +rte_rcu_qsbr_free_resource_t free_fn; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. This can be NULL. +*/ +struct rte_rcu_qsbr *v; +/**< RCU QSBR variable to use for this defer queue */ +}; + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq ; +/** * @warning * @b EXPERIMENTAL: this API may change without prior notice @@ -692,6 +772,114 @@ __rte_experimentalint rte_rcu_qsbr_dump (FILE *f,struct rte_rcu_qsbr *v); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +* +* @param params +* Parameters to create a defer queue. +* @return +* On success - Valid pointer to defer queue +* On error - NULL +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - ENOMEM - Not enough memory +*/ +__rte_experimental +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters *params); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Enqueue one resource to the defer queue and start the grace period. +* The resource will be freed later after at least one grace period +* is over. +* +* If the defer queue is full, it will attempt to reclaim resources. +* It will also reclaim resources at regular intervals to avoid +* the defer queue from growing too big. +* +* Multi-thread safety is provided as the defer queue configuration. +* When multi-thread safety is requested, it is possible that the +* resources are not stored in their order of deletion. This results +* in resources being held in the defer queue longer than they should. +* +* @param dq +* Defer queue to allocate an entry from. +* @param e +* Pointer to resource data to copy to the defer queue. The size of +* the data to copy is equal to the element size provided when the +* defer queue was created. +* @return +* On success - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +* - ENOSPC - Defer queue is full. This condition can not happen +* if the defer queue size is equal (or larger) than the +* number of elements in the data structure. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Free quesed resources from the defer queue. +* +* This API is multi-thread safe. +* +* @param dq +* Defer queue to free an entry from. +* @param n +* Maximum number of resources to free. +* @param freed +* Number of resources that were freed. +* @param pending +* Number of resources pending on the defer queue. This number might not +* be acurate if multi-thread safety is configured. +* @return +* On successful reclamation of at least 1 resource - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Delete a defer queue. +* +* It tries to reclaim all the resources on the defer queue. +* If any of the resources have not completed the grace period +* the reclamation stops and returns immediately. The rest of +* the resources are not reclaimed and the defer queue is not +* freed. +* +* @param dq +* Defer queue to delete. +* @return +* On success - 0 +* On error - 1 +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - EAGAIN - Some of the resources have not completed at least 1 grace +* period, try again. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq); + #ifdef __cplusplus }#endif lib/librte_rcu/rte_rcu_version.map +4
-0@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };lib/meson.build +4
-3@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on this@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev', - 'rcu', 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', #fib lib depends on rib --2.17.1
Re: [dpdk-dev] [PATCH v4 1/4] lib/rcu: add resource reclamation APIs
<snip>
Quoted 226 lines
Quoted 3 lines
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Few nits, thoughts, please see below.
Apart from that - LGTM.Quoted 142 lines
Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 2 + lib/librte_rcu/rcu_qsbr_pvt.h | 57 +++++++ lib/librte_rcu/rte_rcu_qsbr.c | 243 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 188 ++++++++++++++++++++++ lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 498 insertions(+), 4 deletions(-) create mode100644
lib/librte_rcu/rcu_qsbr_pvt.hlib/librte_rcu/Makefile index renamed from lib/librte_rcu/Makefile +0
-0
c4bb28d77..95f8a57e2100644lib/librte_rcu/Makefile +1
-0@@ -8,7 +8,7 @@ LIB = librte_rcu.a CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.maplib/librte_rcu/meson.build +0
-0@@ -10,3 +10,5 @@ headers = files('rte_rcu_qsbr.h') if cc.get_id() =='clang' and
dpdk_conf.get('RTE_ARCH_64')== falselib/librte_rcu/meson.build +2
-0ext_deps += cc.find_library('atomic') endif + +deps += ['ring'](quoted diff) +0
-0
b/lib/librte_rcu/rcu_qsbr_pvt.hnew file mode 100644 index 000000000..413f28587lib/librte_rcu/rcu_qsbr_pvt.h new file +57
-0@@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2019 Arm Limited +*/ + +#ifndef _RTE_RCU_QSBR_PVT_H_ +#define _RTE_RCU_QSBR_PVT_H_ + +/** +* This file is private to the RCU library. It should not be included +* by the user of this library. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_ring.h> +#include <rte_ring_elem.h> + +#include "rte_rcu_qsbr.h" + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq { +struct rte_rcu_qsbr *v;/**< RCU QSBR variable used by this queue.*/ +struct rte_ring *r;/**< RCU QSBR defer queue. */ +uint32_t size; +/**< Number of elements in the defer queue */ +uint32_t esize; +/**< Size (in bytes) of data, including the token, stored on the +* defer queue. +*/ +uint32_t trigger_reclaim_limit; +/**< Trigger automatic reclamation after the defer queue +* has atleast these many resources waiting. +*/ +uint32_t max_reclaim_size; +/**< Reclaim at the max these many resources during auto +* reclamation. +*/ +rte_rcu_qsbr_free_resource_t free_fn; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. +*/ +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_RCU_QSBR_PVT_H_ */ (quoted diff) +0
-0
b/lib/librte_rcu/rte_rcu_qsbr.cindex2f3fad776..e8c1e386f100644lib/librte_rcu/rte_rcu_qsbr.c +3
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2018-2019 Arm Limited */ #include <stdio.h> @@ -18,8 +18,10 @@#include <rte_per_lcore.h> #include <rte_lcore.h> #include <rte_errno.h> +#include <rte_ring_elem.h> #include "rte_rcu_qsbr.h" +#include "rcu_qsbr_pvt.h" /* Get the memory size of QSBR variable */ size_t @@ -270,6 +272,245 @@ rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)lib/librte_rcu/rte_rcu_qsbr.c +17
-0return 0 ; } +/* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +*/ +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters +*params) { +struct rte_rcu_qsbr_dq *dq; +uint32_t qs_fifo_size; +unsigned int flags; + +if (params ==NULL || params->free_fn ==NULL || + params->v ==NULL || params->name ==NULL || + params->size ==0 || params->esize ==0 || + (params->esize %4 !=0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ;lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+ +return NULL ; + } +/* If auto reclamation is configured, reclaim limit +* should be a valid value. +*/ +if ((params->trigger_reclaim_limit <= params->size) && + (params->max_reclaim_size ==0 )) { + rte_log(RTE_LOG_ERR, rte_rcu_log_type, + "%s(): Invalid input parameter, size = %u,
trigger_reclaim_limit= %u,max_reclaim_size= %u\n",Quoted 65 lines
lib/librte_rcu/rte_rcu_qsbr.c +1
-0+ __func__, params->size, params-
trigger_reclaim_limit,lib/librte_rcu/rte_rcu_qsbr.c +63
-0+ params->max_reclaim_size); + rte_errno =EINVAL ; + +return NULL ; + } + + dq =rte_zmalloc (NULL ,sizeof (struct rte_rcu_qsbr_dq ), +RTE_CACHE_LINE_SIZE ); +if (dq ==NULL ) { + rte_errno =ENOMEM ; + +return NULL ; + } + +/* Decide the flags for the ring. +* If MT safety is requested, use RTS for ring enqueue as most +* use cases involve dq-enqueue happening on the control plane. +* Ring dequeue is always HTS due to the possibility of revert. +*/ + flags =RING_F_MP_RTS_ENQ ; +if (params->flags &RTE_RCU_QSBR_DQ_MT_UNSAFE ) + flags =RING_F_SP_ENQ ; + flags |=RING_F_MC_HTS_DEQ ; +/* round up qs_fifo_size to next power of two that is not less than +* max_size. +*/ + qs_fifo_size =rte_align32pow2 (params->size +1 ); +/* Add token size to ring element size */ + dq->r =rte_ring_create_elem (params->name, + __RTE_QSBR_TOKEN_SIZE + params->esize, + qs_fifo_size,SOCKET_ID_ANY , flags); +if (dq->r ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): defer queue create failed\n" , __func__); +rte_free (dq); +return NULL ; + } + + dq->v = params->v; + dq->size = params->size; + dq->esize = __RTE_QSBR_TOKEN_SIZE + params->esize; + dq->trigger_reclaim_limit = params->trigger_reclaim_limit; + dq->max_reclaim_size = params->max_reclaim_size; + dq->free_fn = params->free_fn; + dq->p = params->p; + +return dq; +} + +/* Enqueue one resource to the defer queue to free after the grace +* period is over. +*/ +int rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e) { +uint64_t token; +uint32_t cur_size, free_size; + +if (dq ==NULL || e ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + }
Quoted 8 lines
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 371
Quoted 5 lines
lib/librte_rcu/rte_rcu_qsbr.c +5
-0+ +/* Start the grace period */ + token =rte_rcu_qsbr_start (dq->v); + +/* Reclaim resources if the queue is 1/8th full. This helps Comment about 1/8 is probably left from older version? As I understand now it is configurable parameter.
Ack, will correct this.
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)
Quoted 9 lines
lib/librte_rcu/rte_rcu_qsbr.c +8
-0+* the queue from growing too large and allows time for reader +* threads to report their quiescent state. +*/ + cur_size =rte_ring_count (dq->r); +if (cur_size > dq->trigger_reclaim_limit) { +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Triggering reclamation\n" , __func__); + rte_rcu_qsbr_dq_reclaim(dq, dq->max_reclaim_size, NULL,NULL);
Quoted 9 lines
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 384
Quoted 5 lines
lib/librte_rcu/rte_rcu_qsbr.c +5
-0+ } + +/* Check if there is space for atleast 1 resource */ + free_size =rte_ring_free_count (dq->r); +if (!free_size) {Is there any point to do this check at all?
You are doing enqueue below and handle situation with not enough space in the ring anyway.
Ack
Quoted 48 lines
Quoted 33 lines
lib/librte_rcu/rte_rcu_qsbr.c +32
-0+rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Defer queue is full\n" , __func__); +/* Note that the token generated above is not used. +* Other than wasting tokens, it should not cause any +* other issues. +*/ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Skipped enqueuing token = %" PRIu64"\n" , + __func__, token); + + rte_errno =ENOSPC ; +return 1 ; + } + +/* Enqueue the token and resource. Generating the token +* and enqueuing (token + resource) on the queue is not an +* atomic operation. This might result in tokens enqueued +* out of order on the queue. So, some tokens might wait +* longer than they are required to be reclaimed. +*/ +char data[dq->esize]; +memcpy (data, &token, __RTE_QSBR_TOKEN_SIZE); +memcpy (data + __RTE_QSBR_TOKEN_SIZE, e, + dq->esize - __RTE_QSBR_TOKEN_SIZE); +/* Check the status as enqueue might fail since the other thread +* might have used up the freed space. +* Enqueue uses the configured flags when the DQ was created. +*/ +if (rte_ring_enqueue_elem (dq->r, data, dq->esize) !=0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Enqueue failed\n" , __func__); +/* Note that the token generated above is not used. lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 426
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+* Other than wasting tokens, it should not cause any +* other issues. +*/ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Skipped enqueuing token = %" PRIu64"\n" , + __func__, token); + + rte_errno =ENOSPC ; +return 1 ; + }Just as a thought: in theory if we'll use
MP_HTS(/SP)ring we can avoid wasting RCU tokens:
Quoted 7 lines
if (rte_ring_enqueue_elem_bulk_start(dq->r, 1, NULL) != 0) { token = rte_rcu_qsbr_start(dq->v); memcpy(data, &token, __RTE_QSBR_TOKEN_SIZE); rte_ring_enqueue_elem_finish(dq->r, data, dq->esize, 1); }Though it might slowdown things if we'll have a lot of parallel
dq_enqueue. So not sure is it worth it or not.
Adding peek APIs for RTS would be better. That should take care of the parallel dw_enqueue. Not sure if I gave you the comment. My ring patch supported these APIs.
Quoted 24 lines
Quoted 24 lines
lib/librte_rcu/rte_rcu_qsbr.c +23
-0+ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Enqueued token = %" PRIu64"\n" , __func__, token); + +return 0 ; +} + +/* Reclaim resources from the defer queue. */ int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending) { +uint32_t cnt; +uint64_t token; + +if (dq ==NULL || n ==0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + + cnt =0 ; +
Quoted 8 lines
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 456
Quoted 5 lines
lib/librte_rcu/rte_rcu_qsbr.c +5
-0+char e[dq->esize]; +/* Check reader threads quiescent state and reclaim resources */ +while ((cnt < n) && + (rte_ring_dequeue_bulk_elem_start (dq->r, e, + dq->esize,1 ,NULL ) !=0 )) {Another thought - any point to use
burst_elem_start()here to retrieve more then 1 elem in one go? Something like:
I think it makes sense.
Quoted 15 lines
char e[32][dq->size];
while ((cnt < n) { k = RTE_MAX(32, cnt - n); k = rte_ring_dequeue_burst_elem_start(dq->r, e, dq->esize, k, NULL); if (k = 0) break; for (i = 0; i != k; i++) { memcpy(&token, e[i], sizeof(uint64_t)); if (rte_rcu_qsbr_check(dq->v, token, false) != 1) break; } k = i; rte_ring_dequeue_elem_finish(dq->r, k); for (i = 0; i != k; i++) dq->free_fn(dq->p, e[i] + __RTE_QSBR_TOKEN_SIZE);
I think it also makes sense to change the free_fn to take 'n' number of tokens.
Quoted 10 lines
n += k; if (k == 0) break;?
Also if at enqueue we guarantee strict ordrer (via enqueue_start/enqueue_finish), then here we probably can do _check_ from the last retrieved token here?
In theory that might help to minimize number of checks.
I.E. do:
Quoted 3 lines
for (i = k; i-- !=0; ) {
memcpy(&token, e[i], sizeof(uint64_t));
if (rte_rcu_qsbr_check(dq->v, token, false) != 1)There is a higher chance that later tokens are not acked. This introduces more polling of the counters.
The rte_rcu_qsbr_check has an optimization. While acking the current token, it will also caches the greatest token acked. It uses the cached token for the subsequent calls. I think this provides a better optimization.
Quoted 326 lines
break; }k = i + 1;
...Quoted 72 lines
lib/librte_rcu/rte_rcu_qsbr.c +56
-0+memcpy (&token, e,sizeof (uint64_t )); + +/* Reclaim the resource */ +if (rte_rcu_qsbr_check (dq->v, token, false) !=1 ) { +rte_ring_dequeue_finish (dq->r,0 ); +break ; + } +rte_ring_dequeue_finish (dq->r,1 ); + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed token = %" PRIu64"\n" , + __func__, *(uint64_t *)e); + + dq->free_fn (dq->p, e + __RTE_QSBR_TOKEN_SIZE); + + cnt++; + } + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed %u resources\n" , __func__, cnt); + +if (freed !=NULL ) + *freed = cnt; +if (pending !=NULL ) + *pending =rte_ring_count (dq->r); + +return 0 ; +} + +/* Delete a defer queue. */ +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq) { +unsigned int pending; + +if (dq ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +/* Reclaim all the resources */ +rte_rcu_qsbr_dq_reclaim (dq, ~0 ,NULL , &pending); +if (pending !=0 ) { + rte_errno =EAGAIN ; + +return 1 ; + } + +rte_ring_free (dq->r); +rte_free (dq); + +return 0 ; +} +int rte_rcu_log_type;RTE_INIT (rte_rcu_register)(quoted diff) +0
-0
b/lib/librte_rcu/rte_rcu_qsbr.hindex0b5585925..213f9b029100644lib/librte_rcu/rte_rcu_qsbr.h +1
-0@@ -34,6 +34,7 @@ extern "C" {#include <rte_lcore.h> #include <rte_debug.h> #include <rte_atomic.h> +#include <rte_ring.h> extern int rte_rcu_log_type;@@ -84,6 +85,7 @@ struct rte_rcu_qsbr_cnt { #defineQuoted 1 line
__RTE_QSBR_CNT_THR_OFFLINE0 #define__RTE_QSBR_CNT_INIT1#define
Quoted 154 lines
__RTE_QSBR_CNT_MAX((uint64_t)~0)lib/librte_rcu/rte_rcu_qsbr.h +78
-2+#define __RTE_QSBR_TOKEN_SIZE sizeof(uint64_t)/* RTE Quiescent State variable structure. * This structure has two elements that vary in size based on the @@ -114,6 +116,84 @@ struct rte_rcu_qsbr { */ } __rte_cache_aligned; +/** +* Call back function called to free the resources. +* +* @param p +* Pointer provided while creating the defer queue +* @param e +* Pointer to the resource data stored on the defer queue +* +* @return +* None +*/ +typedef void (*rte_rcu_qsbr_free_resource_t )(void *p,void *e); + +#define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE + +/** +* Various flags supported. +*/ +/**< Enqueue and reclaim operations are multi-thread safe by default. +* The call back functions registered to free the resources are +* assumed to be multi-thread safe. +* Set this flag is multi-thread safety is not required. +*/ +#define RTE_RCU_QSBR_DQ_MT_UNSAFE 1 + +/** +* Parameters used when creating the defer queue. +*/ +struct rte_rcu_qsbr_dq_parameters { +const char *name; +/**< Name of the queue. */ +uint32_t flags; +/**< Flags to control API behaviors */ +uint32_t size; +/**< Number of entries in queue. Typically, this will be +* the same as the maximum number of entries supported in the +* lock free data structure. +* Data structures with unbounded number of entries is not +* supported currently. +*/ +uint32_t esize; +/**< Size (in bytes) of each element in the defer queue. +* This has to be multiple of 4B. +*/ +uint32_t trigger_reclaim_limit; +/**< Trigger automatic reclamation after the defer queue +* has atleast these many resources waiting. This auto +* reclamation is triggered in rte_rcu_qsbr_dq_enqueue API +* call. +* If this is greater than 'size', auto reclamation is +* not triggered. +* If this is set to 0, auto reclamation is triggered +* in every call to rte_rcu_qsbr_dq_enqueue API. +*/ +uint32_t max_reclaim_size; +/**< When automatic reclamation is enabled, reclaim at the max +* these many resources. This should contain a valid value, if +* auto reclamation is on. Setting this to 'size' or greater will +* reclaim all possible resources currently on the defer queue. +*/ +rte_rcu_qsbr_free_resource_t free_fn; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. This can be NULL. +*/ +struct rte_rcu_qsbr *v; +/**< RCU QSBR variable to use for this defer queue */ }; + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq ; +/** * @warning * @b EXPERIMENTAL: this API may change without prior notice @@ -692,6 +772,114 @@ __rte_experimental int rte_rcu_qsbr_dump(FILE *f, struct
rte_rcu_qsbr*v);lib/librte_rcu/rte_rcu_qsbr.h +62
-0+/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +* +* @param params +* Parameters to create a defer queue. +* @return +* On success - Valid pointer to defer queue +* On error - NULL +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - ENOMEM - Not enough memory +*/ +__rte_experimental +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters +*params); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Enqueue one resource to the defer queue and start the grace period. +* The resource will be freed later after at least one grace period +* is over. +* +* If the defer queue is full, it will attempt to reclaim resources. +* It will also reclaim resources at regular intervals to avoid +* the defer queue from growing too big. +* +* Multi-thread safety is provided as the defer queue configuration. +* When multi-thread safety is requested, it is possible that the +* resources are not stored in their order of deletion. This results +* in resources being held in the defer queue longer than they should. +* +* @param dq +* Defer queue to allocate an entry from. +* @param e +* Pointer to resource data to copy to the defer queue. The size of +* the data to copy is equal to the element size provided when the +* defer queue was created. +* @return +* On success - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +* - ENOSPC - Defer queue is full. This condition can not happen +* if the defer queue size is equal (or larger) than the +* number of elements in the data structure. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Free quesed resources from the defer queue. +* lib/librte_rcu/rte_rcu_qsbr.h __rte_experimental
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.h +10
-0+* This API is multi-thread safe. +* +* @param dq +* Defer queue to free an entry from. +* @param n +* Maximum number of resources to free. +* @param freed +* Number of resources that were freed. +* @param pending +* Number of resources pending on the defer queue. This number might not
Quoted 82 lines
+ * be acurate if multi-thread safety is configured.
+ * @return+ * On successful reclamation of at least 1 resource - 0 + * On error - 1 with rte_errno set to + * - EINVAL - NULL parameters are passed+ */
lib/librte_rcu/rte_rcu_qsbr.h +31
-0+__rte_experimental +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Delete a defer queue. +* +* It tries to reclaim all the resources on the defer queue. +* If any of the resources have not completed the grace period +* the reclamation stops and returns immediately. The rest of +* the resources are not reclaimed and the defer queue is not +* freed. +* +* @param dq +* Defer queue to delete. +* @return +* On success - 0 +* On error - 1 +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - EAGAIN - Some of the resources have not completed at least 1 grace +* period, try again. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq); + #ifdef __cplusplus }#endif lib/librte_rcu/rte_rcu_version.map +4
-0@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };lib/meson.build index renamed from lib/meson.build +0
-0
9c3cc55d5..15e91a303100644lib/meson.build +4
-3@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on this@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev', - 'rcu', 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', #fib lib depends on rib --2.17.1
Re: [dpdk-dev] [PATCH v4 1/4] lib/rcu: add resource reclamation APIs
Quoted 39 lines
Quoted 39 lines
Quoted 19 lines
lib/librte_rcu/rte_rcu_qsbr.c +19
-0+ +/* Enqueue the token and resource. Generating the token +* and enqueuing (token + resource) on the queue is not an +* atomic operation. This might result in tokens enqueued +* out of order on the queue. So, some tokens might wait +* longer than they are required to be reclaimed. +*/ +char data[dq->esize]; +memcpy (data, &token, __RTE_QSBR_TOKEN_SIZE); +memcpy (data + __RTE_QSBR_TOKEN_SIZE, e, + dq->esize - __RTE_QSBR_TOKEN_SIZE); +/* Check the status as enqueue might fail since the other thread +* might have used up the freed space. +* Enqueue uses the configured flags when the DQ was created. +*/ +if (rte_ring_enqueue_elem (dq->r, data, dq->esize) !=0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Enqueue failed\n" , __func__); +/* Note that the token generated above is not used. lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 426
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.c +10
-0+* Other than wasting tokens, it should not cause any +* other issues. +*/ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Skipped enqueuing token = %" PRIu64"\n" , + __func__, token); + + rte_errno =ENOSPC ; +return 1 ; + }Just as a thought: in theory if we'll use
MP_HTS(/SP)ring we can avoid wasting RCU tokens:if (rte_ring_enqueue_elem_bulk_start(dq->r, 1, NULL) != 0) { token = rte_rcu_qsbr_start(dq->v); memcpy(data, &token, __RTE_QSBR_TOKEN_SIZE); rte_ring_enqueue_elem_finish(dq->r, data, dq->esize, 1); }
Quoted 4 lines
Quoted 2 lines
Though it might slowdown things if we'll have a lot of parallel
dq_enqueue. So not sure is it worth it or not.Adding peek APIs for RTS would be better. That should take care of the parallel
dw_enqueue. Not sure if I gave you the comment. My ring patch supported these APIs.
AFAIK, peek API is not possible for RTS mode.
Probably you are talking about Scatter-Gather API introduced in your RFC (_reserve_; update ring entries manually; _commit_)?
Anyway, if there is no much value in my idea above, then feel free to drop it.
Quoted 65 lines
Quoted 25 lines
Quoted 25 lines
lib/librte_rcu/rte_rcu_qsbr.c +23
-0+ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Enqueued token = %" PRIu64"\n" , __func__, token); + +return 0 ; +} + +/* Reclaim resources from the defer queue. */ int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending) { +uint32_t cnt; +uint64_t token; + +if (dq ==NULL || n ==0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + + cnt =0 ; +Quoted 8 lines
lib/librte_rcu/rte_rcu_qsbr.c rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v) · line 456
Quoted 5 lines
lib/librte_rcu/rte_rcu_qsbr.c +5
-0+char e[dq->esize]; +/* Check reader threads quiescent state and reclaim resources */ +while ((cnt < n) && + (rte_ring_dequeue_bulk_elem_start (dq->r, e, + dq->esize,1 ,NULL ) !=0 )) {Another thought - any point to use
burst_elem_start()here to retrieve more then 1 elem in one go? Something like:I think it makes sense.
Quoted 15 lines
char e[32][dq->size];
while ((cnt < n) { k = RTE_MAX(32, cnt - n); k = rte_ring_dequeue_burst_elem_start(dq->r, e, dq->esize, k, NULL); if (k = 0) break; for (i = 0; i != k; i++) { memcpy(&token, e[i], sizeof(uint64_t)); if (rte_rcu_qsbr_check(dq->v, token, false) != 1) break; } k = i; rte_ring_dequeue_elem_finish(dq->r, k); for (i = 0; i != k; i++) dq->free_fn(dq->p, e[i] + __RTE_QSBR_TOKEN_SIZE);I think it also makes sense to change the
free_fnto take 'n' number of tokens.Quoted 10 lines
n += k; if (k == 0) break;?
Also if at enqueue we guarantee strict ordrer (via enqueue_start/enqueue_finish), then here we probably can do _check_ from the last retrieved token here?
In theory that might help to minimize number of checks.
I.E. do:Quoted 3 lines
for (i = k; i-- !=0; ) { memcpy(&token, e[i], sizeof(uint64_t)); if (rte_rcu_qsbr_check(dq->v, token, false) != 1)
Quoted 3 lines
There is a higher chance that later tokens are not acked. This introduces more polling of the counters.
Therte_rcu_qsbr_checkhas an optimization. While acking the current token, it will also caches the greatest token acked. It uses the cached token for the subsequent calls. I think this provides a better optimization.
Ok.
PATCH v4 2/4 test/rcu: test cases for RCU defer queue APIs
In reply to Honnappa NagarahalliAdd test cases for RCU defer queue APIs.
--- app/test/test_rcu_qsbr.c | 365 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 363 insertions(+), 2 deletions(-)
app/test/test_rcu_qsbr.c +363 -2
@@ -1,8 +1,9 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2019 Arm Limited */ #include <stdio.h> +#include <string.h> #include <rte_pause.h> #include <rte_rcu_qsbr.h> #include <rte_hash.h>
@@ -15,7 +16,8 @@#include "test.h" /* Check condition and return an error if true. */ -#define TEST_RCU_QSBR_RETURN_IF_ERROR (cond, str, ...) do { \ +#define TEST_RCU_QSBR_RETURN_IF_ERROR (cond, str, ...) \ +do { \ if (cond) { \ printf("ERROR file %s, line %d: " str "\n", __FILE__, \ __LINE__, ##__VA_ARGS__); \
@@ -23,6 +25,16 @@ } \ }while (0 ) +/* Check condition and go to label if true. */ +#define TEST_RCU_QSBR_GOTO_IF_ERROR (label, cond, str, ...) \ +do { \ + if (cond) { \ + printf("ERROR file %s, line %d: " str "\n", __FILE__, \ + __LINE__, ##__VA_ARGS__); \ + goto label; \ + } \ +} while (0) +/* Make sure that this has the same value as __RTE_QSBR_CNT_INIT */ #define TEST_RCU_QSBR_CNT_INIT 1
@@ -34,6 +46,7 @@ static uint32_t *keys;#define COUNTER_VALUE 4096static uint32_t *hash_data[RTE_MAX_LCORE ][TOTAL_ENTRY ];static uint8_t writer_done; +static uint8_t cb_failed;static struct rte_rcu_qsbr *t[RTE_MAX_LCORE ];static struct rte_hash *h[RTE_MAX_LCORE ];
@@ -585,6 +598,330 @@ test_rcu_qsbr_thread_offline(void)return 0 ; } +static void +test_rcu_qsbr_free_resource1 (void *p,void *e) +{ +if (p !=NULL || e !=NULL ) { +printf ("%s: Test failed\n" , __func__); + cb_failed =1 ; + } +} + +static void +test_rcu_qsbr_free_resource2 (void *p,void *e) +{ +if (p !=NULL || e ==NULL ) { +printf ("%s: Test failed\n" , __func__); + cb_failed =1 ; + } +} + +/* +* rte_rcu_qsbr_dq_create: create a queue used to store the data structure +* elements that can be freed later. This queue is referred to as 'defer queue'. +*/ +static int +test_rcu_qsbr_dq_create (void ) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_create()\n" ); + +/* Pass invalid parameters */ + dq =rte_rcu_qsbr_dq_create (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.free_fn = test_rcu_qsbr_free_resource1; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.size =1 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.esize =3 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size =0 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +/* Pass all valid parameters */ + params.esize =16 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); +rte_rcu_qsbr_dq_delete (dq); + + params.esize =16 ; + params.flags =RTE_RCU_QSBR_DQ_MT_UNSAFE ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); +rte_rcu_qsbr_dq_delete (dq); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_enqueue (void ) +{ +int ret; +uint64_t r; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_enqueue()\n" ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.free_fn = test_rcu_qsbr_free_resource1; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_enqueue (NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid params" ); + + ret =rte_rcu_qsbr_dq_enqueue (dq,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid params" ); + + ret =rte_rcu_qsbr_dq_enqueue (NULL , &r); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid params" ); + + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==1 ),"dq delete valid params" ); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_reclaim: Reclaim resources from the defer queue. +*/ +static int +test_rcu_qsbr_dq_reclaim (void ) +{ +int ret; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_reclaim()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_reclaim (NULL ,10 ,NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq reclaim invalid params" ); + +/* Pass invalid parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.free_fn = test_rcu_qsbr_free_resource1; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =3 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); + ret =rte_rcu_qsbr_dq_reclaim (dq,0 ,NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq reclaim invalid params" ); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_delete: Delete a defer queue. +*/ +static int +test_rcu_qsbr_dq_delete (void ) +{ +int ret; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_delete()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_delete (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq delete invalid params" ); + +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.free_fn = test_rcu_qsbr_free_resource1; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete valid params" ); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_functional (int32_t size,int32_t esize,uint32_t flags) +{ +int i, j, ret; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; +uint64_t *e; +uint64_t sc =200 ; +int max_entries; + +printf ("\nTest rte_rcu_qsbr_dq_xxx functional tests()\n" ); +printf ("Size = %d, esize = %d, flags = 0x%x\n" , size, esize, flags); + + e = (uint64_t *)rte_zmalloc (NULL , esize,RTE_CACHE_LINE_SIZE ); +if (e ==NULL ) +return 0 ; + cb_failed =0 ; + +/* Initialize the RCU variable. No threads are registered */ +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.flags = flags; + params.free_fn = test_rcu_qsbr_free_resource2; + params.v = t[0 ]; + params.size = size; + params.esize = esize; + params.trigger_reclaim_limit = size >>3 ; + params.max_reclaim_size = (size >>4 )?(size >>4 ):1 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); + +/* Given the size calculate the maximum number of entries +* that can be stored on the defer queue (look at the logic used +* in capacity calculation of rte_ring). +*/ + max_entries =rte_align32pow2 (size +1 ) -1 ; +printf ("max_entries = %d\n" , max_entries); + +/* Enqueue few counters starting with the value 'sc' */ +/* The queue size will be rounded up to 2. The enqueue API also +* reclaims if the queue size is above certain limit. Since, there +* are no threads registered, reclamation succedes. Hence, it should +* be possible to enqueue more than the provided queue size. +*/ +for (i =0 ; i <10 ; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret !=0 ), +"dq enqueue functional, i = %d" , i); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (cb_failed ==1 ),"CB failed" ); + +/* Register a thread on the RCU QSBR variable. Reclamation will not +* succeed. It should not be possible to enqueue more than the size +* number of resources. +*/ +rte_rcu_qsbr_thread_register (t[0 ],1 ); +rte_rcu_qsbr_thread_online (t[0 ],1 ); + +for (i =0 ; i < max_entries; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret !=0 ), +"dq enqueue functional, max_entries = %d, i = %d" , + max_entries, i); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Enqueue fails as queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret ==0 ),"defer queue is not full" ); + +/* Delete should fail as there are elements in defer queue which +* cannot be reclaimed. +*/ + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq delete valid params" ); + +/* Report quiescent state, enqueue should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); +for (i =0 ; i < max_entries; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (cb_failed ==1 ),"CB failed" ); + +/* Queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret ==0 ),"defer queue is not full" ); + +/* Report quiescent state, delete should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete valid params" ); + +rte_free (e); + +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_RETURN_IF_ERROR ((cb_failed ==1 ),"CB failed" ); + +return 0 ; + +end: +rte_free (e); + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete valid params" ); +return -1 ; +} +/* * rte_rcu_qsbr_dump: Dump status of a single QS variable to a file */
@@ -1028,6 +1365,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_thread_offline () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_create () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_reclaim () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_delete () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_enqueue () <0 ) + goto test_fail; +printf ("\nFunctional tests\n" );if (test_rcu_qsbr_sw_sv_3qs () <0 )
@@ -1036,6 +1385,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_mw_mv_mqs () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_functional (1 ,8 ,0 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (2 ,8 ,RTE_RCU_QSBR_DQ_MT_UNSAFE ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (303 ,16 ,0 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (7 ,128 ,RTE_RCU_QSBR_DQ_MT_UNSAFE ) <0 ) + goto test_fail; +free_rcu ();printf ("\n" );
Signature · 1 line
2.17.1
PATCH v4 3/4 doc/rcu: add RCU integration design details
In reply to Honnappa NagarahalliFrom: Ruifeng Wang <[email protected]>
Add a section to describe a design to integrate QSBR RCU library with other libraries in DPDK.
--- doc/guides/prog_guide/rcu_lib.rst | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)
doc/guides/prog_guide/rcu_lib.rst +59 -0
@@ -190,3 +190,62 @@ However, when ``CONFIG_RTE_LIBRTE_RCU_DEBUG`` is enabled, these APIs aid in debugging issues. One can mark the access to shared data structures on the reader side using these APIs. The ``rte_rcu_qsbr_quiescent()`` will check if all the locks are unlocked. + +Resource reclamation framework for DPDK +--------------------------------------- + +Lock-free algorithms place additional burden of resource reclamation on +the application. When a writer deletes an entry from a data structure, the writer: + +#. Has to start the grace period +#. Has to store a reference to the deleted resources in a FIFO +#. Should check if the readers have completed a grace period and free the resources. This can also be done when the writer runs out of free resources. + +There are several APIs provided to help with this process. The writer +can create a FIFO to store the references to deleted resources using ``rte_rcu_qsbr_dq_create()``. +The resources can be enqueued to this FIFO using ``rte_rcu_qsbr_dq_enqueue()``. +If the FIFO is full, ``rte_rcu_qsbr_dq_enqueue`` will reclaim the resources before enqueuing. It will also reclaim resources on regular basis to keep the FIFO from growing too large. If the writer runs out of resources, the writer can call ``rte_rcu_qsbr_dq_reclaim`` API to reclaim resources. ``rte_rcu_qsbr_dq_delete`` is provided to reclaim any remaining resources and free the FIFO while shutting down. + +However, if this resource reclamation process were to be integrated in lock-free data structure libraries, it +hides this complexity from the application and makes it easier for the application to adopt lock-free algorithms. The following paragraphs discuss how the reclamation process can be integrated in DPDK libraries. + +In any DPDK application, the resource reclamation process using QSBR can be split into 4 parts: + +#. Initialization +#. Quiescent State Reporting +#. Reclaiming Resources +#. Shutdown + +The design proposed here assigns different parts of this process to client libraries and applications. The term 'client library' refers to lock-free data structure libraries such at rte_hash, rte_lpm etc. in DPDK or similar libraries outside of DPDK. The term 'application' refers to the packet processing application that makes use of DPDK such as L3 Forwarding example application, OVS, VPP etc.. + +The application has to handle 'Initialization' and 'Quiescent State Reporting'. So, + +* the application has to create the RCU variable and register the reader threads to report their quiescent state. +* the application has to register the same RCU variable with the client library. +* reader threads in the application have to report the quiescent state. This allows for the application to control the length of the critical section/how frequently the application wants to report the quiescent state. + +The client library will handle 'Reclaiming Resources' part of the process. The +client libraries will make use of the writer thread context to execute the memory +reclamation algorithm. So, + +* client library should provide an API to register a RCU variable that it will use. It should call ``rte_rcu_qsbr_dq_create()`` to create the FIFO to store the references to deleted entries. +* client library should use ``rte_rcu_qsbr_dq_enqueue`` to enqueue the deleted resources on the FIFO and start the grace period. +* if the library runs out of resources while adding entries, it should call ``rte_rcu_qsbr_dq_reclaim`` to reclaim the resources and try the resource allocation again. + +The 'Shutdown' process needs to be shared between the application and the +client library. + +* the application should make sure that the reader threads are not using the shared data structure, unregister the reader threads from the QSBR variable before calling the client library's shutdown function. + +* client library should call ``rte_rcu_qsbr_dq_delete`` to reclaim any remaining resources and free the FIFO. + +Integrating the resource reclamation with client libraries removes the burden from +the application and makes it easy to use lock-free algorithms. + +This design has several advantages over currently known methods. + +#. Application does not need a dedicated thread to reclaim resources. Memory + reclamation happens as part of the writer thread with little impact on + performance. +#. The client library has better control over the resources. For ex: the client + library can attempt to reclaim when it has run out of resources.
Signature · 1 line
2.17.1
PATCH v4 4/4 lib/rcu: add additional debug logs
In reply to Honnappa NagarahalliAdded additional debug logs. These helped in debugging RCU defer APIs.
--- lib/librte_rcu/rte_rcu_qsbr.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
lib/librte_rcu/rte_rcu_qsbr.h +8 -1
@@ -716,8 +716,15 @@ rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)RTE_ASSERT (v !=NULL );/* Check if all the readers have already acknowledged this token */ -if (likely (t <= v->acked_token)) +if (likely (t <= v->acked_token)) { +__RTE_RCU_DP_LOG (DEBUG , +"%s: check: token = %" PRIu64", wait = %d" , + __func__, t, wait); +__RTE_RCU_DP_LOG (DEBUG , +"%s: status: least acked token = %" PRIu64"" , + __func__, v->acked_token);return 1 ; + }if (likely (v->num_threads == v->max_threads))return __rte_rcu_qsbr_check_all (v, t, wait);
Signature · 1 line
2.17.1
v5 cover Add RCU reclamation APIs
In reply to Honnappa Nagarahalliv5
1) rte_rcu_dq_delete does not return an error on NULL parameter
2) Removed good amount of memcopy's where possible
3) Changed the call back function to be friendly for batching.
v4
1) RCU reclamation APIs changed to provide lot more flexibility
a) The rte_rcu_qsbr_dq_enqueue and rte_rcu_qsbr_dq_reclaim APIs can be configured to be MT safe
b) The auto reclamation limit and how much to reclaim
can be configured
c) rte_rcu_qsbr_dq_reclaim API returns the number of resources reclaimed and the number of pending resources on the defer queue
d) rte_rcu_qsbr_dq_reclaim API takes maximum number of resources to reclaim as a parameter
2) Multiple minor fixes
a) Private header file and test function names changed to remove 'rte_'
b) Compilation for shared library
c) Split the test cases into a separate commit
d) Uses rte_ring_xxx_elem APIs to support flexible ring element size
v3
1) Separated from the original series
(https://patches.dpdk.org/cover/58811/)
2) Added reclamation APIs and test cases (Stephen, Yipeng)
This is not a new patch. This patch set is separated from the LPM changes as the size of the changes in RCU library has grown due to comments from community. These APIs will help reduce the changes in LPM and hash libraries that are getting integrated with RCU library.
This adds 4 new APIs to RCU library to create a defer queue, enqueue deleted resources, reclaim resources and delete the defer queue.
The rationale for the APIs is documented in 3/4.
Honnappa Nagarahalli (3):
lib/rcu: add resource reclamation APIs
test/rcu: test cases for RCU defer queue APIs
lib/rcu: add additional debug logs
Ruifeng Wang (1):
doc/rcu: add RCU integration design details
app/test/test_rcu_qsbr.c | 365 ++++++++++++++++++++++++++++- doc/guides/prog_guide/rcu_lib.rst | 59 +++++ lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 7 + lib/librte_rcu/rcu_qsbr_pvt.h | 66 ++++++ lib/librte_rcu/rte_rcu_qsbr.c | 227 +++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 203 +++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 9 files changed, 931 insertions(+), 8 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.h
Signature · 1 line
2.17.1
PATCH v5 1/4 lib/rcu: add resource reclamation APIs
Add resource reclamation APIs to make it simple for applications and libraries to integrate rte_rcu library.
--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 7 + lib/librte_rcu/rcu_qsbr_pvt.h | 66 +++++++++ lib/librte_rcu/rte_rcu_qsbr.c | 227 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 194 +++++++++++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 501 insertions(+), 5 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.h
lib/librte_rcu/Makefile +1 -1
@@ -7,7 +7,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_rcu.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.map
lib/librte_rcu/meson.build +7 -0
@@ -3,3 +3,10 @@ sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') + +# for clang 32-bit compiles we need libatomic for 64-bit atomic ops +if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false + ext_deps += cc.find_library('atomic') +endif + +deps += ['ring']
lib/librte_rcu/rcu_qsbr_pvt.h new file +66 -0
66 lines not on this page · raw message
lib/librte_rcu/rte_rcu_qsbr.c +226 -1
246 lines not on this page · raw message
lib/librte_rcu/rte_rcu_qsbr.h +193 -1
222 lines not on this page · raw message
lib/librte_rcu/rte_rcu_version.map +4 -0
@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };
lib/meson.build +4 -2
@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on this
@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev', - 'rcu', 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', #fib lib depends on rib
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 1/4] lib/rcu: add resource reclamation APIs
Quoted 36 lines
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 7 + lib/librte_rcu/rcu_qsbr_pvt.h | 66 +++++++++ lib/librte_rcu/rte_rcu_qsbr.c | 227 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 194 +++++++++++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 501 insertions(+), 5 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.hlib/librte_rcu/Makefile +1
-1@@ -7,7 +7,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_rcu.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.maplib/librte_rcu/meson.build +0
-0
lib/librte_rcu/meson.build line 11
Quoted 9 lines
lib/librte_rcu/meson.build +6
-0@@ -3,3 +3,10 @@ sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') + +# for clang 32-bit compiles we need libatomic for 64-bit atomic ops +if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false + ext_deps += cc.find_library('atomic') +endif +
As a nit - as Pavan patch is already integrated into mainline, this is not necessary any more, I think.
Also noticed that most of make builds failed due to dependency problem:
http://mails.dpdk.org/archives/test-report/2020-April/127765.html
I can't reproduce it locally, but my guess that we need to move rcu above ring in this mk file: mk/rte.app.mk
Probably something like that:
mk/rte.app.mk +1 -1
@@ -91,13 +91,13 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool _LDLIBS-$(CONFIG_RTE_LIBRTE_STACK) += -lrte_stack _LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += -lrte_mempool_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_OCTEONTX2_MEMPOOL) += -lrte_mempool_octeontx2 +_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu _LDLIBS-$(CONFIG_RTE_LIBRTE_RING) += -lrte_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_PCI) += -lrte_pci _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrte_eal _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE) += -lrte_cmdline _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched -_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu
Apart from that - LGTM
Quoted 598 lines
lib/librte_rcu/meson.build +1
-0+deps += ['ring']lib/librte_rcu/rcu_qsbr_pvt.h new file +66
-0@@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Arm Limited +*/ + +#ifndef _RTE_RCU_QSBR_PVT_H_ +#define _RTE_RCU_QSBR_PVT_H_ + +/** +* This file is private to the RCU library. It should not be included +* by the user of this library. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <rte_ring.h> +#include <rte_ring_elem.h> + +#include "rte_rcu_qsbr.h" + +/* Defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq { +struct rte_rcu_qsbr *v;/**< RCU QSBR variable used by this queue.*/ +struct rte_ring *r;/**< RCU QSBR defer queue. */ +uint32_t size; +/**< Number of elements in the defer queue */ +uint32_t esize; +/**< Size (in bytes) of data, including the token, stored on the +* defer queue. +*/ +uint32_t trigger_reclaim_limit; +/**< Trigger automatic reclamation after the defer queue +* has atleast these many resources waiting. +*/ +uint32_t max_reclaim_size; +/**< Reclaim at the max these many resources during auto +* reclamation. +*/ +rte_rcu_qsbr_free_resource_t free_fn; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. +*/ +}; + +/* Internal structure to represent the element on the defer queue. +* Use alias as a character array is type casted to a variable +* of this structure type. +*/ +typedef struct { +uint64_t token;/**< Token */ +uint8_t elem[0 ];/**< Pointer to user element */ +} __attribute__((__may_alias__))__rte_rcu_qsbr_dq_elem_t ; + +#ifdef __cplusplus +} +#endif + +#endif /* _RTE_RCU_QSBR_PVT_H_ */ lib/librte_rcu/rte_rcu_qsbr.c +226
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2018-2020 Arm Limited */ #include <stdio.h> @@ -18,8 +18,10 @@#include <rte_per_lcore.h> #include <rte_lcore.h> #include <rte_errno.h> +#include <rte_ring_elem.h> #include "rte_rcu_qsbr.h" +#include "rcu_qsbr_pvt.h" /* Get the memory size of QSBR variable */ size_t @@ -270,6 +272,229 @@ rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)return 0 ; } +/* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +*/ +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters *params) +{ +struct rte_rcu_qsbr_dq *dq; +uint32_t qs_fifo_size; +unsigned int flags; + +if (params ==NULL || params->free_fn ==NULL || + params->v ==NULL || params->name ==NULL || + params->size ==0 || params->esize ==0 || + (params->esize %4 !=0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return NULL ; + } +/* If auto reclamation is configured, reclaim limit +* should be a valid value. +*/ +if ((params->trigger_reclaim_limit <= params->size) && + (params->max_reclaim_size ==0 )) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter, size = %u, trigger_reclaim_limit = %u, max_reclaim_size = %u\n" , + __func__, params->size, params->trigger_reclaim_limit, + params->max_reclaim_size); + rte_errno =EINVAL ; + +return NULL ; + } + + dq =rte_zmalloc (NULL ,sizeof (struct rte_rcu_qsbr_dq ), +RTE_CACHE_LINE_SIZE ); +if (dq ==NULL ) { + rte_errno =ENOMEM ; + +return NULL ; + } + +/* Decide the flags for the ring. +* If MT safety is requested, use RTS for ring enqueue as most +* use cases involve dq-enqueue happening on the control plane. +* Ring dequeue is always HTS due to the possibility of revert. +*/ + flags =RING_F_MP_RTS_ENQ ; +if (params->flags &RTE_RCU_QSBR_DQ_MT_UNSAFE ) + flags =RING_F_SP_ENQ ; + flags |=RING_F_MC_HTS_DEQ ; +/* round up qs_fifo_size to next power of two that is not less than +* max_size. +*/ + qs_fifo_size =rte_align32pow2 (params->size +1 ); +/* Add token size to ring element size */ + dq->r =rte_ring_create_elem (params->name, + __RTE_QSBR_TOKEN_SIZE + params->esize, + qs_fifo_size,SOCKET_ID_ANY , flags); +if (dq->r ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): defer queue create failed\n" , __func__); +rte_free (dq); +return NULL ; + } + + dq->v = params->v; + dq->size = params->size; + dq->esize = __RTE_QSBR_TOKEN_SIZE + params->esize; + dq->trigger_reclaim_limit = params->trigger_reclaim_limit; + dq->max_reclaim_size = params->max_reclaim_size; + dq->free_fn = params->free_fn; + dq->p = params->p; + +return dq; +} + +/* Enqueue one resource to the defer queue to free after the grace +* period is over. +*/ +int rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e) +{ +__rte_rcu_qsbr_dq_elem_t *dq_elem; +uint32_t cur_size; + +if (dq ==NULL || e ==NULL ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + +char data[dq->esize]; + dq_elem = (__rte_rcu_qsbr_dq_elem_t *)data; +/* Start the grace period */ + dq_elem->token =rte_rcu_qsbr_start (dq->v); + +/* Reclaim resources if the queue size has hit the reclaim +* limit. This helps the queue from growing too large and +* allows time for reader threads to report their quiescent state. +*/ + cur_size =rte_ring_count (dq->r); +if (cur_size > dq->trigger_reclaim_limit) { +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Triggering reclamation\n" , __func__); +rte_rcu_qsbr_dq_reclaim (dq, dq->max_reclaim_size, +NULL ,NULL ,NULL ); + } + +/* Enqueue the token and resource. Generating the token and +* enqueuing (token + resource) on the queue is not an +* atomic operation. When the defer queue is shared by multiple +* writers, this might result in tokens enqueued out of order +* on the queue. So, some tokens might wait longer than they +* are required to be reclaimed. +*/ +memcpy (dq_elem->elem, e, dq->esize - __RTE_QSBR_TOKEN_SIZE); +/* Check the status as enqueue might fail since the other threads +* might have used up the freed space. +* Enqueue uses the configured flags when the DQ was created. +*/ +if (rte_ring_enqueue_elem (dq->r, data, dq->esize) !=0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Enqueue failed\n" , __func__); +/* Note that the token generated above is not used. +* Other than wasting tokens, it should not cause any +* other issues. +*/ +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Skipped enqueuing token = %" PRIu64"\n" , + __func__, dq_elem->token); + + rte_errno =ENOSPC ; +return 1 ; + } + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Enqueued token = %" PRIu64"\n" , __func__, dq_elem->token); + +return 0 ; +} + +/* Reclaim resources from the defer queue. */ +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending, +unsigned int *available) +{ +uint32_t cnt; +__rte_rcu_qsbr_dq_elem_t *dq_elem; + +if (dq ==NULL || n ==0 ) { +rte_log (RTE_LOG_ERR , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + rte_errno =EINVAL ; + +return 1 ; + } + + cnt =0 ; + +char data[dq->esize]; +/* Check reader threads quiescent state and reclaim resources */ +while (cnt < n && +rte_ring_dequeue_bulk_elem_start (dq->r, &data, + dq->esize,1 , available) !=0 ) { + dq_elem = (__rte_rcu_qsbr_dq_elem_t *)data; + +/* Reclaim the resource */ +if (rte_rcu_qsbr_check (dq->v, dq_elem->token, false) !=1 ) { +rte_ring_dequeue_elem_finish (dq->r,0 ); +break ; + } +rte_ring_dequeue_elem_finish (dq->r,1 ); + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed token = %" PRIu64"\n" , + __func__, dq_elem->token); + + dq->free_fn (dq->p, dq_elem->elem,1 ); + + cnt++; + } + +rte_log (RTE_LOG_INFO , rte_rcu_log_type, +"%s(): Reclaimed %u resources\n" , __func__, cnt); + +if (freed !=NULL ) + *freed = cnt; +if (pending !=NULL ) + *pending =rte_ring_count (dq->r); + +return 0 ; +} + +/* Delete a defer queue. */ +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq) +{ +unsigned int pending; + +if (dq ==NULL ) { +rte_log (RTE_LOG_DEBUG , rte_rcu_log_type, +"%s(): Invalid input parameter\n" , __func__); + +return 0 ; + } + +/* Reclaim all the resources */ +rte_rcu_qsbr_dq_reclaim (dq, ~0 ,NULL , &pending,NULL ); +if (pending !=0 ) { + rte_errno =EAGAIN ; + +return 1 ; + } + +rte_ring_free (dq->r); +rte_free (dq); + +return 0 ; +} +int rte_rcu_log_type;RTE_INIT (rte_rcu_register)lib/librte_rcu/rte_rcu_qsbr.h +193
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2018-2020 Arm Limited */ #ifndef _RTE_RCU_QSBR_H_@@ -34,6 +34,7 @@ extern "C" {#include <rte_lcore.h> #include <rte_debug.h> #include <rte_atomic.h> +#include <rte_ring.h> extern int rte_rcu_log_type;@@ -84,6 +85,7 @@ struct rte_rcu_qsbr_cnt {#define __RTE_QSBR_CNT_THR_OFFLINE 0#define __RTE_QSBR_CNT_INIT 1#define __RTE_QSBR_CNT_MAX ((uint64_t)~0) +#define __RTE_QSBR_TOKEN_SIZE sizeof(uint64_t)/* RTE Quiescent State variable structure. * This structure has two elements that vary in size based on the @@ -114,6 +116,86 @@ struct rte_rcu_qsbr {*/ } __rte_cache_aligned; +/** +* Call back function called to free the resources. +* +* @param p +* Pointer provided while creating the defer queue +* @param e +* Pointer to the resource data stored on the defer queue +* @param n +* Number of resources to free. Currently, this is set to 1. +* +* @return +* None +*/ +typedef void (*rte_rcu_qsbr_free_resource_t )(void *p,void *e,unsigned int n); + +#define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE + +/** +* Various flags supported. +*/ +/**< Enqueue and reclaim operations are multi-thread safe by default. +* The call back functions registered to free the resources are +* assumed to be multi-thread safe. +* Set this flag is multi-thread safety is not required. +*/ +#define RTE_RCU_QSBR_DQ_MT_UNSAFE 1 + +/** +* Parameters used when creating the defer queue. +*/ +struct rte_rcu_qsbr_dq_parameters { +const char *name; +/**< Name of the queue. */ +uint32_t flags; +/**< Flags to control API behaviors */ +uint32_t size; +/**< Number of entries in queue. Typically, this will be +* the same as the maximum number of entries supported in the +* lock free data structure. +* Data structures with unbounded number of entries is not +* supported currently. +*/ +uint32_t esize; +/**< Size (in bytes) of each element in the defer queue. +* This has to be multiple of 4B. +*/ +uint32_t trigger_reclaim_limit; +/**< Trigger automatic reclamation after the defer queue +* has atleast these many resources waiting. This auto +* reclamation is triggered in rte_rcu_qsbr_dq_enqueue API +* call. +* If this is greater than 'size', auto reclamation is +* not triggered. +* If this is set to 0, auto reclamation is triggered +* in every call to rte_rcu_qsbr_dq_enqueue API. +*/ +uint32_t max_reclaim_size; +/**< When automatic reclamation is enabled, reclaim at the max +* these many resources. This should contain a valid value, if +* auto reclamation is on. Setting this to 'size' or greater will +* reclaim all possible resources currently on the defer queue. +*/ +rte_rcu_qsbr_free_resource_t free_fn; +/**< Function to call to free the resource. */ +void *p; +/**< Pointer passed to the free function. Typically, this is the +* pointer to the data structure to which the resource to free +* belongs. This can be NULL. +*/ +struct rte_rcu_qsbr *v; +/**< RCU QSBR variable to use for this defer queue */ +}; + +/* RTE defer queue structure. +* This structure holds the defer queue. The defer queue is used to +* hold the deleted entries from the data structure that are not +* yet freed. +*/ +struct rte_rcu_qsbr_dq ; +/** * @warning * @b EXPERIMENTAL: this API may change without prior notice @@ -692,6 +774,116 @@ __rte_experimentalint rte_rcu_qsbr_dump (FILE *f,struct rte_rcu_qsbr *v); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Create a queue used to store the data structure elements that can +* be freed later. This queue is referred to as 'defer queue'. +* +* @param params +* Parameters to create a defer queue. +* @return +* On success - Valid pointer to defer queue +* On error - NULL +* Possible rte_errno codes are: +* - EINVAL - NULL parameters are passed +* - ENOMEM - Not enough memory +*/ +__rte_experimental +struct rte_rcu_qsbr_dq * +rte_rcu_qsbr_dq_create (const struct rte_rcu_qsbr_dq_parameters *params); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Enqueue one resource to the defer queue and start the grace period. +* The resource will be freed later after at least one grace period +* is over. +* +* If the defer queue is full, it will attempt to reclaim resources. +* It will also reclaim resources at regular intervals to avoid +* the defer queue from growing too big. +* +* Multi-thread safety is provided as the defer queue configuration. +* When multi-thread safety is requested, it is possible that the +* resources are not stored in their order of deletion. This results +* in resources being held in the defer queue longer than they should. +* +* @param dq +* Defer queue to allocate an entry from. +* @param e +* Pointer to resource data to copy to the defer queue. The size of +* the data to copy is equal to the element size provided when the +* defer queue was created. +* @return +* On success - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +* - ENOSPC - Defer queue is full. This condition can not happen +* if the defer queue size is equal (or larger) than the +* number of elements in the data structure. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_enqueue (struct rte_rcu_qsbr_dq *dq,void *e); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Free quesed resources from the defer queue. +* +* This API is multi-thread safe. +* +* @param dq +* Defer queue to free an entry from. +* @param n +* Maximum number of resources to free. +* @param freed +* Number of resources that were freed. +* @param pending +* Number of resources pending on the defer queue. This number might not +* be acurate if multi-thread safety is configured. +* @param available +* Number of resources that can be added to the defer queue. +* This number might not be acurate if multi-thread safety is configured. +* @return +* On successful reclamation of at least 1 resource - 0 +* On error - 1 with rte_errno set to +* - EINVAL - NULL parameters are passed +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_reclaim (struct rte_rcu_qsbr_dq *dq,unsigned int n, +unsigned int *freed,unsigned int *pending,unsigned int *available); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Delete a defer queue. +* +* It tries to reclaim all the resources on the defer queue. +* If any of the resources have not completed the grace period +* the reclamation stops and returns immediately. The rest of +* the resources are not reclaimed and the defer queue is not +* freed. +* +* @param dq +* Defer queue to delete. +* @return +* On success - 0 +* On error - 1 +* Possible rte_errno codes are: +* - EAGAIN - Some of the resources have not completed at least 1 grace +* period, try again. +*/ +__rte_experimental +int +rte_rcu_qsbr_dq_delete (struct rte_rcu_qsbr_dq *dq); + #ifdef __cplusplus }#endif lib/librte_rcu/rte_rcu_version.map +4
-0@@ -8,6 +8,10 @@ EXPERIMENTAL { rte_rcu_qsbr_synchronize; rte_rcu_qsbr_thread_register; rte_rcu_qsbr_thread_unregister; + rte_rcu_qsbr_dq_create; + rte_rcu_qsbr_dq_enqueue; + rte_rcu_qsbr_dq_reclaim; + rte_rcu_qsbr_dq_delete; local: *; };lib/meson.build +4
-3@@ -11,7 +11,9 @@ libraries = [ 'kvargs', # eal depends on kvargs 'eal', # everything depends on eal - 'ring', 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core + 'ring', + 'rcu', # rcu depends on ring + 'mempool', 'mbuf', 'net', 'meter', 'ethdev', 'pci', # core 'cmdline', 'metrics', # bitrate/latency stats depends on this 'hash', # efd depends on this@@ -22,7 +24,7 @@ libraries = [ 'gro', 'gso', 'ip_frag', 'jobstats', 'kni', 'latencystats', 'lpm', 'member', 'power', 'pdump', 'rawdev', - 'rcu', 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', + 'rib', 'reorder', 'sched', 'security', 'stack', 'vhost', # ipsec lib depends on net, crypto and security 'ipsec', #fib lib depends on rib --2.17.1
Re: [dpdk-dev] [PATCH v5 1/4] lib/rcu: add resource reclamation APIs
Quoted 38 lines · Konstantin
Quoted 38 lines
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 7 + lib/librte_rcu/rcu_qsbr_pvt.h | 66 +++++++++ lib/librte_rcu/rte_rcu_qsbr.c | 227 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 194 +++++++++++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 501 insertions(+), 5 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.hlib/librte_rcu/Makefile +1
-1@@ -7,7 +7,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_rcu.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.maplib/librte_rcu/meson.build +0
-0@@ -3,3 +3,10 @@
Quoted 11 lines
lib/librte_rcu/meson.build line 11
Quoted 8 lines
lib/librte_rcu/meson.build +6
-0sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') + +# for clang 32-bit compiles we need libatomic for 64-bit atomic ops +if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false + ext_deps += cc.find_library('atomic') +endif +As a nit - as Pavan patch is already integrated into mainline, this is not necessary any more, I think.
I can handle this.
Quoted 11 lines
Also noticed that most of make builds failed due to dependency problem:
http://mails.dpdk.org/archives/test-report/2020-April/127765.html
I can't reproduce it locally, but my guess that we need to move rcu above ring in this mk file:mk/rte.app.mk
Probably something like that:mk/rte.app.mk +0
-0@@ -91,13 +91,13 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool _LDLIBS-$(CONFIG_RTE_LIBRTE_STACK) += -lrte_stack
mk/rte.app.mk _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool · line 100
Quoted 10 lines
mk/rte.app.mk +1
-1_LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += -lrte_mempool_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_OCTEONTX2_MEMPOOL) += -lrte_mempool_octeontx2 +_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu _LDLIBS-$(CONFIG_RTE_LIBRTE_RING) += -lrte_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_PCI) += -lrte_pci _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrte_eal _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE) += -lrte_cmdline _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched -_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu
No, just moving will not express a dependency.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v5 1/4] lib/rcu: add resource reclamation APIs
Quoted 71 lines · David Marchand
Quoted 38 lines · Konstantin
Quoted 38 lines
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 7 + lib/librte_rcu/rcu_qsbr_pvt.h | 66 +++++++++ lib/librte_rcu/rte_rcu_qsbr.c | 227 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 194 +++++++++++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 501 insertions(+), 5 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.hlib/librte_rcu/Makefile +1
-1@@ -7,7 +7,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_rcu.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.maplib/librte_rcu/meson.build +0
-0@@ -3,3 +3,10 @@Quoted 11 lines
lib/librte_rcu/meson.build line 11
Quoted 8 lines
lib/librte_rcu/meson.build +6
-0sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') + +# for clang 32-bit compiles we need libatomic for 64-bit atomic ops +if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false + ext_deps += cc.find_library('atomic') +endif +As a nit - as Pavan patch is already integrated into mainline, this is not necessary any more, I think.
I can handle this.
Quoted 15 lines
Also noticed that most of make builds failed due to dependency problem:
http://mails.dpdk.org/archives/test-report/2020-April/127765.html
I can't reproduce it locally, but my guess that we need to move rcu above ring in this mk file:mk/rte.app.mk
Probably something like that:mk/rte.app.mk +1
-0@@ -91,13 +91,13 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool _LDLIBS-$(CONFIG_RTE_LIBRTE_STACK) += -lrte_stack _LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += -lrte_mempool_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_OCTEONTX2_MEMPOOL) += -lrte_mempool_octeontx2 +_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu _LDLIBS-$(CONFIG_RTE_LIBRTE_RING) += -lrte_ring
Quoted 8 lines
mk/rte.app.mk _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool · line 100
Quoted 6 lines
mk/rte.app.mk +0
-1_LDLIBS-$(CONFIG_RTE_LIBRTE_PCI) += -lrte_pci _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrte_eal _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE) += -lrte_cmdline _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched -_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcuNo, just moving will not express a dependency.
Fixed with:
lib/Makefile +1 -1
@@ -118,6 +118,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += librte_telemetry DEPDIRS-librte_telemetry := librte_eal librte_metrics librte_ethdev DIRS-$(CONFIG_RTE_LIBRTE_RCU) += librte_rcu -DEPDIRS-librte_rcu := librte_eal +DEPDIRS-librte_rcu := librte_eal librte_ring ifeq ($(CONFIG_RTE_EXEC_ENV_LINUX),y) DIRS-$(CONFIG_RTE_LIBRTE_KNI) += librte_kni
lib/librte_rcu/meson.build +0 -5
@@ -4,9 +4,4 @@ sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') -# for clang 32-bit compiles we need libatomic for 64-bit atomic ops -if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false - ext_deps += cc.find_library('atomic') -endif - deps += ['ring']
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v5 1/4] lib/rcu: add resource reclamation APIs
Quoted 90 lines
Quoted 71 lines · David Marchand
Quoted 38 lines · Konstantin
Quoted 38 lines
Add resource reclamation APIs to make it simple for applications and libraries to integrate
rte_rculibrary.Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/Makefile | 2 +- lib/librte_rcu/meson.build | 7 + lib/librte_rcu/rcu_qsbr_pvt.h | 66 +++++++++ lib/librte_rcu/rte_rcu_qsbr.c | 227 ++++++++++++++++++++++++++++- lib/librte_rcu/rte_rcu_qsbr.h | 194 +++++++++++++++++++++++- lib/librte_rcu/rte_rcu_version.map | 4 + lib/meson.build | 6 +- 7 files changed, 501 insertions(+), 5 deletions(-) create mode 100644 lib/librte_rcu/rcu_qsbr_pvt.hlib/librte_rcu/Makefile +1
-1@@ -7,7 +7,7 @@ include $(RTE_SDK)/mk/rte.vars.mk LIB = librte_rcu.a CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -O3 -LDLIBS += -lrte_eal +LDLIBS += -lrte_eal -lrte_ring EXPORT_MAP := rte_rcu_version.maplib/librte_rcu/meson.build +0
-0@@ -3,3 +3,10 @@Quoted 11 lines
lib/librte_rcu/meson.build line 11
Quoted 8 lines
lib/librte_rcu/meson.build +6
-0sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') + +# for clang 32-bit compiles we need libatomic for 64-bit atomic ops +if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false + ext_deps += cc.find_library('atomic') +endif +As a nit - as Pavan patch is already integrated into mainline, this is not necessary any more, I think.
I can handle this.
Quoted 15 lines
Also noticed that most of make builds failed due to dependency problem:
http://mails.dpdk.org/archives/test-report/2020-April/127765.html
I can't reproduce it locally, but my guess that we need to move rcu above ring in this mk file:mk/rte.app.mk
Probably something like that:mk/rte.app.mk +1
-0@@ -91,13 +91,13 @@ _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool _LDLIBS-$(CONFIG_RTE_LIBRTE_STACK) += -lrte_stack _LDLIBS-$(CONFIG_RTE_DRIVER_MEMPOOL_RING) += -lrte_mempool_ring _LDLIBS-$(CONFIG_RTE_LIBRTE_OCTEONTX2_MEMPOOL) += -lrte_mempool_octeontx2 +_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcu _LDLIBS-$(CONFIG_RTE_LIBRTE_RING) += -lrte_ringQuoted 8 lines
mk/rte.app.mk _LDLIBS-$(CONFIG_RTE_LIBRTE_MEMPOOL) += -lrte_mempool · line 100
Quoted 6 lines
mk/rte.app.mk +0
-1_LDLIBS-$(CONFIG_RTE_LIBRTE_PCI) += -lrte_pci _LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrte_eal _LDLIBS-$(CONFIG_RTE_LIBRTE_CMDLINE) += -lrte_cmdline _LDLIBS-$(CONFIG_RTE_LIBRTE_REORDER) += -lrte_reorder _LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrte_sched -_LDLIBS-$(CONFIG_RTE_LIBRTE_RCU) += -lrte_rcuNo, just moving will not express a dependency.
Fixed with:
lib/Makefile +0
-0
lib/Makefile DIRS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += librte_telemetry · line 120
Quoted 4 lines
lib/Makefile +1
-1@@ -118,6 +118,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_TELEMETRY) += librte_telemetry DEPDIRS-librte_telemetry := librte_eal librte_metrics librte_ethdev DIRS-$(CONFIG_RTE_LIBRTE_RCU) += librte_rcu -DEPDIRS-librte_rcu := librte_eal +DEPDIRS-librte_rcu := librte_eal librte_ring
Right, totally forgot about that part.
Quoted 21 lines
lib/Makefile +0
-0ifeq ($(CONFIG_RTE_EXEC_ENV_LINUX),y) DIRS-$(CONFIG_RTE_LIBRTE_KNI) += librte_knilib/librte_rcu/meson.build +0
-6@@ -4,9 +4,4 @@ sources = files('rte_rcu_qsbr.c') headers = files('rte_rcu_qsbr.h') -# for clang 32-bit compiles we need libatomic for 64-bit atomic ops -if cc.get_id() == 'clang' and dpdk_conf.get('RTE_ARCH_64') == false - ext_deps += cc.find_library('atomic') -endif - deps += ['ring'] --David Marchand
PATCH v5 2/4 test/rcu: test cases for RCU defer queue APIs
In reply to Honnappa NagarahalliAdd test cases for RCU defer queue APIs.
--- app/test/test_rcu_qsbr.c | 365 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 363 insertions(+), 2 deletions(-)
app/test/test_rcu_qsbr.c +363 -2
@@ -1,8 +1,9 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright (c) 2018 Arm Limited +* Copyright (c) 2019-2020 Arm Limited */ #include <stdio.h> +#include <string.h> #include <rte_pause.h> #include <rte_rcu_qsbr.h> #include <rte_hash.h>
@@ -15,7 +16,8 @@#include "test.h" /* Check condition and return an error if true. */ -#define TEST_RCU_QSBR_RETURN_IF_ERROR (cond, str, ...) do { \ +#define TEST_RCU_QSBR_RETURN_IF_ERROR (cond, str, ...) \ +do { \ if (cond) { \ printf("ERROR file %s, line %d: " str "\n", __FILE__, \ __LINE__, ##__VA_ARGS__); \
@@ -23,6 +25,16 @@ } \ }while (0 ) +/* Check condition and go to label if true. */ +#define TEST_RCU_QSBR_GOTO_IF_ERROR (label, cond, str, ...) \ +do { \ + if (cond) { \ + printf("ERROR file %s, line %d: " str "\n", __FILE__, \ + __LINE__, ##__VA_ARGS__); \ + goto label; \ + } \ +} while (0) +/* Make sure that this has the same value as __RTE_QSBR_CNT_INIT */ #define TEST_RCU_QSBR_CNT_INIT 1
@@ -34,6 +46,7 @@ static uint32_t *keys;#define COUNTER_VALUE 4096static uint32_t *hash_data[RTE_MAX_LCORE ][TOTAL_ENTRY ];static uint8_t writer_done; +static uint8_t cb_failed;static struct rte_rcu_qsbr *t[RTE_MAX_LCORE ];static struct rte_hash *h[RTE_MAX_LCORE ];
@@ -585,6 +598,330 @@ test_rcu_qsbr_thread_offline(void)return 0 ; } +static void +test_rcu_qsbr_free_resource1 (void *p,void *e,unsigned int n) +{ +if (p !=NULL || e !=NULL || n !=1 ) { +printf ("%s: Test failed\n" , __func__); + cb_failed =1 ; + } +} + +static void +test_rcu_qsbr_free_resource2 (void *p,void *e,unsigned int n) +{ +if (p !=NULL || e ==NULL || n !=1 ) { +printf ("%s: Test failed\n" , __func__); + cb_failed =1 ; + } +} + +/* +* rte_rcu_qsbr_dq_create: create a queue used to store the data structure +* elements that can be freed later. This queue is referred to as 'defer queue'. +*/ +static int +test_rcu_qsbr_dq_create (void ) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_create()\n" ); + +/* Pass invalid parameters */ + dq =rte_rcu_qsbr_dq_create (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.free_fn = test_rcu_qsbr_free_resource1; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.size =1 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.esize =3 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size =0 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq !=NULL ),"dq create invalid params" ); + +/* Pass all valid parameters */ + params.esize =16 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); +rte_rcu_qsbr_dq_delete (dq); + + params.esize =16 ; + params.flags =RTE_RCU_QSBR_DQ_MT_UNSAFE ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); +rte_rcu_qsbr_dq_delete (dq); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_enqueue (void ) +{ +int ret; +uint64_t r; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_enqueue()\n" ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.free_fn = test_rcu_qsbr_free_resource1; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_enqueue (NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid params" ); + + ret =rte_rcu_qsbr_dq_enqueue (dq,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid params" ); + + ret =rte_rcu_qsbr_dq_enqueue (NULL , &r); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq enqueue invalid params" ); + + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==1 ),"dq delete valid params" ); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_reclaim: Reclaim resources from the defer queue. +*/ +static int +test_rcu_qsbr_dq_reclaim (void ) +{ +int ret; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_reclaim()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_reclaim (NULL ,10 ,NULL ,NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq reclaim invalid params" ); + +/* Pass invalid parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.free_fn = test_rcu_qsbr_free_resource1; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =3 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); + ret =rte_rcu_qsbr_dq_reclaim (dq,0 ,NULL ,NULL ,NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=1 ),"dq reclaim invalid params" ); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_delete: Delete a defer queue. +*/ +static int +test_rcu_qsbr_dq_delete (void ) +{ +int ret; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; + +printf ("\nTest rte_rcu_qsbr_dq_delete()\n" ); + +/* Pass invalid parameters */ + ret =rte_rcu_qsbr_dq_delete (NULL ); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete invalid params" ); + +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.free_fn = test_rcu_qsbr_free_resource1; +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + params.v = t[0 ]; + params.size =1 ; + params.esize =16 ; + params.trigger_reclaim_limit =0 ; + params.max_reclaim_size = params.size; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete valid params" ); + +return 0 ; +} + +/* +* rte_rcu_qsbr_dq_enqueue: enqueue one resource to the defer queue, +* to be freed later after atleast one grace period is over. +*/ +static int +test_rcu_qsbr_dq_functional (int32_t size,int32_t esize,uint32_t flags) +{ +int i, j, ret; +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; +struct rte_rcu_qsbr_dq *dq; +uint64_t *e; +uint64_t sc =200 ; +int max_entries; + +printf ("\nTest rte_rcu_qsbr_dq_xxx functional tests()\n" ); +printf ("Size = %d, esize = %d, flags = 0x%x\n" , size, esize, flags); + + e = (uint64_t *)rte_zmalloc (NULL , esize,RTE_CACHE_LINE_SIZE ); +if (e ==NULL ) +return 0 ; + cb_failed =0 ; + +/* Initialize the RCU variable. No threads are registered */ +rte_rcu_qsbr_init (t[0 ],RTE_MAX_LCORE ); + +/* Create a queue with simple parameters */ +memset (¶ms,0 ,sizeof (struct rte_rcu_qsbr_dq_parameters )); +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"TEST_RCU" ); + params.name = rcu_dq_name; + params.flags = flags; + params.free_fn = test_rcu_qsbr_free_resource2; + params.v = t[0 ]; + params.size = size; + params.esize = esize; + params.trigger_reclaim_limit = size >>3 ; + params.max_reclaim_size = (size >>4 )?(size >>4 ):1 ; + dq =rte_rcu_qsbr_dq_create (¶ms); +TEST_RCU_QSBR_RETURN_IF_ERROR ((dq ==NULL ),"dq create valid params" ); + +/* Given the size calculate the maximum number of entries +* that can be stored on the defer queue (look at the logic used +* in capacity calculation of rte_ring). +*/ + max_entries =rte_align32pow2 (size +1 ) -1 ; +printf ("max_entries = %d\n" , max_entries); + +/* Enqueue few counters starting with the value 'sc' */ +/* The queue size will be rounded up to 2. The enqueue API also +* reclaims if the queue size is above certain limit. Since, there +* are no threads registered, reclamation succedes. Hence, it should +* be possible to enqueue more than the provided queue size. +*/ +for (i =0 ; i <10 ; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret !=0 ), +"dq enqueue functional, i = %d" , i); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (cb_failed ==1 ),"CB failed" ); + +/* Register a thread on the RCU QSBR variable. Reclamation will not +* succeed. It should not be possible to enqueue more than the size +* number of resources. +*/ +rte_rcu_qsbr_thread_register (t[0 ],1 ); +rte_rcu_qsbr_thread_online (t[0 ],1 ); + +for (i =0 ; i < max_entries; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret !=0 ), +"dq enqueue functional, max_entries = %d, i = %d" , + max_entries, i); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Enqueue fails as queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret ==0 ),"defer queue is not full" ); + +/* Delete should fail as there are elements in defer queue which +* cannot be reclaimed. +*/ + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret ==0 ),"dq delete valid params" ); + +/* Report quiescent state, enqueue should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); +for (i =0 ; i < max_entries; i++) { + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret !=0 ), +"dq enqueue functional" ); +for (j =0 ; j < esize/8 ; j++) + e[j] = sc++; + } + +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (cb_failed ==1 ),"CB failed" ); + +/* Queue is full */ + ret =rte_rcu_qsbr_dq_enqueue (dq, e); +TEST_RCU_QSBR_GOTO_IF_ERROR (end, (ret ==0 ),"defer queue is not full" ); + +/* Report quiescent state, delete should succeed */ +rte_rcu_qsbr_quiescent (t[0 ],1 ); + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete valid params" ); + +rte_free (e); + +/* Validate that call back function did not return any error */ +TEST_RCU_QSBR_RETURN_IF_ERROR ((cb_failed ==1 ),"CB failed" ); + +return 0 ; + +end: +rte_free (e); + ret =rte_rcu_qsbr_dq_delete (dq); +TEST_RCU_QSBR_RETURN_IF_ERROR ((ret !=0 ),"dq delete valid params" ); +return -1 ; +} +/* * rte_rcu_qsbr_dump: Dump status of a single QS variable to a file */
@@ -1028,6 +1365,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_thread_offline () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_create () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_reclaim () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_delete () <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_enqueue () <0 ) + goto test_fail; +printf ("\nFunctional tests\n" );if (test_rcu_qsbr_sw_sv_3qs () <0 )
@@ -1036,6 +1385,18 @@ test_rcu_qsbr_main(void)if (test_rcu_qsbr_mw_mv_mqs () <0 ) goto test_fail; +if (test_rcu_qsbr_dq_functional (1 ,8 ,0 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (2 ,8 ,RTE_RCU_QSBR_DQ_MT_UNSAFE ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (303 ,16 ,0 ) <0 ) + goto test_fail; + +if (test_rcu_qsbr_dq_functional (7 ,128 ,RTE_RCU_QSBR_DQ_MT_UNSAFE ) <0 ) + goto test_fail; +free_rcu ();printf ("\n" );
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 2/4] test/rcu: test cases for RCU defer queue APIs
Quoted 11 lines
Add test cases for RCU defer queue APIs.
Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Ola Liljedhal <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- app/test/test_rcu_qsbr.c | 365 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 363 insertions(+), 2 deletions(-)--
Quoted 1 line
2.17.1
PATCH v5 3/4 doc/rcu: add RCU integration design details
In reply to Honnappa NagarahalliFrom: Ruifeng Wang <[email protected]>
Add a section to describe a design to integrate QSBR RCU library with other libraries in DPDK.
--- doc/guides/prog_guide/rcu_lib.rst | 59 +++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+)
doc/guides/prog_guide/rcu_lib.rst +59 -0
@@ -190,3 +190,62 @@ However, when ``CONFIG_RTE_LIBRTE_RCU_DEBUG`` is enabled, these APIs aid in debugging issues. One can mark the access to shared data structures on the reader side using these APIs. The ``rte_rcu_qsbr_quiescent()`` will check if all the locks are unlocked. + +Resource reclamation framework for DPDK +--------------------------------------- + +Lock-free algorithms place additional burden of resource reclamation on +the application. When a writer deletes an entry from a data structure, the writer: + +#. Has to start the grace period +#. Has to store a reference to the deleted resources in a FIFO +#. Should check if the readers have completed a grace period and free the resources. + +There are several APIs provided to help with this process. The writer +can create a FIFO to store the references to deleted resources using ``rte_rcu_qsbr_dq_create()``. +The resources can be enqueued to this FIFO using ``rte_rcu_qsbr_dq_enqueue()``. +If the FIFO is full, ``rte_rcu_qsbr_dq_enqueue`` will reclaim the resources before enqueuing. It will also reclaim resources on regular basis to keep the FIFO from growing too large. If the writer runs out of resources, the writer can call ``rte_rcu_qsbr_dq_reclaim`` API to reclaim resources. ``rte_rcu_qsbr_dq_delete`` is provided to reclaim any remaining resources and free the FIFO while shutting down. + +However, if this resource reclamation process were to be integrated in lock-free data structure libraries, it +hides this complexity from the application and makes it easier for the application to adopt lock-free algorithms. The following paragraphs discuss how the reclamation process can be integrated in DPDK libraries. + +In any DPDK application, the resource reclamation process using QSBR can be split into 4 parts: + +#. Initialization +#. Quiescent State Reporting +#. Reclaiming Resources +#. Shutdown + +The design proposed here assigns different parts of this process to client libraries and applications. The term 'client library' refers to lock-free data structure libraries such at rte_hash, rte_lpm etc. in DPDK or similar libraries outside of DPDK. The term 'application' refers to the packet processing application that makes use of DPDK such as L3 Forwarding example application, OVS, VPP etc.. + +The application has to handle 'Initialization' and 'Quiescent State Reporting'. So, + +* the application has to create the RCU variable and register the reader threads to report their quiescent state. +* the application has to register the same RCU variable with the client library. +* reader threads in the application have to report the quiescent state. This allows for the application to control the length of the critical section/how frequently the application wants to report the quiescent state. + +The client library will handle 'Reclaiming Resources' part of the process. The +client libraries will make use of the writer thread context to execute the memory +reclamation algorithm. So, + +* client library should provide an API to register a RCU variable that it will use. It should call ``rte_rcu_qsbr_dq_create()`` to create the FIFO to store the references to deleted entries. +* client library should use ``rte_rcu_qsbr_dq_enqueue`` to enqueue the deleted resources on the FIFO and start the grace period. +* if the library runs out of resources while adding entries, it should call ``rte_rcu_qsbr_dq_reclaim`` to reclaim the resources and try the resource allocation again. + +The 'Shutdown' process needs to be shared between the application and the +client library. + +* the application should make sure that the reader threads are not using the shared data structure, unregister the reader threads from the QSBR variable before calling the client library's shutdown function. + +* client library should call ``rte_rcu_qsbr_dq_delete`` to reclaim any remaining resources and free the FIFO. + +Integrating the resource reclamation with client libraries removes the burden from +the application and makes it easy to use lock-free algorithms. + +This design has several advantages over currently known methods. + +#. Application does not need a dedicated thread to reclaim resources. Memory + reclamation happens as part of the writer thread with little impact on + performance. +#. The client library has better control over the resources. For ex: the client + library can attempt to reclaim when it has run out of resources.
Signature · 1 line
2.17.1
PATCH v5 4/4 lib/rcu: add additional debug logs
In reply to Honnappa NagarahalliAdded additional debug logs. These helped in debugging RCU defer APIs.
--- lib/librte_rcu/rte_rcu_qsbr.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
lib/librte_rcu/rte_rcu_qsbr.h +8 -1
@@ -718,8 +718,15 @@ rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)RTE_ASSERT (v !=NULL );/* Check if all the readers have already acknowledged this token */ -if (likely (t <= v->acked_token)) +if (likely (t <= v->acked_token)) { +__RTE_RCU_DP_LOG (DEBUG , +"%s: check: token = %" PRIu64", wait = %d" , + __func__, t, wait); +__RTE_RCU_DP_LOG (DEBUG , +"%s: status: least acked token = %" PRIu64"" , + __func__, v->acked_token);return 1 ; + }if (likely (v->num_threads == v->max_threads))return __rte_rcu_qsbr_check_all (v, t, wait);
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 4/4] lib/rcu: add additional debug logs
Quoted 23 lines
Added additional debug logs. These helped in debugging RCU defer APIs.
Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Gavin Hu <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>--- lib/librte_rcu/rte_rcu_qsbr.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)lib/librte_rcu/rte_rcu_qsbr.h +3
-1@@ -718,8 +718,15 @@ rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)RTE_ASSERT (v !=NULL );/* Check if all the readers have already acknowledged this token */ -if (likely (t <= v->acked_token)) +if (likely (t <= v->acked_token)) { + __RTE_RCU_DP_LOG(DEBUG, + "%s: check: token = %"PRIu64", wait = %d",
lib/librte_rcu/rte_rcu_qsbr.h rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)
Quoted 10 lines
lib/librte_rcu/rte_rcu_qsbr.h +5
-1+ __func__, t, wait); +__RTE_RCU_DP_LOG (DEBUG , +"%s: status: least acked token = %" PRIu64"" , + __func__, v->acked_token);return 1 ; + }if (likely (v->num_threads == v->max_threads))return __rte_rcu_qsbr_check_all (v, t, wait); --
Quoted 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 0/4] Add RCU reclamation APIs
In reply to Honnappa NagarahalliQuoted 8 lines · Honnappa Nagarahalli
This is not a new patch. This patch set is separated from the LPM changes as the size of the changes in RCU library has grown due to comments from community. These APIs will help reduce the changes in LPM and hash libraries that are getting integrated with RCU library.
This adds 4 new APIs to RCU library to create a defer queue, enqueue deleted resources, reclaim resources and delete the defer queue.
Fixed a few typos.
Added missing dependency of librte_rcu on librte_ring for make.
Series applied, thanks.
Signature · 1 line
David Marchand
v3 cover RCU integration with LPM library
In reply to Ruifeng WangThis patch set is dependent on https://patches.dpdk.org/cover/60270/
This patchset integrates RCU QSBR support with LPM library.
Please refer to RCU documentation in the above mentioned patch series. This patch set follows the suggested design of integrating RCU library with other libraries in DPDK.
RCU is used to safely free tbl8 groups that can be recycled. tbl8 groups will not be reclaimed or reused until readers stopped referencing it.
This is implemented as an optional feature to ensure the existing applications are not affected. New API rte_lpm_rcu_qsbr_add is introduced for application to register a RCU variable that LPM library will use. This provides user the handle to enable this feature.
v3:
1) Integration with new RCU defer queue APIs (much smaller and simpler code in LPM library itself)
2) Separated the 'test/lpm: reset total time' patch from this series
3) Added multi-writer performance test. The performance difference between with and without RCU varies and is not small for multi-writer. However, this is due to the tbl8 group allocation algorithm in LPM, which is a linear search algorithm (given that the test case uses large number of tbl8 groups). We should look to change this algorithm to O(1) in the future.
4) Incorporated applicable feedback from Vladimir
Honnappa Nagarahalli (1):
test/lpm: add RCU integration performance tests
Ruifeng Wang (2):
lib/lpm: integrate RCU QSBR
app/test: add test case for LPM RCU integration
app/test/test_lpm.c | 152 ++++++++- app/test/test_lpm_perf.c | 487 ++++++++++++++++++++++++++++- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++- lib/librte_lpm/rte_lpm.h | 21 ++ lib/librte_lpm/rte_lpm_version.map | 6 + 7 files changed, 757 insertions(+), 16 deletions(-)
Signature · 1 line
2.17.1
PATCH v3 1/3 lib/lpm: integrate RCU QSBR
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)
lib/librte_lpm/Makefile +2 -1
@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library name LIB = librte_lpm.a +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.map
lib/librte_lpm/meson.build +2 -0
@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers = files('rte_lpm.h', 'rte_lpm6.h') # since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']
lib/librte_lpm/rte_lpm.c +91 -11
169 lines not on this page · raw message
lib/librte_lpm/rte_lpm.h +21 -0
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_
@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {
@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/**
@@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void rte_lpm_free_v1604 (struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr *v); +/** * Add a rule to the LPM table. *
lib/librte_lpm/rte_lpm_version.map +6 -0
@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR
Hi Honnappa,
Quoted 73 lines · Honnappa Nagarahalli
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)lib/librte_lpm/Makefile +2
-1@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library name LIB = librte_lpm.a +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +2
-0@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers = files('rte_lpm.h', 'rte_lpm6.h') # since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <string.h> @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 394
Quoted 2 lines
lib/librte_lpm/rte_lpm.c +0
-0@@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604);
As a general comment, are you going to add rcu support to the legacy _v20 ?
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 399
Quoted 5 lines
lib/librte_lpm/rte_lpm.c +4
-0+struct __rte_lpm_rcu_dq_entry { +uint32_t tbl8_group_index; +uint32_t pad; +};
Is this struct necessary? I mean in tbl8_free_v1604() you can pass tbl8_group_index as a pointer without "e.pad = 0;".
And what about 32bit environment?
Quoted 24 lines
lib/librte_lpm/rte_lpm.c +24
-0+ +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data) +{ +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_rcu_qsbr *v) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; + +if ((lpm ==NULL ) || (v ==NULL )) { + rte_errno =EINVAL ;
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 433
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+return 1 ; + } + +if (lpm->dq) { + rte_errno =EEXIST ; +return 1 ; + } + +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"LPM_RCU_%s" , lpm->name);
Consider moving this logic into rte_rcu_qsbr_dq_create(). I think there you could prefix the name with just RCU_ . So it would be possible to move include <rte_ring.h> into the rte_rcu_qsbr.c from rte_rcu_qsbr.h and get rid of RTE_RCU_QSBR_DQ_NAMESIZE macro in rte_rcu_qsbr.h file.
Quoted 187 lines
lib/librte_lpm/rte_lpm.c +50
-11+ params.name = rcu_dq_name; + params.size = lpm->number_tbl8s; + params.esize =sizeof (struct __rte_lpm_rcu_dq_entry ); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8; + params.v = v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM ,"LPM QS defer queue creation failed\n" ); +return 1 ; + } + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8) }static int32_t -tbl8_alloc_v1604 (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +__tbl8_alloc_v1604 (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *RTE_LPM_TBL8_GROUP_NUM_ENTRIES ]; +for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES ];/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = {@@ -712,6 +769,21 @@ tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)return -ENOSPC ; } +static int32_t +tbl8_alloc_v1604 (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =__tbl8_alloc_v1604 (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim some. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq) ==0 ) + group_idx =__tbl8_alloc_v1604 (lpm); + } + +return group_idx; +} +static void tbl8_free_v20 (struct rte_lpm_tbl_entry_v20 *tbl8,uint32_t tbl8_group_start) {@@ -728,13 +800,21 @@ tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start) }static void -tbl8_free_v1604 (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free_v1604 (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry e; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (lpm->dq !=NULL ) { + e.tbl8_group_index = tbl8_group_start; + e.pad =0 ; +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&e); + }else { +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + } }static __rte_noinline int32_t@@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth,if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, }/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);if (tbl8_group_index <0 ) {return tbl8_group_index;@@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = {@@ -1834,7 +1914,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,__atomic_store (&lpm->tbl24[tbl24_index], &new_tbl24_entry, __ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +21
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void rte_lpm_free_v1604 (struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr *v); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-0@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR
<snip>
Quoted 78 lines
Hi Honnappa,
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,
Quoted 73 lines · Honnappa Nagarahalli
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
a7946a1c5..ca9e16312100644lib/librte_lpm/Makefile +2
-1@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library name LIB = librte_lpm.a +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +2
-0@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers = files('rte_lpm.h', 'rte_lpm6.h') # since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
3a929a1b1..ca58d4b35100644lib/librte_lpm/rte_lpm.c +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <string.h> @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);@@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,16.04);
Quoted 3 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 394
Quoted 2 lines
lib/librte_lpm/rte_lpm.c +0
-0MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604);As a general comment, are you going to add rcu support to the legacy
_v20?
I do not see a requirement from my side. What's your suggestion?
Quoted 8 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 399
Quoted 5 lines
lib/librte_lpm/rte_lpm.c +4
-0+struct __rte_lpm_rcu_dq_entry { +uint32_t tbl8_group_index; +uint32_t pad; +};Is this struct necessary? I mean in
tbl8_free_v1604()you can passtbl8_group_indexas a pointer without "e.pad = 0;".
Agree, that is another way. This structure will go away once the ring library supports storing 32b elements.
Quoted 2 lines
And what about 32bit environment?
Waiting for rte_ring to support 32b elements (the patch is being discussed).
Quoted 31 lines
Quoted 31 lines
lib/librte_lpm/rte_lpm.c +30
-0+ +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_rcu_qsbr *v) { +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; + +if ((lpm ==NULL ) || (v ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->dq) { + rte_errno =EEXIST ; +return 1 ; + } +
Quoted 8 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, · line 431
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +2
-0+/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"LPM_RCU_%s" , lpm- name);Consider moving this logic into
rte_rcu_qsbr_dq_create(). I think there you could prefix the name with justRCU_. So it would be possible to move include <rte_ring.h> into therte_rcu_qsbr.cfromrte_rcu_qsbr.hand get rid ofRTE_RCU_QSBR_DQ_NAMESIZEmacro inrte_rcu_qsbr.hfile.
Macro is required to provide a length for the name, similar to what rte_ring does. What would be the length of the 'name' if RTE_RCU_QSBR_DQ_NAMESIZE is removed?
If the 'RCU_' has to be prefixed in 'rte_rcu_qsbr_dq_create', then RTE_RCU_QSBR_DQ_NAMESIZE needs to be readjusted in the header file. I am trying to keep it simple by constructing the string in a single function.
Quoted 209 lines
lib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20
Quoted 20 lines
lib/librte_lpm/rte_lpm.c +15
-0+ params.name = rcu_dq_name; + params.size = lpm->number_tbl8s; + params.esize =sizeof (struct __rte_lpm_rcu_dq_entry ); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8; + params.v = v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM ,"LPM QS defer queue creation failed\n" ); +return 1 ; + } + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20*tbl8)
lib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8)
Quoted 13 lines
lib/librte_lpm/rte_lpm.c +1
-3}static int32_t -tbl8_alloc_v1604 (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +__tbl8_alloc_v1604 (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8)
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES];Quoted 3 lines
lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) {struct
rte_lpm_tbl_entrynew_tbl8_entry= { @@ -712,6 +769,21 @@
Quoted 10 lines
tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
return-ENOSPC;
}lib/librte_lpm/rte_lpm.c +6
-0+static int32_t +tbl8_alloc_v1604 (struct rte_lpm *lpm) { +int32_t group_idx;/* tbl8 group index. */ + + group_idx =__tbl8_alloc_v1604 (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) {lib/librte_lpm/rte_lpm.c tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +8
-0+/* If there are no tbl8 groups try to reclaim some. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq) ==0 ) + group_idx =__tbl8_alloc_v1604 (lpm); + } + +return group_idx; +} +static void tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t
tbl8_group_start)lib/librte_lpm/rte_lpm.c tbl8_free_v20(struct rte_lpm_tbl_entry_v20
Quoted 2 lines
{lib/librte_lpm/rte_lpm.c +0
-0@@ -728,13 +800,21 @@ tbl8_free_v20(struct rte_lpm_tbl_entry_v20*tbl8,
uint32_ttbl8_group_start)Quoted 11 lines
lib/librte_lpm/rte_lpm.c +2
-2}static void -tbl8_free_v1604 (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free_v1604 (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry e;lib/librte_lpm/rte_lpm.c tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +8
-2-__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (lpm->dq !=NULL ) { + e.tbl8_group_index = tbl8_group_start; + e.pad =0 ; +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&e); + }else { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c add_depth_big_v1604(struct rte_lpm *lpm,
Quoted 17 lines
lib/librte_lpm/rte_lpm.c +2
-0+ __ATOMIC_RELAXED); + } }static
__rte_noinlineint32_tlib/librte_lpm/rte_lpm.c +1
-1@@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index < 0) {lib/librte_lpm/rte_lpm.c +0
-0@@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm,
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-1}/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);if (tbl8_group_index <0 ) {return
tbl8_group_index;lib/librte_lpm/rte_lpm.c +0
-0@@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +1
-1*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -1834,7+1914,7 @@
Quoted 2 lines
delete_depth_big_v1604(struct
rte_lpm*lpm,uint32_tip_masked,
__atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,Quoted 66 lines
__ATOMIC_RELAXED); __atomic_thread_fence(__ATOMIC_RELEASE);lib/librte_lpm/rte_lpm.c +1
-1-tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h index renamed from lib/librte_lpm/rte_lpm.h +0
-0
906ec4483..49c12a68d100644lib/librte_lpm/rte_lpm.h +22
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void rte_lpm_free_v1604 (struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr +*v); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +0
-0lib/librte_lpm/rte_lpm_version.map DPDK_17.05 { · line 52
Quoted 9 lines
lib/librte_lpm/rte_lpm_version.map +6
-0@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};--
Regards,
Vladimir
Re: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR
In reply to Honnappa NagarahalliHi guys,
Quoted 116 lines
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)lib/librte_lpm/Makefile +2
-1@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library name LIB = librte_lpm.a +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +2
-0@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers = files('rte_lpm.h', 'rte_lpm6.h') # since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c +42
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <string.h> @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);@@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604); +struct __rte_lpm_rcu_dq_entry { +uint32_t tbl8_group_index; +uint32_t pad; +}; + +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data) +{ +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_rcu_qsbr *v) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; + +if ((lpm ==NULL ) || (v ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->dq) { + rte_errno =EEXIST ; +return 1 ; + } + +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"LPM_RCU_%s" , lpm->name); + params.name = rcu_dq_name;
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 444
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+ params.size = lpm->number_tbl8s; + params.esize =sizeof (struct __rte_lpm_rcu_dq_entry ); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8; + params.v = v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM ,"LPM QS defer queue creation failed\n" ); +return 1 ; + }
Few thoughts about that function:
It names rcu_qsbr_add() but in fact it allocates defer queue for give rcu var. So first thought - is it always necessary?
For some use-cases I suppose user might be ok to wait for quiescent state change inside tbl8_free()?
Another thing you do allocate defer queue, but it is internal, so user can't call reclaim() manually, which looks strange.
Why not to return defer_queue pointer to the user, so he can call reclaim() himself at appropriate time?
Third thing - you always allocate defer queue with size equal to number of tbl8. Though I understand it could be up to 16M tbl8 groups inside the LPM.
Do we really need defer queue that long?
Especially considering that current rcu_defer_queue will start reclamation when 1/8 of defer_quueue becomes full and wouldn't reclaim more then 1/16 of it. Probably better to let user to decide himself how long defer_queue he needs for that LPM?
Konstantin
Quoted 178 lines
lib/librte_lpm/rte_lpm.c +39
-11+ +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8) }static int32_t -tbl8_alloc_v1604 (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +__tbl8_alloc_v1604 (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *RTE_LPM_TBL8_GROUP_NUM_ENTRIES ]; +for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES ];/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = {@@ -712,6 +769,21 @@ tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)return -ENOSPC ; } +static int32_t +tbl8_alloc_v1604 (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =__tbl8_alloc_v1604 (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim some. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq) ==0 ) + group_idx =__tbl8_alloc_v1604 (lpm); + } + +return group_idx; +} +static void tbl8_free_v20 (struct rte_lpm_tbl_entry_v20 *tbl8,uint32_t tbl8_group_start) {@@ -728,13 +800,21 @@ tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start) }static void -tbl8_free_v1604 (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free_v1604 (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry e; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (lpm->dq !=NULL ) { + e.tbl8_group_index = tbl8_group_start; + e.pad =0 ; +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&e); + }else { +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + } }static __rte_noinline int32_t@@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth,if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, }/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);if (tbl8_group_index <0 ) {return tbl8_group_index;@@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = {@@ -1834,7 +1914,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,__atomic_store (&lpm->tbl24[tbl24_index], &new_tbl24_entry, __ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +21
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void rte_lpm_free_v1604 (struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr *v); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-1@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; --2.17.1
Re: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR
<snip>
Quoted 1 line
Hi guys,
I have tried to consolidate design related questions here. If I have missed anything, please add.
Quoted 119 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,
Quoted 73 lines
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
a7946a1c5..ca9e16312100644lib/librte_lpm/Makefile +0
-0@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library nameLIB = librte_lpm.a
lib/librte_lpm/Makefile +2
-0+CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +1
-0@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers =
files('rte_lpm.h', 'rte_lpm6.h')# since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers +=files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')deps += ['hash']lib/librte_lpm/meson.build +1
-0+deps += ['rcu']lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
3a929a1b1..ca58d4b35100644lib/librte_lpm/rte_lpm.c +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <string.h> @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);@@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,16.04);
Quoted 45 lines
lib/librte_lpm/rte_lpm.c +41
-0MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604); +struct __rte_lpm_rcu_dq_entry { +uint32_t tbl8_group_index; +uint32_t pad; +}; + +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_rcu_qsbr *v) { +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; + +if ((lpm ==NULL ) || (v ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->dq) { + rte_errno =EEXIST ; +return 1 ; + } + +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"LPM_RCU_%s" , lpm- name); + params.name = rcu_dq_name; + params.size = lpm->number_tbl8s; + params.esize =sizeof (struct __rte_lpm_rcu_dq_entry ); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8;
Quoted 8 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 444
Quoted 6 lines
lib/librte_lpm/rte_lpm.c +6
-0+ params.v = v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM ,"LPM QS defer queue creation failed\n" ); +return 1 ; + }Few thoughts about that function:
Few things to keep in mind, the goal of the design is to make it easy for the applications to adopt lock-free algorithms. The reclamation process in the writer is a major portion of code one has to write for using lock-free algorithms. The current design is such that the writer does not have to change any code or write additional code other than calling 'rte_lpm_rcu_qsbr_add'.
Quoted 2 lines
It names
rcu_qsbr_add()but in fact it allocates defer queue for give rcu var. So first thought - is it always necessary?
This is part of the design. If the application does not want to use this integrated logic then, it does not have to call this API. It can use the RCU defer APIs to implement its own logic. But, if I ask the question, does this integrated logic address most of the use cases of the LPM library, I think the answer is yes.
Quoted 3 lines
For some use-cases I suppose user might be ok to wait for quiescent state change
insidetbl8_free()?
Yes, that is a possibility (for ex: no frequent route changes). But, I think that is very trivial for the application to implement. Though, the LPM library has to separate the 'delete' and 'free' operations. Similar operations are provided in rte_hash library. IMO, we should follow consistent approach.
Quoted 4 lines
Another thing you do allocate defer queue, but it is internal, so user can't call
reclaim()manually, which looks strange.
Why not to returndefer_queuepointer to the user, so he can callreclaim()himself at appropriate time?
The intention of the design is to take the complexity away from the user of LPM library. IMO, the current design will address most uses cases of LPM library. If we expose the 2 parameters (when to trigger reclamation and how much to reclaim) in the 'rte_lpm_rcu_qsbr_add' API, it should provide enough flexibility to the application.
Quoted 4 lines
Third thing - you always allocate defer queue with size equal to number of tbl8.
Though I understand it could be up to 16M tbl8 groups inside the LPM. Do we really need defer queue that long?
No, we do not need it to be this long. It is this long today to avoid returning no-space on the defer queue error.
Quoted 5 lines
Especially considering that current
rcu_defer_queuewill start reclamation when 1/8 ofdefer_quueuebecomes full and wouldn't reclaim more then 1/16 of it.
Probably better to let user to decide himself how longdefer_queuehe needs for that LPM?
It makes sense to expose it to the user if the writer-writer concurrency is lock-free (no memory allocation allowed to expand the defer queue size when the queue is full). However, LPM is not lock-free on the writer side. If we think the writer could be lock-free in the future, it has to be exposed to the user.
Quoted 2 lines
Konstantin
Pulling questions/comments from other threads:
Can we leave reclamation to some other house-keeping thread to do (sort of garbage collector). Or such mode is not supported/planned?
[Honnappa] If the reclamation cost is small, the current method provides advantages over having a separate thread to do reclamation. I did not plan to provide such an option. But may be it makes sense to keep the options open (especially from ABI perspective). May be we should add a flags field which will allow us to implement different methods in the future?
Quoted 193 lines
Quoted 13 lines
lib/librte_lpm/rte_lpm.c +4
-0+ +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8) } static int32_tlib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 · line 690
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-3-tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) +__tbl8_alloc_v1604 (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8)
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES];Quoted 3 lines
lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) {struct
rte_lpm_tbl_entrynew_tbl8_entry= { @@ -712,6 +769,21 @@
Quoted 32 lines
tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)lib/librte_lpm/rte_lpm.c +14
-0return -ENOSPC ; } +static int32_t +tbl8_alloc_v1604 (struct rte_lpm *lpm) { +int32_t group_idx;/* tbl8 group index. */ + + group_idx =__tbl8_alloc_v1604 (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim some. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq) ==0 ) + group_idx =__tbl8_alloc_v1604 (lpm); + } + +return group_idx; +} +static void tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t
tbl8_group_start) { @@ -728,13 +800,21 @@tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)}static void
-tbl8_free_v1604(structrte_lpm_tbl_entry*tbl8,uint32_ttbl8_group_start)lib/librte_lpm/rte_lpm.c +2
-1+tbl8_free_v1604(struct rte_lpm *lpm, uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry e;lib/librte_lpm/rte_lpm.c tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +8
-2-__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (lpm->dq !=NULL ) { + e.tbl8_group_index = tbl8_group_start; + e.pad =0 ; +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&e); + }else { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c add_depth_big_v1604(struct rte_lpm *lpm,
Quoted 17 lines
lib/librte_lpm/rte_lpm.c +3
-1+ __ATOMIC_RELAXED); + } }static __rte_noinline int32_t@@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);lib/librte_lpm/rte_lpm.c +0
-0/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm,
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-1}/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);if (tbl8_group_index <0 ) {lib/librte_lpm/rte_lpm.c +0
-0return tbl8_group_index;@@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +1
-1*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -1834,7+1914,7 @@
Quoted 2 lines
delete_depth_big_v1604(struct
rte_lpm*lpm,uint32_tip_masked,
__atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,lib/librte_lpm/rte_lpm.h rte_lpm_free_v20(struct rte_lpm_v20 *lpm);
Quoted 35 lines
lib/librte_lpm/rte_lpm.c +1
-1__ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h index renamed from lib/librte_lpm/rte_lpm.h +0
-0
906ec4483..49c12a68d100644lib/librte_lpm/rte_lpm.h +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void
Quoted 41 lines
lib/librte_lpm/rte_lpm.h +19
-0rte_lpm_free_v1604(struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr +*v); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-1@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; --2.17.1
Re: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR
Quoted 143 lines
<snip>
Quoted 1 line
Hi guys,
I have tried to consolidate design related questions here. If I have missed anything, please add.
Quoted 119 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,
Quoted 73 lines
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
a7946a1c5..ca9e16312100644lib/librte_lpm/Makefile +0
-0@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library nameLIB = librte_lpm.a
lib/librte_lpm/Makefile +2
-0+CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +1
-0@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers =
files('rte_lpm.h', 'rte_lpm6.h')# since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers +=files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')deps += ['hash']lib/librte_lpm/meson.build +1
-0+deps += ['rcu']lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
3a929a1b1..ca58d4b35100644lib/librte_lpm/rte_lpm.c +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <string.h> @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);@@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,16.04);
Quoted 45 lines
lib/librte_lpm/rte_lpm.c +41
-0MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604); +struct __rte_lpm_rcu_dq_entry { +uint32_t tbl8_group_index; +uint32_t pad; +}; + +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_rcu_qsbr *v) { +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; + +if ((lpm ==NULL ) || (v ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->dq) { + rte_errno =EEXIST ; +return 1 ; + } + +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"LPM_RCU_%s" , lpm- name); + params.name = rcu_dq_name; + params.size = lpm->number_tbl8s; + params.esize =sizeof (struct __rte_lpm_rcu_dq_entry ); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8;Quoted 8 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 444
Quoted 6 lines
lib/librte_lpm/rte_lpm.c +6
-0+ params.v = v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM ,"LPM QS defer queue creation failed\n" ); +return 1 ; + }Few thoughts about that function:
Few things to keep in mind, the goal of the design is to make it easy for the applications to adopt lock-free algorithms. The reclamation process in the writer is a major portion of code one has to write for using lock-free algorithms. The current design is such that the writer does not have to change any code or write additional code other than calling '
rte_lpm_rcu_qsbr_add'.Quoted 2 lines
It names
rcu_qsbr_add()but in fact it allocates defer queue for give rcu var. So first thought - is it always necessary?This is part of the design. If the application does not want to use this integrated logic then, it does not have to call this API. It can use the RCU defer APIs to implement its own logic. But, if I ask the question, does this integrated logic address most of the use cases of the LPM library, I think the answer is yes.
Quoted 5 lines
Quoted 3 lines
For some use-cases I suppose user might be ok to wait for quiescent state change
insidetbl8_free()?Yes, that is a possibility (for ex: no frequent route changes). But, I think that is very trivial for the application to implement. Though, the LPM library has to separate the 'delete' and 'free' operations.
Exactly.
That's why it is not trivial with current LPM library.
In fact to do that himself right now, user would have to implement and support his own version of LPM code.
Honestly, I don't understand why you consider it as a drawback. From my perspective only few things need to be changed:
1. Add 2 parameters to 'rte_lpm_rcu_qsbr_add():
number of elems in defer_queue
reclaim() threshold value.
If the user doesn't want to provide any values, that's fine we can use default ones here (as you do it right now).
2. Make rte_lpm_rcu_qsbr_add() to return pointer to the defer_queue.
Again if user doesn't want to call reclaim() himself, he can just ignore return value.
These 2 changes will provide us with necessary flexibility that would help to cover more use-cases:
- user can decide how big should be the defer queue
- user can decide when/how he wants to do reclaim()
Konstantin
Quoted 228 lines
Similar operations are provided in
rte_hashlibrary. IMO, we should follow consistent approach.Quoted 4 lines
Another thing you do allocate defer queue, but it is internal, so user can't call
reclaim()manually, which looks strange.
Why not to returndefer_queuepointer to the user, so he can callreclaim()himself at appropriate time?The intention of the design is to take the complexity away from the user of LPM library. IMO, the current design will address most uses cases of LPM library. If we expose the 2 parameters (when to trigger reclamation and how much to reclaim) in the '
rte_lpm_rcu_qsbr_add' API, it should provide enough flexibility to the application.Quoted 4 lines
Third thing - you always allocate defer queue with size equal to number of tbl8.
Though I understand it could be up to 16M tbl8 groups inside the LPM. Do we really need defer queue that long?No, we do not need it to be this long. It is this long today to avoid returning no-space on the defer queue error.
Quoted 5 lines
Especially considering that current
rcu_defer_queuewill start reclamation when 1/8 ofdefer_quueuebecomes full and wouldn't reclaim more then 1/16 of it.
Probably better to let user to decide himself how longdefer_queuehe needs for that LPM?It makes sense to expose it to the user if the writer-writer concurrency is lock-free (no memory allocation allowed to expand the defer queue size when the queue is full). However, LPM is not lock-free on the writer side. If we think the writer could be lock-free in the future, it has to be exposed to the user.
Quoted 2 lines
Konstantin
Pulling questions/comments from other threads:
Can we leave reclamation to some other house-keeping thread to do (sort of garbage collector). Or such mode is not supported/planned?[Honnappa] If the reclamation cost is small, the current method provides advantages over having a separate thread to do reclamation. I did not plan to provide such an option. But may be it makes sense to keep the options open (especially from ABI perspective). May be we should add a flags field which will allow us to implement different methods in the future?
Quoted 193 lines
Quoted 13 lines
lib/librte_lpm/rte_lpm.c +4
-0+ +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8) } static int32_tlib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 · line 690
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-3-tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) +__tbl8_alloc_v1604 (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8)
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES];Quoted 3 lines
lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) {struct
rte_lpm_tbl_entrynew_tbl8_entry= { @@ -712,6 +769,21 @@
Quoted 32 lines
tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)lib/librte_lpm/rte_lpm.c +14
-0return -ENOSPC ; } +static int32_t +tbl8_alloc_v1604 (struct rte_lpm *lpm) { +int32_t group_idx;/* tbl8 group index. */ + + group_idx =__tbl8_alloc_v1604 (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim some. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq) ==0 ) + group_idx =__tbl8_alloc_v1604 (lpm); + } + +return group_idx; +} +static void tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t
tbl8_group_start) { @@ -728,13 +800,21 @@tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)}static void
-tbl8_free_v1604(structrte_lpm_tbl_entry*tbl8,uint32_ttbl8_group_start)lib/librte_lpm/rte_lpm.c +2
-1+tbl8_free_v1604(struct rte_lpm *lpm, uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry e;lib/librte_lpm/rte_lpm.c tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +8
-2-__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (lpm->dq !=NULL ) { + e.tbl8_group_index = tbl8_group_start; + e.pad =0 ; +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&e); + }else { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c add_depth_big_v1604(struct rte_lpm *lpm,
Quoted 17 lines
lib/librte_lpm/rte_lpm.c +3
-1+ __ATOMIC_RELAXED); + } }static __rte_noinline int32_t@@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);lib/librte_lpm/rte_lpm.c +0
-0/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm,
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-1}/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);if (tbl8_group_index <0 ) {lib/librte_lpm/rte_lpm.c +0
-0return tbl8_group_index;@@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +1
-1*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -1834,7+1914,7 @@
Quoted 2 lines
delete_depth_big_v1604(struct
rte_lpm*lpm,uint32_tip_masked,
__atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,lib/librte_lpm/rte_lpm.h rte_lpm_free_v20(struct rte_lpm_v20 *lpm);
Quoted 35 lines
lib/librte_lpm/rte_lpm.c +1
-1__ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h index renamed from lib/librte_lpm/rte_lpm.h +0
-0
906ec4483..49c12a68d100644lib/librte_lpm/rte_lpm.h +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void
Quoted 41 lines
lib/librte_lpm/rte_lpm.h +19
-0rte_lpm_free_v1604(struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr +*v); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-1@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; --2.17.1
Re: [dpdk-dev] [PATCH v3 1/3] lib/lpm: integrate RCU QSBR
<snip>
Quoted 152 lines
Quoted 3 lines
Quoted 2 lines
Hi guys,
I have tried to consolidate design related questions here. If I have missed
anything, please add.
Quoted 131 lines
Quoted 123 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,
Quoted 77 lines
From: Ruifeng Wang <[email protected]>
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- lib/librte_lpm/Makefile | 3 +- lib/librte_lpm/meson.build | 2 + lib/librte_lpm/rte_lpm.c | 102 +++++++++++++++++++++++++---- lib/librte_lpm/rte_lpm.h | 21 ++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 5 files changed, 122 insertions(+), 12 deletions(-)lib/librte_lpm/Makefile +0
-0index
a7946a1c5..ca9e16312100644lib/librte_lpm/Makefile +0
-0@@ -6,9 +6,10 @@ include $(RTE_SDK)/mk/rte.vars.mk # library nameLIB = librte_lpm.a
lib/librte_lpm/Makefile +2
-1+CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.map(quoted diff) +0
-0b/lib/librte_lpm/meson.build index
a5176d8ae..19a35107f100644lib/librte_lpm/meson.build +1
-0@@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 2 +allow_experimental_apis = true sources = files('rte_lpm.c', 'rte_lpm6.c') headers =
files('rte_lpm.h', 'rte_lpm6.h')# since header files have different names, we can install all vector headers # without worrying about which architecture we actually need headers +=files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')deps += ['hash']lib/librte_lpm/meson.build +1
-0+deps += ['rcu']lib/librte_lpm/rte_lpm.c +0
-0index
3a929a1b1..ca58d4b35100644lib/librte_lpm/rte_lpm.c +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <string.h> @@ -381,6 +382,8 @@ rte_lpm_free_v1604(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);@@ -390,6 +393,59 @@ BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604,16.04);
Quoted 45 lines
lib/librte_lpm/rte_lpm.c +41
-0MAP_STATIC_SYMBOL(void rte_lpm_free(struct rte_lpm *lpm), rte_lpm_free_v1604); +struct __rte_lpm_rcu_dq_entry { +uint32_t tbl8_group_index; +uint32_t pad; +}; + +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry *e = + (struct __rte_lpm_rcu_dq_entry *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[e->tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_rcu_qsbr *v) { +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params; + +if ((lpm ==NULL ) || (v ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->dq) { + rte_errno =EEXIST ; +return 1 ; + } + +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name),"LPM_RCU_%s" , lpm- name); + params.name = rcu_dq_name; + params.size = lpm->number_tbl8s; + params.esize =sizeof (struct __rte_lpm_rcu_dq_entry ); + params.f = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8;Quoted 8 lines
lib/librte_lpm/rte_lpm.c BIND_DEFAULT_SYMBOL(rte_lpm_free, _v1604, 16.04); · line 444
Quoted 6 lines
lib/librte_lpm/rte_lpm.c +6
-0+ params.v = v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM ,"LPM QS defer queue creation failed\n" ); +return 1 ; + }Few thoughts about that function:
Quoted 3 lines
Few things to keep in mind, the goal of the design is to make it easy for the applications to adopt lock-free algorithms. The reclamation process in the writer is a major portion of code one has to write for using
lock-free algorithms. The current design is such that the writer does not have to change any code or write additional code other than calling '
rte_lpm_rcu_qsbr_add'.Quoted 6 lines
Quoted 3 lines
It names
rcu_qsbr_add()but in fact it allocates defer queue for give rcu var. So first thought - is it always necessary?This is part of the design. If the application does not want to use this integrated logic then, it does not have to call this API. It can use the RCU defer APIs to implement its own logic. But, if I ask the question,
does this integrated logic address most of the use cases of the LPM library, I think the answer is yes.
Quoted 3 lines
Quoted 3 lines
For some use-cases I suppose user might be ok to wait for quiescent state change inside
tbl8_free()?
Quoted 8 lines
Quoted 2 lines
Yes, that is a possibility (for ex: no frequent route changes). But, I think that is very trivial for the application to implement. Though, the LPM
library has to separate the 'delete' and 'free' operations.
Exactly.
That's why it is not trivial with current LPM library.
In fact to do that himself right now, user would have to implement and support his own version of LPM code.
😊, well we definitely don't want them to write their own library (if DPDK LPM is enough)
IMO, we need to be consistent with other libraries in terms of APIs. That's another topic.
I do not see any problem to implement this or provide facility to implement this in the future in the APIs now. We can add 'flags' field which will allow for other methods of reclamation.
Quoted 9 lines
Honestly, I don't understand why you consider it as a drawback. From my perspective only few things need to be changed:
1. Add 2 parameters to '
rte_lpm_rcu_qsbr_add():
number of elems indefer_queue
reclaim()threshold value.
If the user doesn't want to provide any values, that's fine we can use default ones here (as you do it right now).
I think we have agreed on this, I see the value in doing this.
Quoted 3 lines
2. Make
rte_lpm_rcu_qsbr_add()to return pointer to thedefer_queue. Again if user doesn't want to callreclaim()himself, he can just ignore return value.
Given the goal of reducing the burden on the user, this is not in that direction. But if you see a use case for it, I don't have any issues. Vladimir asked for it as well in the other thread.
Quoted 246 lines
These 2 changes will provide us with necessary flexibility that would help to cover more use-cases:
- user can decide how big should be the defer queue
- user can decide when/how he wants to doreclaim()Konstantin
Quoted 10 lines
Similar operations are provided in
rte_hashlibrary. IMO, we should follow consistent approach.Quoted 5 lines
Another thing you do allocate defer queue, but it is internal, so user can't call
reclaim()manually, which looks strange.
Why not to returndefer_queuepointer to the user, so he can callreclaim()himself at appropriate time?The intention of the design is to take the complexity away from the user of LPM library. IMO, the current design will address most uses cases of
LPM library. If we expose the 2 parameters (when to trigger reclamation and how much to reclaim) in the '
rte_lpm_rcu_qsbr_add'Quoted 7 lines
API, it should provide enough flexibility to the application.
Quoted 4 lines
Third thing - you always allocate defer queue with size equal to number of tbl8.
Though I understand it could be up to 16M tbl8 groups inside the LPM. Do we really need defer queue that long?No, we do not need it to be this long. It is this long today to avoid returning
no-space on the defer queue error.
Quoted 10 lines
Quoted 7 lines
Especially considering that current
rcu_defer_queuewill start reclamation when 1/8 ofdefer_quueuebecomes full and wouldn't reclaim more then
1/16 of it.
Probably better to let user to decide himself how longdefer_queuehe needs for that LPM?It makes sense to expose it to the user if the writer-writer concurrency is lock-free (no memory allocation allowed to expand the defer queue size when the queue is full). However, LPM is not lock-free on
the writer side. If we think the writer could be lock-free in the future, it has to be exposed to the user.
Quoted 5 lines
Quoted 3 lines
Konstantin
Pulling questions/comments from other threads:
Can we leave reclamation to some other house-keeping thread to do (sort ofgarbage collector). Or such mode is not supported/planned?
Quoted 4 lines
[Honnappa] If the reclamation cost is small, the current method provides advantages over having a separate thread to do reclamation. I did not plan to provide such an option. But may be it makes sense to keep the
options open (especially from ABI perspective). May be we should add a flags field which will allow us to implement different methods in the future?
Quoted 194 lines
Quoted 194 lines
Quoted 14 lines
lib/librte_lpm/rte_lpm.c +4
-0+ +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -679,14 +735,15 @@ tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8) } static int32_tlib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 · line 690
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-3-tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) +__tbl8_alloc_v1604 (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c tbl8_alloc_v20(struct rte_lpm_tbl_entry_v20 *tbl8)
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES];Quoted 3 lines
lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) {struct
rte_lpm_tbl_entrynew_tbl8_entry= { @@ -712,6 +769,21 @@
Quoted 32 lines
tbl8_alloc_v1604(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)lib/librte_lpm/rte_lpm.c +14
-0return -ENOSPC ; } +static int32_t +tbl8_alloc_v1604 (struct rte_lpm *lpm) { +int32_t group_idx;/* tbl8 group index. */ + + group_idx =__tbl8_alloc_v1604 (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim some. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq) ==0 ) + group_idx =__tbl8_alloc_v1604 (lpm); + } + +return group_idx; +} +static void tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t
tbl8_group_start) { @@ -728,13 +800,21 @@tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)}static void
-tbl8_free_v1604(structrte_lpm_tbl_entry*tbl8,uint32_ttbl8_group_start)lib/librte_lpm/rte_lpm.c +2
-1+tbl8_free_v1604(struct rte_lpm *lpm, uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +struct __rte_lpm_rcu_dq_entry e;lib/librte_lpm/rte_lpm.c tbl8_free_v20(struct rte_lpm_tbl_entry_v20 *tbl8, uint32_t tbl8_group_start)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +8
-2-__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (lpm->dq !=NULL ) { + e.tbl8_group_index = tbl8_group_start; + e.pad =0 ; +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&e); + }else { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c add_depth_big_v1604(struct rte_lpm *lpm,
Quoted 17 lines
lib/librte_lpm/rte_lpm.c +3
-1+ __ATOMIC_RELAXED); + } }static __rte_noinline int32_t@@ -1037,7 +1117,7 @@ add_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);lib/librte_lpm/rte_lpm.c +0
-0/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -1083,7 +1163,7 @@ add_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm,
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-1}/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc_v1604 (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc_v1604 (lpm);if (tbl8_group_index <0 ) {lib/librte_lpm/rte_lpm.c +0
-0return tbl8_group_index;@@ -1818,7 +1898,7 @@ delete_depth_big_v1604(struct rte_lpm *lpm,
uint32_tip_masked,lib/librte_lpm/rte_lpm.c delete_depth_big_v1604(struct rte_lpm *lpm, uint32_t ip_masked,
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +1
-1*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -1834,7+1914,7 @@
Quoted 2 lines
delete_depth_big_v1604(struct
rte_lpm*lpm,uint32_tip_masked,
__atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,lib/librte_lpm/rte_lpm.h rte_lpm_free_v20(struct rte_lpm_v20 *lpm);
Quoted 35 lines
lib/librte_lpm/rte_lpm.c +1
-1__ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free_v1604 (lpm->tbl8, tbl8_group_start); +tbl8_free_v1604 (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +3
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #ifndef _RTE_LPM_H_@@ -21,6 +22,7 @@#include <rte_common.h> #include <rte_vect.h> #include <rte_compat.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -186,6 +188,7 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +struct rte_rcu_qsbr_dq *dq;/**< RCU QSBR defer queue.*/ };/** @@ -248,6 +251,24 @@ rte_lpm_free_v20(struct rte_lpm_v20 *lpm);void
Quoted 41 lines
lib/librte_lpm/rte_lpm.h +19
-0rte_lpm_free_v1604(struct rte_lpm *lpm); +/** +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param v +* RCU QSBR variable +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_rcu_qsbr +*v); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-1@@ -44,3 +44,9 @@ DPDK_17.05 { rte_lpm6_lookup_bulk_func; } DPDK_16.04; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; --2.17.1
PATCH v3 2/3 app/test: add test case for LPM RCU integration
In reply to Honnappa NagarahalliFrom: Ruifeng Wang <[email protected]>
Add positive and negative tests for API rte_lpm_rcu_qsbr_add. Also test LPM library behavior when RCU QSBR is enabled.
--- app/test/test_lpm.c | 152 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 1 deletion(-)
app/test/test_lpm.c +151 -1
@@ -8,6 +8,7 @@#include <rte_ip.h> #include <rte_lpm.h> +#include <rte_malloc.h> #include "test.h" #include "test_xmmt_ops.h"
@@ -40,6 +41,8 @@ static int32_t test15(void);static int32_t test16 (void );static int32_t test17 (void );static int32_t test18 (void ); +static int32_t test19 (void ); +static int32_t test20 (void );rte_lpm_test tests[] = {/* Test Cases */
@@ -61,7 +64,9 @@ rte_lpm_test tests[] = { test15, test16, test17, - test18 + test18, + test19, + test20 };#define NUM_LPM_TESTS (sizeof(tests)/sizeof(tests[0]))
@@ -1266,6 +1271,151 @@ test18(void)return PASS ; } +/* +* rte_lpm_rcu_qsbr_add positive and negative tests. +* - Add RCU QSBR variable to LPM +* - Add another RCU QSBR variable to LPM +* - Check LPM attached RCU QSBR variable and FIFO queue +*/ +int32_t +test19 (void ) +{ +struct rte_lpm *lpm =NULL ; +struct rte_lpm_config config; +size_t sz; +struct rte_rcu_qsbr *qsv; +struct rte_rcu_qsbr *qsv2; +int32_t status; + + config.max_rules =MAX_RULES ; + config.number_tbl8s =NUMBER_TBL8S ; + config.flags =0 ; + + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Create RCU QSBR variable */ + sz =rte_rcu_qsbr_get_memsize (RTE_MAX_LCORE ); + qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (qsv !=NULL ); + + status =rte_rcu_qsbr_init (qsv,RTE_MAX_LCORE ); +TEST_LPM_ASSERT (status ==0 ); + +/* Attach RCU QSBR to LPM table */ + status =rte_lpm_rcu_qsbr_add (lpm, qsv); +TEST_LPM_ASSERT (status ==0 ); + +/* Create and attach another RCU QSBR to LPM table */ + qsv2 = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (qsv2 !=NULL ); + + status =rte_lpm_rcu_qsbr_add (lpm, qsv2); +TEST_LPM_ASSERT (status !=0 ); + +TEST_LPM_ASSERT (lpm->dq !=NULL ); + +rte_lpm_free (lpm); +rte_free (qsv); +rte_free (qsv2); + +return PASS ; +} + +/* +* rte_lpm_rcu_qsbr_add functional test. +* - Create LPM which supports 1 tbl8 group at max +* - Add RCU QSBR variable to LPM +* - Add a rule with depth=28 (> 24) +* - Register a reader thread (not a real thread) +* - Reader lookup existing rule +* - Writer delete the rule +* - Reader lookup the rule +* - Writer re-add the rule (no available tbl8 group) +* - Reader report quiescent state and unregister +* - Writer re-add the rule +* - Reader lookup the rule +*/ +int32_t +test20 (void ) +{ +struct rte_lpm *lpm =NULL ; +struct rte_lpm_config config; +size_t sz; +struct rte_rcu_qsbr *qsv; +int32_t status; +uint32_t ip, next_hop, next_hop_return; +uint8_t depth; + + config.max_rules =MAX_RULES ; + config.number_tbl8s =1 ; + config.flags =0 ; + + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Create RCU QSBR variable */ + sz =rte_rcu_qsbr_get_memsize (1 ); + qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (qsv !=NULL ); + + status =rte_rcu_qsbr_init (qsv,1 ); +TEST_LPM_ASSERT (status ==0 ); + +/* Attach RCU QSBR to LPM table */ + status =rte_lpm_rcu_qsbr_add (lpm, qsv); +TEST_LPM_ASSERT (status ==0 ); + + ip =RTE_IPV4 (192 ,18 ,100 ,100 ); + depth =28 ; + next_hop =1 ; + status =rte_lpm_add (lpm, ip, depth, next_hop); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (lpm->tbl24[ip>>8 ].valid_group); + +/* Register pseudo reader */ + status =rte_rcu_qsbr_thread_register (qsv,0 ); +TEST_LPM_ASSERT (status ==0 ); +rte_rcu_qsbr_thread_online (qsv,0 ); + + status =rte_lpm_lookup (lpm, ip, &next_hop_return); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (next_hop_return == next_hop); + +/* Writer update */ + status =rte_lpm_delete (lpm, ip, depth); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (!lpm->tbl24[ip>>8 ].valid); + + status =rte_lpm_lookup (lpm, ip, &next_hop_return); +TEST_LPM_ASSERT (status !=0 ); + + status =rte_lpm_add (lpm, ip, depth, next_hop); +TEST_LPM_ASSERT (status !=0 ); + +/* Reader quiescent */ +rte_rcu_qsbr_quiescent (qsv,0 ); + + status =rte_lpm_add (lpm, ip, depth, next_hop); +TEST_LPM_ASSERT (status ==0 ); + +rte_rcu_qsbr_thread_offline (qsv,0 ); + status =rte_rcu_qsbr_thread_unregister (qsv,0 ); +TEST_LPM_ASSERT (status ==0 ); + + status =rte_lpm_lookup (lpm, ip, &next_hop_return); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (next_hop_return == next_hop); + +rte_lpm_free (lpm); +rte_free (qsv); + +return PASS ; +} +/* * Do all unit tests. */
Signature · 1 line
2.17.1
PATCH v3 3/3 test/lpm: add RCU integration performance tests
In reply to Honnappa NagarahalliAdd performance tests for RCU integration. The performance difference with and without RCU integration is very small (~1% to ~2%) on both Arm and x86 platforms.
--- app/test/test_lpm_perf.c | 487 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 484 insertions(+), 3 deletions(-)
app/test/test_lpm_perf.c +484 -3
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h>
@@ -10,12 +11,28 @@#include <rte_cycles.h> #include <rte_random.h> #include <rte_branch_prediction.h> +#include <rte_malloc.h> #include <rte_ip.h> #include <rte_lpm.h> +#include <rte_rcu_qsbr.h> #include "test.h" #include "test_xmmt_ops.h" +struct rte_lpm *lpm; +static struct rte_rcu_qsbr *rv; +static volatile uint8_t writer_done; +static volatile uint32_t thr_id; +static rte_atomic64_t gwrite_cycles; +static rte_atomic64_t gwrites; +/* LPM APIs are not thread safe, use mutex to provide thread safety */ +static pthread_mutex_t lpm_mutex =PTHREAD_MUTEX_INITIALIZER ; + +/* Report quiescent state interval every 8192 lookups. Larger critical +* sections in reader will result in writer polling multiple times. +*/ +#define QSBR_REPORTING_INTERVAL 1024 +#define TEST_LPM_ASSERT (cond) do { \ if (!(cond)) { \ printf("Error at line %d: \n", __LINE__); \
@@ -24,6 +41,7 @@ } while(0)#define ITERATIONS (1 << 10) +#define RCU_ITERATIONS 10#define BATCH_SIZE (1 << 12)#define BULK_SIZE 32
@@ -35,9 +53,13 @@ struct route_rule { };struct route_rule large_route_table[MAX_RULE_NUM ]; +/* Route table for routes with depth > 24 */ +struct route_rule large_ldepth_route_table[MAX_RULE_NUM ];static uint32_t num_route_entries; +static uint32_t num_ldepth_route_entries;#define NUM_ROUTE_ENTRIES num_route_entries +#define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entriesenum {IP_CLASS_A ,
@@ -191,7 +213,7 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)uint32_t ip_head_mask;uint32_t rule_num;uint32_t k; -struct route_rule *ptr_rule; +struct route_rule *ptr_rule, *ptr_ldepth_rule;if (ip_class ==IP_CLASS_A ) {/* IP Address class A */ fixed_bit_num =IP_HEAD_BIT_NUM_A ;
@@ -236,10 +258,20 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)*/ start =lrand48 () & mask; ptr_rule = &large_route_table[num_route_entries]; + ptr_ldepth_rule = &large_ldepth_route_table[num_ldepth_route_entries];for (k =0 ; k < rule_num; k++) { ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth)) | ip_head_mask; ptr_rule->depth = depth; +/* If the depth of the route is more than 24, store it +* in another table as well. +*/ +if (depth >24 ) { + ptr_ldepth_rule->ip = ptr_rule->ip; + ptr_ldepth_rule->depth = ptr_rule->depth; + ptr_ldepth_rule++; + num_ldepth_route_entries++; + } ptr_rule++; start = (start + step) & mask; }
@@ -273,6 +305,7 @@ static void generate_large_route_rule_table(void)uint8_t depth; num_route_entries =0 ; + num_ldepth_route_entries =0 ;memset (large_route_table,0 ,sizeof (large_route_table));for (ip_class =IP_CLASS_A ; ip_class <=IP_CLASS_C ; ip_class++) {
@@ -316,10 +349,454 @@ print_route_distribution(const struct route_rule *table, uint32_t n)printf ("\n" ); } +/* Check condition and return an error if true. */ +static uint16_t enabled_core_ids[RTE_MAX_LCORE ]; +static unsigned int num_cores; + +/* Simple way to allocate thread ids in 0 to RTE_MAX_LCORE space */ +static inline uint32_t +alloc_thread_id (void ) +{ +uint32_t tmp_thr_id; + + tmp_thr_id =__atomic_fetch_add (&thr_id,1 , __ATOMIC_RELAXED); +if (tmp_thr_id >=RTE_MAX_LCORE ) +printf ("Invalid thread id %u\n" , tmp_thr_id); + +return tmp_thr_id; +} + +/* +* Reader thread using rte_lpm data structure without RCU. +*/ +static int +test_lpm_reader (__attribute__((unused))void *arg) +{ +int i; +uint32_t ip_batch[QSBR_REPORTING_INTERVAL ]; +uint32_t next_hop_return =0 ; + +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) + ip_batch[i] =rte_rand (); + +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (lpm, ip_batch[i], &next_hop_return); + + }while (!writer_done); + +return 0 ; +} + +/* +* Reader thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_reader (__attribute__((unused))void *arg) +{ +int i; +uint32_t thread_id =alloc_thread_id (); +uint32_t ip_batch[QSBR_REPORTING_INTERVAL ]; +uint32_t next_hop_return =0 ; + +/* Register this thread to report quiescent state */ +rte_rcu_qsbr_thread_register (rv, thread_id); +rte_rcu_qsbr_thread_online (rv, thread_id); + +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) + ip_batch[i] =rte_rand (); + +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (lpm, ip_batch[i], &next_hop_return); + +/* Update quiescent state */ +rte_rcu_qsbr_quiescent (rv, thread_id); + }while (!writer_done); + +rte_rcu_qsbr_thread_offline (rv, thread_id); +rte_rcu_qsbr_thread_unregister (rv, thread_id); + +return 0 ; +} + +/* +* Writer thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_writer (__attribute__((unused))void *arg) +{ +unsigned int i, j, si, ei; +uint64_t begin, total_cycles; +uint8_t core_id = (uint8_t )((uintptr_t )arg); +uint32_t next_hop_add =0xAA ; + +/* 2 writer threads are used */ +if (core_id %2 ==0 ) { + si =0 ; + ei =NUM_LDEPTH_ROUTE_ENTRIES /2 ; + }else { + si =NUM_LDEPTH_ROUTE_ENTRIES /2 ; + ei =NUM_LDEPTH_ROUTE_ENTRIES ; + } + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j = si; j < ei; j++) { +pthread_mutex_lock (&lpm_mutex); +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + } +pthread_mutex_unlock (&lpm_mutex); + } + +/* Delete all the entries */ +for (j = si; j < ei; j++) { +pthread_mutex_lock (&lpm_mutex); +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + } +pthread_mutex_unlock (&lpm_mutex); + } + } + + total_cycles =rte_rdtsc_precise () - begin; + +rte_atomic64_add (&gwrite_cycles, total_cycles); +rte_atomic64_add (&gwrites, +2 *NUM_LDEPTH_ROUTE_ENTRIES *RCU_ITERATIONS ); + +return 0 ; +} + +/* +* Functional test: +* 2 writers, rest are readers +*/ +static int +test_lpm_rcu_perf_multi_writer (void ) +{ +struct rte_lpm_config config; +size_t sz; +unsigned int i; +uint16_t core_id; + +if (rte_lcore_count () <3 ) { +printf ("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n" ); +return TEST_SKIPPED ; + } + + num_cores =0 ; +RTE_LCORE_FOREACH_SLAVE (core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + +printf ("\nPerf test: 2 writers, %d readers, RCU integration enabled\n" , + num_cores -2 ); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Init RCU variable */ + sz =rte_rcu_qsbr_get_memsize (num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc ("rcu0" , sz, +RTE_CACHE_LINE_SIZE ); +rte_rcu_qsbr_init (rv, num_cores); + +/* Assign the RCU variable to LPM */ +if (rte_lpm_rcu_qsbr_add (lpm, rv) !=0 ) { +printf ("RCU variable assignment failed\n" ); + goto error; + } + + writer_done =0 ; +rte_atomic64_init (&gwrite_cycles); +rte_atomic64_init (&gwrites); +rte_atomic64_clear (&gwrite_cycles); +rte_atomic64_clear (&gwrites); + +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =2 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , + enabled_core_ids[i]); + +/* Launch writer threads */ +for (i =0 ; i <2 ; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t )i, + enabled_core_ids[i]); + +/* Wait for writer threads */ +for (i =0 ; i <2 ; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +printf ("Total LPM Adds: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %lu cycles\n" , +rte_atomic64_read (&gwrite_cycles) /rte_atomic64_read (&gwrites) + ); + +/* Wait and check return value from reader threads */ + writer_done =1 ; +for (i =2 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); +rte_free (rv); + lpm =NULL ; + rv =NULL ; + +/* Test without RCU integration */ +printf ("\nPerf test: 2 writers, %d readers, RCU integration disabled\n" , + num_cores -2 ); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + + writer_done =0 ; +rte_atomic64_init (&gwrite_cycles); +rte_atomic64_init (&gwrites); +rte_atomic64_clear (&gwrite_cycles); +rte_atomic64_clear (&gwrites); +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =2 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_reader,NULL , + enabled_core_ids[i]); + +/* Launch writer threads */ +for (i =0 ; i <2 ; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t )i, + enabled_core_ids[i]); + +/* Wait for writer threads */ +for (i =0 ; i <2 ; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +printf ("Total LPM Adds: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %lu cycles\n" , +rte_atomic64_read (&gwrite_cycles) /rte_atomic64_read (&gwrites) + ); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =2 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); + +return 0 ; + +error: + writer_done =1 ; +/* Wait until all readers have exited */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (lpm); +rte_free (rv); + +return -1 ; +} + +/* +* Functional test: +* Single writer, rest are readers +*/ +static int +test_lpm_rcu_perf (void ) +{ +struct rte_lpm_config config; +uint64_t begin, total_cycles; +size_t sz; +unsigned int i, j; +uint16_t core_id; +uint32_t next_hop_add =0xAA ; + +if (rte_lcore_count () <2 ) { +printf ("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n" ); +return TEST_SKIPPED ; + } + + num_cores =0 ; +RTE_LCORE_FOREACH_SLAVE (core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + +printf ("\nPerf test: 1 writer, %d readers, RCU integration enabled\n" , + num_cores); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Init RCU variable */ + sz =rte_rcu_qsbr_get_memsize (num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc ("rcu0" , sz, +RTE_CACHE_LINE_SIZE ); +rte_rcu_qsbr_init (rv, num_cores); + +/* Assign the RCU variable to LPM */ +if (rte_lpm_rcu_qsbr_add (lpm, rv) !=0 ) { +printf ("RCU variable assignment failed\n" ); + goto error; + } + + writer_done =0 ; +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =0 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , + enabled_core_ids[i]); + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + goto error; + } + +/* Delete all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + goto error; + } + } + total_cycles =rte_rdtsc_precise () - begin; + +printf ("Total LPM Adds: %d\n" ,ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %g cycles\n" , + (double )total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *ITERATIONS )); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =0 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); +rte_free (rv); + lpm =NULL ; + rv =NULL ; + +/* Test without RCU integration */ +printf ("\nPerf test: 1 writer, %d readers, RCU integration disabled\n" , + num_cores); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + + writer_done =0 ; +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =0 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_reader,NULL , + enabled_core_ids[i]); + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + goto error; + } + +/* Delete all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + goto error; + } + } + total_cycles =rte_rdtsc_precise () - begin; + +printf ("Total LPM Adds: %d\n" ,ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %g cycles\n" , + (double )total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *ITERATIONS )); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =0 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) +printf ("Warning: lcore %u not finished.\n" , + enabled_core_ids[i]); + +rte_lpm_free (lpm); + +return 0 ; + +error: + writer_done =1 ; +/* Wait until all readers have exited */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (lpm); +rte_free (rv); + +return -1 ; +} +static int test_lpm_perf (void ) { -struct rte_lpm *lpm =NULL ;struct rte_lpm_config config; config.max_rules =2000000 ;
@@ -343,7 +820,7 @@ test_lpm_perf(void) lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config);TEST_LPM_ASSERT (lpm !=NULL ); -/* Measue add. */ +/* Measure add. */ begin =rte_rdtsc ();for (i =0 ; i <NUM_ROUTE_ENTRIES ; i++) {
@@ -478,6 +955,10 @@ test_lpm_perf(void)rte_lpm_delete_all (lpm);rte_lpm_free (lpm); +test_lpm_rcu_perf (); + +test_lpm_rcu_perf_multi_writer (); +return 0 ; }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v3 3/3] test/lpm: add RCU integration performance tests
Quoted 8 lines · Honnappa Nagarahalli
Add performance tests for RCU integration. The performance difference with and without RCU integration is very small (~1% to ~2%) on both Arm and x86 platforms.
Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Gavin Hu <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>---
I see the following:
lib/meson.build:89:5: ERROR: Problem encountered: Missing dependency rcu for library rte_lpm
Maybe there's something wrong with the environment? This isn't the first time I've seen a dependency detection problem with meson.
Quoted 578 lines
app/test/test_lpm_perf.c | 487 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 484 insertions(+), 3 deletions(-)app/test/test_lpm_perf.c +484
-3@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2019 Arm Limited */ #include <stdio.h> @@ -10,12 +11,28 @@#include <rte_cycles.h> #include <rte_random.h> #include <rte_branch_prediction.h> +#include <rte_malloc.h> #include <rte_ip.h> #include <rte_lpm.h> +#include <rte_rcu_qsbr.h> #include "test.h" #include "test_xmmt_ops.h" +struct rte_lpm *lpm; +static struct rte_rcu_qsbr *rv; +static volatile uint8_t writer_done; +static volatile uint32_t thr_id; +static rte_atomic64_t gwrite_cycles; +static rte_atomic64_t gwrites; +/* LPM APIs are not thread safe, use mutex to provide thread safety */ +static pthread_mutex_t lpm_mutex =PTHREAD_MUTEX_INITIALIZER ; + +/* Report quiescent state interval every 8192 lookups. Larger critical +* sections in reader will result in writer polling multiple times. +*/ +#define QSBR_REPORTING_INTERVAL 1024 +#define TEST_LPM_ASSERT (cond) do { \ if (!(cond)) { \ printf("Error at line %d: \n", __LINE__); \@@ -24,6 +41,7 @@ } while(0)#define ITERATIONS (1 << 10) +#define RCU_ITERATIONS 10#define BATCH_SIZE (1 << 12)#define BULK_SIZE 32@@ -35,9 +53,13 @@ struct route_rule { };struct route_rule large_route_table[MAX_RULE_NUM ]; +/* Route table for routes with depth > 24 */ +struct route_rule large_ldepth_route_table[MAX_RULE_NUM ];static uint32_t num_route_entries; +static uint32_t num_ldepth_route_entries;#define NUM_ROUTE_ENTRIES num_route_entries +#define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entriesenum {IP_CLASS_A ,@@ -191,7 +213,7 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)uint32_t ip_head_mask;uint32_t rule_num;uint32_t k; -struct route_rule *ptr_rule; +struct route_rule *ptr_rule, *ptr_ldepth_rule;if (ip_class ==IP_CLASS_A ) {/* IP Address class A */ fixed_bit_num =IP_HEAD_BIT_NUM_A ;@@ -236,10 +258,20 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)*/ start =lrand48 () & mask; ptr_rule = &large_route_table[num_route_entries]; + ptr_ldepth_rule = &large_ldepth_route_table[num_ldepth_route_entries];for (k =0 ; k < rule_num; k++) { ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth)) | ip_head_mask; ptr_rule->depth = depth; +/* If the depth of the route is more than 24, store it +* in another table as well. +*/ +if (depth >24 ) { + ptr_ldepth_rule->ip = ptr_rule->ip; + ptr_ldepth_rule->depth = ptr_rule->depth; + ptr_ldepth_rule++; + num_ldepth_route_entries++; + } ptr_rule++; start = (start + step) & mask; }@@ -273,6 +305,7 @@ static void generate_large_route_rule_table(void)uint8_t depth; num_route_entries =0 ; + num_ldepth_route_entries =0 ;memset (large_route_table,0 ,sizeof (large_route_table));for (ip_class =IP_CLASS_A ; ip_class <=IP_CLASS_C ; ip_class++) {@@ -316,10 +349,454 @@ print_route_distribution(const struct route_rule *table, uint32_t n)printf ("\n" ); } +/* Check condition and return an error if true. */ +static uint16_t enabled_core_ids[RTE_MAX_LCORE ]; +static unsigned int num_cores; + +/* Simple way to allocate thread ids in 0 to RTE_MAX_LCORE space */ +static inline uint32_t +alloc_thread_id (void ) +{ +uint32_t tmp_thr_id; + + tmp_thr_id =__atomic_fetch_add (&thr_id,1 , __ATOMIC_RELAXED); +if (tmp_thr_id >=RTE_MAX_LCORE ) +printf ("Invalid thread id %u\n" , tmp_thr_id); + +return tmp_thr_id; +} + +/* +* Reader thread using rte_lpm data structure without RCU. +*/ +static int +test_lpm_reader (__attribute__((unused))void *arg) +{ +int i; +uint32_t ip_batch[QSBR_REPORTING_INTERVAL ]; +uint32_t next_hop_return =0 ; + +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) + ip_batch[i] =rte_rand (); + +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (lpm, ip_batch[i], &next_hop_return); + + }while (!writer_done); + +return 0 ; +} + +/* +* Reader thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_reader (__attribute__((unused))void *arg) +{ +int i; +uint32_t thread_id =alloc_thread_id (); +uint32_t ip_batch[QSBR_REPORTING_INTERVAL ]; +uint32_t next_hop_return =0 ; + +/* Register this thread to report quiescent state */ +rte_rcu_qsbr_thread_register (rv, thread_id); +rte_rcu_qsbr_thread_online (rv, thread_id); + +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) + ip_batch[i] =rte_rand (); + +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (lpm, ip_batch[i], &next_hop_return); + +/* Update quiescent state */ +rte_rcu_qsbr_quiescent (rv, thread_id); + }while (!writer_done); + +rte_rcu_qsbr_thread_offline (rv, thread_id); +rte_rcu_qsbr_thread_unregister (rv, thread_id); + +return 0 ; +} + +/* +* Writer thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_writer (__attribute__((unused))void *arg) +{ +unsigned int i, j, si, ei; +uint64_t begin, total_cycles; +uint8_t core_id = (uint8_t )((uintptr_t )arg); +uint32_t next_hop_add =0xAA ; + +/* 2 writer threads are used */ +if (core_id %2 ==0 ) { + si =0 ; + ei =NUM_LDEPTH_ROUTE_ENTRIES /2 ; + }else { + si =NUM_LDEPTH_ROUTE_ENTRIES /2 ; + ei =NUM_LDEPTH_ROUTE_ENTRIES ; + } + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j = si; j < ei; j++) { +pthread_mutex_lock (&lpm_mutex); +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + } +pthread_mutex_unlock (&lpm_mutex); + } + +/* Delete all the entries */ +for (j = si; j < ei; j++) { +pthread_mutex_lock (&lpm_mutex); +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + } +pthread_mutex_unlock (&lpm_mutex); + } + } + + total_cycles =rte_rdtsc_precise () - begin; + +rte_atomic64_add (&gwrite_cycles, total_cycles); +rte_atomic64_add (&gwrites, +2 *NUM_LDEPTH_ROUTE_ENTRIES *RCU_ITERATIONS ); + +return 0 ; +} + +/* +* Functional test: +* 2 writers, rest are readers +*/ +static int +test_lpm_rcu_perf_multi_writer (void ) +{ +struct rte_lpm_config config; +size_t sz; +unsigned int i; +uint16_t core_id; + +if (rte_lcore_count () <3 ) { +printf ("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n" ); +return TEST_SKIPPED ; + } + + num_cores =0 ; +RTE_LCORE_FOREACH_SLAVE (core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + +printf ("\nPerf test: 2 writers, %d readers, RCU integration enabled\n" , + num_cores -2 ); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Init RCU variable */ + sz =rte_rcu_qsbr_get_memsize (num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc ("rcu0" , sz, +RTE_CACHE_LINE_SIZE ); +rte_rcu_qsbr_init (rv, num_cores); + +/* Assign the RCU variable to LPM */ +if (rte_lpm_rcu_qsbr_add (lpm, rv) !=0 ) { +printf ("RCU variable assignment failed\n" ); + goto error; + } + + writer_done =0 ; +rte_atomic64_init (&gwrite_cycles); +rte_atomic64_init (&gwrites); +rte_atomic64_clear (&gwrite_cycles); +rte_atomic64_clear (&gwrites); + +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =2 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , + enabled_core_ids[i]); + +/* Launch writer threads */ +for (i =0 ; i <2 ; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t )i, + enabled_core_ids[i]); + +/* Wait for writer threads */ +for (i =0 ; i <2 ; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +printf ("Total LPM Adds: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %lu cycles\n" , +rte_atomic64_read (&gwrite_cycles) /rte_atomic64_read (&gwrites) + ); + +/* Wait and check return value from reader threads */ + writer_done =1 ; +for (i =2 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); +rte_free (rv); + lpm =NULL ; + rv =NULL ; + +/* Test without RCU integration */ +printf ("\nPerf test: 2 writers, %d readers, RCU integration disabled\n" , + num_cores -2 ); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + + writer_done =0 ; +rte_atomic64_init (&gwrite_cycles); +rte_atomic64_init (&gwrites); +rte_atomic64_clear (&gwrite_cycles); +rte_atomic64_clear (&gwrites); +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =2 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_reader,NULL , + enabled_core_ids[i]); + +/* Launch writer threads */ +for (i =0 ; i <2 ; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t )i, + enabled_core_ids[i]); + +/* Wait for writer threads */ +for (i =0 ; i <2 ; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +printf ("Total LPM Adds: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %lu cycles\n" , +rte_atomic64_read (&gwrite_cycles) /rte_atomic64_read (&gwrites) + ); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =2 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); + +return 0 ; + +error: + writer_done =1 ; +/* Wait until all readers have exited */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (lpm); +rte_free (rv); + +return -1 ; +} + +/* +* Functional test: +* Single writer, rest are readers +*/ +static int +test_lpm_rcu_perf (void ) +{ +struct rte_lpm_config config; +uint64_t begin, total_cycles; +size_t sz; +unsigned int i, j; +uint16_t core_id; +uint32_t next_hop_add =0xAA ; + +if (rte_lcore_count () <2 ) { +printf ("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n" ); +return TEST_SKIPPED ; + } + + num_cores =0 ; +RTE_LCORE_FOREACH_SLAVE (core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + +printf ("\nPerf test: 1 writer, %d readers, RCU integration enabled\n" , + num_cores); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Init RCU variable */ + sz =rte_rcu_qsbr_get_memsize (num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc ("rcu0" , sz, +RTE_CACHE_LINE_SIZE ); +rte_rcu_qsbr_init (rv, num_cores); + +/* Assign the RCU variable to LPM */ +if (rte_lpm_rcu_qsbr_add (lpm, rv) !=0 ) { +printf ("RCU variable assignment failed\n" ); + goto error; + } + + writer_done =0 ; +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =0 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , + enabled_core_ids[i]); + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + goto error; + } + +/* Delete all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + goto error; + } + } + total_cycles =rte_rdtsc_precise () - begin; + +printf ("Total LPM Adds: %d\n" ,ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %g cycles\n" , + (double )total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *ITERATIONS )); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =0 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); +rte_free (rv); + lpm =NULL ; + rv =NULL ; + +/* Test without RCU integration */ +printf ("\nPerf test: 1 writer, %d readers, RCU integration disabled\n" , + num_cores); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + + writer_done =0 ; +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =0 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_reader,NULL , + enabled_core_ids[i]); + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + goto error; + } + +/* Delete all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + goto error; + } + } + total_cycles =rte_rdtsc_precise () - begin; + +printf ("Total LPM Adds: %d\n" ,ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %g cycles\n" , + (double )total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *ITERATIONS )); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =0 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) +printf ("Warning: lcore %u not finished.\n" , + enabled_core_ids[i]); + +rte_lpm_free (lpm); + +return 0 ; + +error: + writer_done =1 ; +/* Wait until all readers have exited */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (lpm); +rte_free (rv); + +return -1 ; +} +static int test_lpm_perf (void ) { -struct rte_lpm *lpm =NULL ;struct rte_lpm_config config; config.max_rules =2000000 ;@@ -343,7 +820,7 @@ test_lpm_perf(void) lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config);TEST_LPM_ASSERT (lpm !=NULL ); -/* Measue add. */ +/* Measure add. */ begin =rte_rdtsc ();for (i =0 ; i <NUM_ROUTE_ENTRIES ; i++) {@@ -478,6 +955,10 @@ test_lpm_perf(void)rte_lpm_delete_all (lpm);rte_lpm_free (lpm); +test_lpm_rcu_perf (); + +test_lpm_rcu_perf_multi_writer (); +return 0 ; }
Re: [dpdk-dev] [PATCH v3 3/3] test/lpm: add RCU integration performance tests
Quoted 11 lines · Aaron Conole
Quoted 8 lines · Honnappa Nagarahalli
Add performance tests for RCU integration. The performance difference with and without RCU integration is very small (~1% to ~2%) on both Arm and x86 platforms.
Signed-off-by Honnappa Nagarahalli <[email protected]>Reviewed-by Gavin Hu <[email protected]>Reviewed-by Ruifeng Wang <[email protected]>---
Quoted 8 lines
I see the following:
lib/meson.build:89:5: ERROR: Problem encountered: Missing dependency rcu for libraryrte_lpmMaybe there's something wrong with the environment? This isn't the first time I've seen a dependency detection problem with meson.
It probably not a detection problem, more likely the rcu library is not being built for some reason. If you apply patch [1] the meson run will print out each library and the dependency object generated for it as each is processed. That should help debug issues like this.
/Bruce
v4 cover RCU integration with LPM library
In reply to Ruifeng WangThis patchset integrates RCU QSBR support with LPM library.
Resource reclaimation implementation was splitted from the original series, and has already been part of RCU library. Rework the series to base LPM integration on RCU reclaimation APIs.
New API rte_lpm_rcu_qsbr_add is introduced for application to register a RCU variable that LPM library will use. This provides user the handle to enable RCU that integrated in LPM library.
Functional tests and performance tests are added to cover the integration with RCU.
---
v4:
Allow user to configure defer queue: size, reclaim threshold, max entries. Return defer queue handler so user can manually trigger reclaimation. Add blocking mode support. Defer queue will not be created.
Honnappa Nagarahalli (1):
test/lpm: add RCU integration performance tests
Ruifeng Wang (2):
lib/lpm: integrate RCU QSBR
test/lpm: add LPM RCU integration functional tests
app/test/test_lpm.c | 293 ++++++++++++++++- app/test/test_lpm_perf.c | 492 ++++++++++++++++++++++++++++- doc/guides/prog_guide/lpm_lib.rst | 32 ++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 +++++++- lib/librte_lpm/rte_lpm.h | 59 ++++ lib/librte_lpm/rte_lpm_version.map | 6 + 8 files changed, 992 insertions(+), 16 deletions(-)
Signature · 1 line
2.17.1
PATCH v4 1/3 lib/lpm: integrate RCU QSBR
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
--- doc/guides/prog_guide/lpm_lib.rst | 32 ++++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 ++++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 211 insertions(+), 12 deletions(-)
doc/guides/prog_guide/lpm_lib.rst +32 -0
@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. + Lookup ~~~~~~
lib/librte_lpm/Makefile +1 -1
@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.map
lib/librte_lpm/meson.build +1 -0
@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']
lib/librte_lpm/rte_lpm.c +112 -11
185 lines not on this page · raw message
lib/librte_lpm/rte_lpm.h +59 -0
88 lines not on this page · raw message
lib/librte_lpm/rte_lpm_version.map +6 -0
@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
<snip>
Quoted 125 lines
Subject: [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 ++++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 ++++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 211 insertions(+), 12 deletions(-)doc/guides/prog_guide/lpm_lib.rst +0
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the nexttable or not.
Prefix expansion is one of the keys of this algorithm, since it improves the
speed dramatically by adding redundancy.doc/guides/prog_guide/lpm_lib.rst +5
-0+Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement +rule is an existing rule that has the longest prefix match with the rule to bedeleted, but has smaller depth.
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +If a replacement rule is found, target tbl24 and tbl8 entries are +updated to have the same depth and next hop value with the replacementrule.
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +If no replacement rule can be found, target tbl24 and tbl8 entries will becleared.
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32bits.
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entryare freed in following cases:
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greaterthan 24.
doc/guides/prog_guide/lpm_lib.rst +9
-0+ +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even +though the readers might be using the tbl8 group entries. This might resultin incorrect lookup results.
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +RCU QSBR process is integrated for safe tbl8 group reclaimation. +Application has certain responsibilities while using this feature. +Please refer to resource reclaimation framework of :ref:`RCU library<RCU_Library>` for more details.
doc/guides/prog_guide/lpm_lib.rst +1
-0+ Lookup ~~~~~~lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
d682785b6..6f06c5c03100644lib/librte_lpm/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build index renamed from lib/librte_lpm/meson.build +0
-0
021ac6d8d..6cfc083c5100644lib/librte_lpm/meson.build +0
-0@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # withoutworrying about which architecture we actually need headers +=
files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')deps += ['hash']lib/librte_lpm/meson.build +1
-0+deps += ['rcu']lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
38ab512a4..30f541179100644lib/librte_lpm/rte_lpm.c +10
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -246,12 +247,85 @@ rte_lpm_free(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);rte_free (te); } +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data,unsigned int n) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +uint32_t tbl8_group_index = *(uint32_t *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +RTE_SET_USED (n);
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 274
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+/* Set tbl8 group invalid */ +__atomic_store (&tbl8[tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq)
I prefer not to return the defer queue to the user here. I see 3 different ways how RCU can be integrated in the libraries:
1) The sync mode in which the defer queue is not created. The rte_rcu_qsbr_synchronize API is called after delete. The resource is freed after rte_rcu_qsbr_synchronize returns and the control is given back to the user.
2) The mode where the defer queue is created. There is a lot of flexibility provided now as the defer queue size, reclaim threshold and how many resources to reclaim are all configurable. IMO, this solves most of the use cases and helps the application integrate lock-less algorithms with minimal effort.
3) This is where the application has its own method of reclamation that does not fall under 1) or 2). To address this use case, I think we should make changes to the LPM library. Today, in LPM, the delete and free are combined into a single API. We can split this single API into 2 separate APIs - delete and free (similar thing was done to rte_hash library) without affecting the ABI. This should provide all the flexibility required for the application to implement any kind of reclamation algorithm it wants. Returning the defer queue to the user in the above API does not solve this use case.
Quoted 286 lines
lib/librte_lpm/rte_lpm.c +57
-3+{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params = {0 }; + +if ((lpm ==NULL ) || (cfg ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->v) { + rte_errno =EEXIST ; +return 1 ; + } + +if (cfg->mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* No other things to do. */ + }else if (cfg->mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name), +"LPM_RCU_%s" , lpm->name); + params.name = rcu_dq_name; + params.size = cfg->dq_size; +if (params.size ==0 ) + params.size = lpm->number_tbl8s; + params.trigger_reclaim_limit = cfg->reclaim_thd; +if (params.trigger_reclaim_limit ==0 ) + params.trigger_reclaim_limit = +RTE_LPM_RCU_DQ_RECLAIM_THD ; + params.max_reclaim_size = cfg->reclaim_max; +if (params.max_reclaim_size ==0 ) + params.max_reclaim_size =RTE_LPM_RCU_DQ_RECLAIM_MAX ; + params.esize =sizeof (uint32_t );/* tbl8 group index */ + params.free_fn = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8; + params.v = cfg->v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM , +"LPM QS defer queue creation failed\n"); +return 1 ; + } +if (dq) + *dq = lpm->dq; + }else { + rte_errno =EINVAL ; +return 1 ; + } + lpm->rcu_mode = cfg->mode; + lpm->v = cfg->v; + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -394,14 +468,15 @@ rule_find(struct rte_lpm *lpm, uint32_t ip_masked,uint8_t depth) * Find, clean and allocate a tbl8. */ static int32_t -tbl8_alloc (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +_tbl8_alloc (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx * RTE_LPM_TBL8_GROUP_NUM_ENTRIES]; +for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES ];lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = { @@ -427,14 +502,40 @@
tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)lib/librte_lpm/rte_lpm.c +9
-0return -ENOSPC ; } +static int32_t +tbl8_alloc (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =_tbl8_alloc (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim one. */ + if (rte_rcu_qsbr_dq_reclaim(lpm->dq, 1, NULL, NULL, NULL)== 0)
lib/librte_lpm/rte_lpm.c +10
-4+ group_idx =_tbl8_alloc (lpm); + } + +return group_idx; +} +static void -tbl8_free (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (!lpm->v) { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c +4
-0+ __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* Wait for quiescent state change. */ + rte_rcu_qsbr_synchronize(lpm->v,
RTE_QSBR_THRID_INVALID);lib/librte_lpm/rte_lpm.c +2
-0+/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c +4
-0+ __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Push into QSBR defer queue. */ + rte_rcu_qsbr_dq_enqueue(lpm->dq, (void*)&tbl8_group_start);
lib/librte_lpm/rte_lpm.c +2
-1+ } }static __rte_noinline int32_t@@ -523,7 +624,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);/* Check tbl8 allocation was successful. */ lib/librte_lpm/rte_lpm.c +1
-1if (tbl8_group_index <0 ) {@@ -569,7 +670,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, }/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);if (tbl8_group_index <0 ) {lib/librte_lpm/rte_lpm.c +1
-1return tbl8_group_index;@@ -977,7 +1078,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_tip_masked, */ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ lib/librte_lpm/rte_lpm.c +1
-0struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -993,7 +1094,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, __atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,lib/librte_lpm/rte_lpm.c +1
-1__ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h index renamed from lib/librte_lpm/rte_lpm.h +0
-0
b9d49ac87..8c054509a100644lib/librte_lpm/rte_lpm.h +13
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #ifndef _RTE_LPM_H_@@ -20,6 +21,7 @@#include <rte_memory.h> #include <rte_common.h> #include <rte_vect.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -62,6 +64,17 @@ extern "C" {/** Bitmask used to indicate successful lookup */ #define RTE_LPM_LOOKUP_SUCCESS 0x01000000 +/** @internal Default threshold to trigger RCU defer queue reclaimation. */ +#define RTE_LPM_RCU_DQ_RECLAIM_THD 32 + +/** @internal Default RCU defer queue entries to reclaim in one go. */ +#define RTE_LPM_RCU_DQ_RECLAIM_MAX 16 + +/* Create defer queue for reclaim. */ +#define RTE_LPM_QSBR_MODE_DQ 0 +/* Use blocking mode reclaim. No defer queue created. */ +#define RTE_LPM_QSBR_MODE_SYNC 0x01 +#if RTE_BYTE_ORDER ==RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ __extension__ @@ -130,6 +143,28 @@ struct rte_lpm {lib/librte_lpm/rte_lpm.h +18
-0__rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ + +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +uint32_t rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +}; + +/** LPM RCU QSBR configuration structure. */ struct rte_lpm_rcu_config +{ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +/* Mode of RCU QSBR. RTE_LPM_QSBR_MODE_xxx +* '0' for default: create defer queue for reclaim. +*/ +uint32_t mode; +/* RCU defer queue size. default: lpm->number_tbl8s. */ +uint32_t dq_size; +uint32_t reclaim_thd;/* Threshold to trigger auto reclaim. +* default:
RTE_LPM_RCU_DQ_RECLAIM_TRHD.lib/librte_lpm/rte_lpm.h +3
-0+*/ +uint32_t reclaim_max;/* Max entries to reclaim in one go. +* default:
RTE_LPM_RCU_DQ_RECLAIM_MAX.lib/librte_lpm/rte_lpm.h +25
-0+*/ };/** @@ -179,6 +214,30 @@ rte_lpm_find_existing(const char *name); void rte_lpm_free(struct rte_lpm *lpm); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param cfg +* RCU QSBR configuration +* @param dq +* handler of created RCU QSBR defer queue +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-1@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; --2.17.1
Re: [dpdk-dev] [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
Hi Honnappa,
Quoted 139 lines · Honnappa Nagarahalli
<snip>
Quoted 125 lines
Subject: [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 ++++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 ++++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 211 insertions(+), 12 deletions(-)doc/guides/prog_guide/lpm_lib.rst +0
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the nexttable or not.
Prefix expansion is one of the keys of this algorithm, since it improves the
speed dramatically by adding redundancy.doc/guides/prog_guide/lpm_lib.rst +5
-0+Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement +rule is an existing rule that has the longest prefix match with the rule to bedeleted, but has smaller depth.
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +If a replacement rule is found, target tbl24 and tbl8 entries are +updated to have the same depth and next hop value with the replacementrule.
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +If no replacement rule can be found, target tbl24 and tbl8 entries will becleared.
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32bits.
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entryare freed in following cases:
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greaterthan 24.
doc/guides/prog_guide/lpm_lib.rst +9
-0+ +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even +though the readers might be using the tbl8 group entries. This might resultin incorrect lookup results.
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +RCU QSBR process is integrated for safe tbl8 group reclaimation. +Application has certain responsibilities while using this feature. +Please refer to resource reclaimation framework of :ref:`RCU library<RCU_Library>` for more details.
doc/guides/prog_guide/lpm_lib.rst +1
-0+ Lookup ~~~~~~lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
d682785b6..6f06c5c03100644lib/librte_lpm/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build index renamed from lib/librte_lpm/meson.build +0
-0
021ac6d8d..6cfc083c5100644lib/librte_lpm/meson.build +0
-0@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # withoutworrying about which architecture we actually need headers +=
files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h')deps += ['hash']lib/librte_lpm/meson.build +1
-0+deps += ['rcu']lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
38ab512a4..30f541179100644lib/librte_lpm/rte_lpm.c +10
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -246,12 +247,85 @@ rte_lpm_free(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);rte_free (te); } +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data,unsigned int n) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +uint32_t tbl8_group_index = *(uint32_t *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +RTE_SET_USED (n);lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 274
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+/* Set tbl8 group invalid */ +__atomic_store (&tbl8[tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq)I prefer not to return the defer queue to the user here. I see 3 different ways how RCU can be integrated in the libraries:
Quoted 5 lines
1) The sync mode in which the defer queue is not created. The
rte_rcu_qsbr_synchronizeAPI is called after delete. The resource is freed afterrte_rcu_qsbr_synchronizereturns and the control is given back to the user.2) The mode where the defer queue is created. There is a lot of flexibility provided now as the defer queue size, reclaim threshold and how many resources to reclaim are all configurable. IMO, this solves most of the use cases and helps the application integrate lock-less algorithms with minimal effort.
3) This is where the application has its own method of reclamation that does not fall under 1) or 2). To address this use case, I think we should make changes to the LPM library. Today, in LPM, the delete and free are combined into a single API. We can split this single API into 2 separate APIs - delete and free (similar thing was done to
rte_hashlibrary) without affecting the ABI. This should provide all the flexibility required for the application to implement any kind of reclamation algorithm it wants. Returning the defer queue to the user in the above API does not solve this use case.
Agree, I don't see any case when user will need the defer queue. From my perspective reclamation of tbl8 is totally internal and user should not worry about it. So, in case of LPM I don't see any real use case when we need to enable third way of how RCU can be integrated. P.S. In rte_fib case we even don't have an opportunity to reclaim it not from the library due to rte_fib struct layout is hidden from user. Moreover, there may not be tbl8 at all, since it supports different algorithms
Quoted 286 lines
Quoted 286 lines
lib/librte_lpm/rte_lpm.c +57
-3+{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params = {0 }; + +if ((lpm ==NULL ) || (cfg ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->v) { + rte_errno =EEXIST ; +return 1 ; + } + +if (cfg->mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* No other things to do. */ + }else if (cfg->mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name), +"LPM_RCU_%s" , lpm->name); + params.name = rcu_dq_name; + params.size = cfg->dq_size; +if (params.size ==0 ) + params.size = lpm->number_tbl8s; + params.trigger_reclaim_limit = cfg->reclaim_thd; +if (params.trigger_reclaim_limit ==0 ) + params.trigger_reclaim_limit = +RTE_LPM_RCU_DQ_RECLAIM_THD ; + params.max_reclaim_size = cfg->reclaim_max; +if (params.max_reclaim_size ==0 ) + params.max_reclaim_size =RTE_LPM_RCU_DQ_RECLAIM_MAX ; + params.esize =sizeof (uint32_t );/* tbl8 group index */ + params.free_fn = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8; + params.v = cfg->v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM , +"LPM QS defer queue creation failed\n"); +return 1 ; + } +if (dq) + *dq = lpm->dq; + }else { + rte_errno =EINVAL ; +return 1 ; + } + lpm->rcu_mode = cfg->mode; + lpm->v = cfg->v; + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -394,14 +468,15 @@ rule_find(struct rte_lpm *lpm, uint32_t ip_masked,uint8_t depth) * Find, clean and allocate a tbl8. */ static int32_t -tbl8_alloc (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +_tbl8_alloc (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx * RTE_LPM_TBL8_GROUP_NUM_ENTRIES]; +for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES ];/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) {
structrte_lpm_tbl_entrynew_tbl8_entry= { @@ -
427,14 +502,40 @@tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
return-ENOSPC;
}lib/librte_lpm/rte_lpm.c +9
-0+static int32_t +tbl8_alloc (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =_tbl8_alloc (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim one. */ + if (rte_rcu_qsbr_dq_reclaim(lpm->dq, 1, NULL, NULL, NULL)== 0)
lib/librte_lpm/rte_lpm.c +10
-4+ group_idx =_tbl8_alloc (lpm); + } + +return group_idx; +} +static void -tbl8_free (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (!lpm->v) { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c +4
-0+ __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* Wait for quiescent state change. */ + rte_rcu_qsbr_synchronize(lpm->v,
RTE_QSBR_THRID_INVALID);lib/librte_lpm/rte_lpm.c +2
-0+/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c +4
-0+ __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Push into QSBR defer queue. */ + rte_rcu_qsbr_dq_enqueue(lpm->dq, (void*)&tbl8_group_start);
lib/librte_lpm/rte_lpm.c +2
-1+ } }static __rte_noinline int32_t@@ -523,7 +624,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index < 0) {lib/librte_lpm/rte_lpm.c +1
-1@@ -569,7 +670,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, }/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);if (tbl8_group_index <0 ) {return
tbl8_group_index;lib/librte_lpm/rte_lpm.c +1
-1@@ -977,7 +1078,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_tip_masked, */ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct
rte_lpm_tbl_entrynew_tbl24_entry= { @@ -993,7
+1094,7 @@ delete_depth_big(structrte_lpm*lpm,uint32_tip_masked,
__atomic_store(&lpm->tbl24[tbl24_index],&new_tbl24_entry,
__ATOMIC_RELAXED);
__atomic_thread_fence(__ATOMIC_RELEASE);lib/librte_lpm/rte_lpm.c +1
-1-tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h index renamed from lib/librte_lpm/rte_lpm.h +0
-0
b9d49ac87..8c054509a100644lib/librte_lpm/rte_lpm.h +13
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #ifndef _RTE_LPM_H_@@ -20,6 +21,7 @@#include <rte_memory.h> #include <rte_common.h> #include <rte_vect.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -62,6 +64,17 @@ extern "C" {/** Bitmask used to indicate successful lookup */ #define RTE_LPM_LOOKUP_SUCCESS 0x01000000 +/** @internal Default threshold to trigger RCU defer queue reclaimation. */ +#define RTE_LPM_RCU_DQ_RECLAIM_THD 32 + +/** @internal Default RCU defer queue entries to reclaim in one go. */ +#define RTE_LPM_RCU_DQ_RECLAIM_MAX 16 + +/* Create defer queue for reclaim. */ +#define RTE_LPM_QSBR_MODE_DQ 0 +/* Use blocking mode reclaim. No defer queue created. */ +#define RTE_LPM_QSBR_MODE_SYNC 0x01 +#if RTE_BYTE_ORDER ==RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ __extension__ @@ -130,6 +143,28 @@ struct rte_lpm {
__rte_cache_aligned; /**< LPM tbl24 table. */
structrte_lpm_tbl_entry*tbl8; /**< LPM tbl8 table. */ structrte_lpm_rule*rules_tbl; /**< LPM rules. */lib/librte_lpm/rte_lpm.h +18
-0+ +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +uint32_t rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +}; + +/** LPM RCU QSBR configuration structure. */ struct rte_lpm_rcu_config +{ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +/* Mode of RCU QSBR. RTE_LPM_QSBR_MODE_xxx +* '0' for default: create defer queue for reclaim. +*/ +uint32_t mode; +/* RCU defer queue size. default: lpm->number_tbl8s. */ +uint32_t dq_size; +uint32_t reclaim_thd;/* Threshold to trigger auto reclaim. +* default:
RTE_LPM_RCU_DQ_RECLAIM_TRHD.lib/librte_lpm/rte_lpm.h +3
-0+*/ +uint32_t reclaim_max;/* Max entries to reclaim in one go. +* default:
RTE_LPM_RCU_DQ_RECLAIM_MAX.lib/librte_lpm/rte_lpm.h +25
-0+*/ };/** @@ -179,6 +214,30 @@ rte_lpm_find_existing(const char *name); void rte_lpm_free(struct rte_lpm *lpm); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param cfg +* RCU QSBR configuration +* @param dq +* handler of created RCU QSBR defer queue +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-1@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +}; --2.17.1
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
In reply to Ruifeng WangHi Ruifeng,
Thanks for patches, see comments below
Quoted 141 lines · Ruifeng Wang
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 ++++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 ++++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 211 insertions(+), 12 deletions(-)doc/guides/prog_guide/lpm_lib.rst +32
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. + Lookup ~~~~~~lib/librte_lpm/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +1
-0@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c +37
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -246,12 +247,85 @@ rte_lpm_free(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);rte_free (te); } +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data,unsigned int n) +{ +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +uint32_t tbl8_group_index = *(uint32_t *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; + +RTE_SET_USED (n); +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params = {0 }; + +if ((lpm ==NULL ) || (cfg ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->v) { + rte_errno =EEXIST ; +return 1 ; + } + +if (cfg->mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* No other things to do. */
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 301
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+ }else if (cfg->mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name), +"LPM_RCU_%s" , lpm->name); + params.name = rcu_dq_name; + params.size = cfg->dq_size; +if (params.size ==0 ) + params.size = lpm->number_tbl8s; + params.trigger_reclaim_limit = cfg->reclaim_thd; +if (params.trigger_reclaim_limit ==0 )
This makes it impossible for a user to configure reclamation triggering with every call. Should we allow it?
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 309
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +8
-0+ params.trigger_reclaim_limit = +RTE_LPM_RCU_DQ_RECLAIM_THD ; + params.max_reclaim_size = cfg->reclaim_max; +if (params.max_reclaim_size ==0 ) + params.max_reclaim_size =RTE_LPM_RCU_DQ_RECLAIM_MAX ; + params.esize =sizeof (uint32_t );/* tbl8 group index */ + params.free_fn = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8;
I think it's better to pass the LPM pointer here rather than tbl8, for example, in case we decide to add some counters in the future
Quoted 42 lines
lib/librte_lpm/rte_lpm.c +23
-3+ params.v = cfg->v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM , +"LPM QS defer queue creation failed\n" ); +return 1 ; + } +if (dq) + *dq = lpm->dq; + }else { + rte_errno =EINVAL ; +return 1 ; + } + lpm->rcu_mode = cfg->mode; + lpm->v = cfg->v; + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -394,14 +468,15 @@ rule_find(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth)* Find, clean and allocate a tbl8. */ static int32_t -tbl8_alloc (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +_tbl8_alloc (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *RTE_LPM_TBL8_GROUP_NUM_ENTRIES ]; +for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES ];/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = {
lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) · line 511
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +7
-0@@ -427,14 +502,40 @@ tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)return -ENOSPC ; } +static int32_t +tbl8_alloc (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =_tbl8_alloc (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) {
I think (group_idx == -ENOSPC) will be safer
lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) · line 514
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+/* If there are no tbl8 groups try to reclaim one. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq,1 ,NULL ,NULL ,NULL ) ==0 ) + group_idx =_tbl8_alloc (lpm);
I'm not really happy with this approach. _tbl8_alloc() produces linear scan through the memory to find a free group_idx and it is the slowest part of rte_lpm_add().
Here after reclamation of some group index we need to rescan a memory again to find it. It would be great if there will be some way to return reclaimed elements. Or just to dequeue elements from dq and reclaim them manually.
Quoted 90 lines
lib/librte_lpm/rte_lpm.c +24
-8+ } + +return group_idx; +} +static void -tbl8_free (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (!lpm->v) { +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* Wait for quiescent state change. */ +rte_rcu_qsbr_synchronize (lpm->v,RTE_QSBR_THRID_INVALID ); +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&tbl8_group_start); + } }static __rte_noinline int32_t@@ -523,7 +624,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth,if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -569,7 +670,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, }/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);if (tbl8_group_index <0 ) {return tbl8_group_index;@@ -977,7 +1078,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t ip_masked,*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = {@@ -993,7 +1094,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t ip_masked,__atomic_store (&lpm->tbl24[tbl24_index], &new_tbl24_entry, __ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +2
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #ifndef _RTE_LPM_H_@@ -20,6 +21,7 @@#include <rte_memory.h> #include <rte_common.h> #include <rte_vect.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -62,6 +64,17 @@ extern "C" {/** Bitmask used to indicate successful lookup */ #define RTE_LPM_LOOKUP_SUCCESS 0x01000000
lib/librte_lpm/rte_lpm.h extern "C" { · line 76
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +10
-0+/** @internal Default threshold to trigger RCU defer queue reclaimation. */ +#define RTE_LPM_RCU_DQ_RECLAIM_THD 32 + +/** @internal Default RCU defer queue entries to reclaim in one go. */ +#define RTE_LPM_RCU_DQ_RECLAIM_MAX 16 + +/* Create defer queue for reclaim. */ +#define RTE_LPM_QSBR_MODE_DQ 0 +/* Use blocking mode reclaim. No defer queue created. */ +#define RTE_LPM_QSBR_MODE_SYNC 0x01
using enums instead of defines?
Quoted 78 lines
lib/librte_lpm/rte_lpm.h +47
-0+#if RTE_BYTE_ORDER ==RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ __extension__@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ + +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +uint32_t rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +}; + +/** LPM RCU QSBR configuration structure. */ +struct rte_lpm_rcu_config { +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +/* Mode of RCU QSBR. RTE_LPM_QSBR_MODE_xxx +* '0' for default: create defer queue for reclaim. +*/ +uint32_t mode; +/* RCU defer queue size. default: lpm->number_tbl8s. */ +uint32_t dq_size; +uint32_t reclaim_thd;/* Threshold to trigger auto reclaim. +* default: RTE_LPM_RCU_DQ_RECLAIM_TRHD. +*/ +uint32_t reclaim_max;/* Max entries to reclaim in one go. +* default: RTE_LPM_RCU_DQ_RECLAIM_MAX. +*/ };/** @@ -179,6 +214,30 @@ rte_lpm_find_existing(const char *name);void rte_lpm_free (struct rte_lpm *lpm); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param cfg +* RCU QSBR configuration +* @param dq +* handler of created RCU QSBR defer queue +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +6
-0@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};
Signature · 2 lines
Regards, Vladimir
Re: [dpdk-dev] [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
Hi Vladimir,
Quoted 12 lines
-----Original Message-----
From: Medvedkin, Vladimir <[email protected]>
Sent: Friday, June 19, 2020 1:22 AM
To: Ruifeng Wang <[email protected]>; Bruce Richardson <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Ray Kinsella <[email protected]>; Neil Horman <[email protected]>
Cc: [email protected]; [email protected]; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: Re: [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
Quoted 3 lines
Hi Ruifeng,
Thanks for patches, see comments below
Thanks for your review.
Quoted 160 lines · Ruifeng Wang
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the
Quoted 25 lines
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 ++++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 ++++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 211 insertions(+), 12 deletions(-)doc/guides/prog_guide/lpm_lib.rst +0
-0@@ -145,6 +145,38 @@ depending on whether we need to move to thenext table or not.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 8 lines
doc/guides/prog_guide/lpm_lib.rst +5
-0Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement +rule is an existing rule that has the longest prefix match with the rule to bedeleted, but has smaller depth.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 155
Quoted 3 lines
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +If a replacement rule is found, target tbl24 and tbl8 entries are +updated to have the same depth and next hop value with thereplacement rule.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 157
Quoted 2 lines
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +If no replacement rule can be found, target tbl24 and tbl8 entries will becleared.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 159
Quoted 2 lines
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +Prefix expansion is performed if the rule's depth is not exactly 24 bits or32 bits.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 161
Quoted 2 lines
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entryare freed in following cases:
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 4 lines
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greaterthan 24.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 9 lines
doc/guides/prog_guide/lpm_lib.rst +9
-0+ +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately +even though the readers might be using the tbl8 group entries. This mightresult in incorrect lookup results.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 178
Quoted 4 lines
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +RCU QSBR process is integrated for safe tbl8 group reclaimation. +Application has certain responsibilities while using this feature. +Please refer to resource reclaimation framework of :ref:`RCU library<RCU_Library>` for more details.
Quoted 55 lines
doc/guides/prog_guide/lpm_lib.rst +1
-0+ Lookup ~~~~~~lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
d682785b6..6f06c5c03100644lib/librte_lpm/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +1
-0@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
38ab512a4..30f541179100644lib/librte_lpm/rte_lpm.c +9
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -246,12 +247,85 @@ rte_lpm_free(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);rte_free (te); } +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data,unsigned int n) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +uint32_t tbl8_group_index = *(uint32_t *)data; +struct rte_lpm_tbl_entry *tbl8 = (struct rte_lpm_tbl_entry *)p; +lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 273
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+RTE_SET_USED (n); +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_lpm_rcu_config*cfg,
Quoted 23 lines
lib/librte_lpm/rte_lpm.c +23
-0+ struct rte_rcu_qsbr_dq **dq) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params = {0 }; + +if ((lpm ==NULL ) || (cfg ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->v) { + rte_errno =EEXIST ; +return 1 ; + } + +if (cfg->mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* No other things to do. */ + }else if (cfg->mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name), +"LPM_RCU_%s" , lpm->name); + params.name = rcu_dq_name; + params.size = cfg->dq_size;
Quoted 8 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 301
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +4
-0+if (params.size ==0 ) + params.size = lpm->number_tbl8s; + params.trigger_reclaim_limit = cfg->reclaim_thd; +if (params.trigger_reclaim_limit ==0 )This makes it impossible for a user to configure reclamation triggering with every call. Should we allow it?
Yes, use (reclaim_thd = 0) to trigger reclamation at each dq enqueue should be a valid case. Will remove value overriding and take it as is.
Quoted 8 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm)
Quoted 7 lines
lib/librte_lpm/rte_lpm.c +5
-0+ params.trigger_reclaim_limit = +RTE_LPM_RCU_DQ_RECLAIM_THD ; + params.max_reclaim_size = cfg->reclaim_max; +if (params.max_reclaim_size ==0 ) + params.max_reclaim_size =
RTE_LPM_RCU_DQ_RECLAIM_MAX;
Quoted 7 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 309
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+ params.esize =sizeof (uint32_t );/* tbl8 group index */ + params.free_fn = __lpm_rcu_qsbr_free_resource; + params.p = lpm->tbl8;I think it's better to pass the LPM pointer here rather than tbl8, for example, in case we decide to add some counters in the future
Use LPM pointer is more extendable.
Will change in next version.
Quoted 52 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm)
Quoted 7 lines
lib/librte_lpm/rte_lpm.c +5
-0+ params.v = cfg->v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { + RTE_LOG(ERR, LPM, + "LPM QS defer queue creationfailed\n");
lib/librte_lpm/rte_lpm.c rule_find(struct rte_lpm *lpm, uint32_t
Quoted 18 lines
lib/librte_lpm/rte_lpm.c +14
-0+return 1 ; + } +if (dq) + *dq = lpm->dq; + }else { + rte_errno =EINVAL ; +return 1 ; + } + lpm->rcu_mode = cfg->mode; + lpm->v = cfg->v; + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -394,14 +468,15 @@ rule_find(struct rte_lpm *lpm, uint32_t
ip_masked,uint8_tdepth)lib/librte_lpm/rte_lpm.c rule_find(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth)
Quoted 12 lines
lib/librte_lpm/rte_lpm.c +1
-3* Find, clean and allocate a tbl8. */ static int32_t -tbl8_alloc (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +_tbl8_alloc (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c rule_find(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth)
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8,
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) {struct rte_lpm_tbl_entry new_tbl8_entry = {@@ -427,14 +502,40 @@ tbl8_alloc(struct rte_lpm_tbl_entry *tbl8,
uint32_tnumber_tbl8s)Quoted 3 lines
lib/librte_lpm/rte_lpm.c +0
-0return -ENOSPC ; }
Quoted 10 lines
lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) · line 511
Quoted 7 lines
lib/librte_lpm/rte_lpm.c +7
-0+static int32_t +tbl8_alloc (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =_tbl8_alloc (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) {I think (
group_idx==-ENOSPC) will be safer
Will change it in next version.
Quoted 5 lines
lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, · line 513
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +2
-0+/* If there are no tbl8 groups try to reclaim one. */ + if (rte_rcu_qsbr_dq_reclaim(lpm->dq, 1, NULL, NULL, NULL)== 0)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, · line 514
Quoted 1 line
lib/librte_lpm/rte_lpm.c +1
-0+ group_idx =_tbl8_alloc (lpm);I'm not really happy with this approach.
_tbl8_alloc()produces linear scan through the memory to find a freegroup_idxand it is the slowest part ofrte_lpm_add().
Here after reclamation of some group index we need to rescan a memory again to find it. It would be great if there will be some way to return reclaimed elements. Or just to dequeue elements from dq and reclaim them manually.
I think there is little chance a rescan will be needed. If RCU QSBR defer queue trigger_reclaim_limit is configured with reasonable value, tbl8 groups will be reclaimed regularly. So defer queue won't get too long.
Return reclaimed elements makes API complex. Not sure if it is useful for other use cases. @Honnappa Nagarahalli, any idea?
Quoted 105 lines
Quoted 9 lines
lib/librte_lpm/rte_lpm.c +5
-1+ } + +return group_idx; +} + static void -tbl8_free (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start)lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +4
-3+tbl8_free(struct rte_lpm *lpm, uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, - __ATOMIC_RELAXED); +if (!lpm->v) { +/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +4
-0+ __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* Wait for quiescent state change. */ + rte_rcu_qsbr_synchronize(lpm->v,
RTE_QSBR_THRID_INVALID);lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, · line 533
Quoted 2 lines
lib/librte_lpm/rte_lpm.c +2
-0+/* Set tbl8 group invalid*/ + __atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +4
-0+ __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Push into QSBR defer queue. */ + rte_rcu_qsbr_dq_enqueue(lpm->dq, (void*)&tbl8_group_start);
lib/librte_lpm/rte_lpm.c add_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 5 lines
lib/librte_lpm/rte_lpm.c +1
-0+ } }static __rte_noinline int32_t@@ -523,7 +624,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c add_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-1if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -569,7 +670,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c delete_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +1
-1}/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm- number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);if (tbl8_group_index <0 ) {return
tbl8_group_index;lib/librte_lpm/rte_lpm.c +0
-0@@ -977,7 +1078,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,lib/librte_lpm/rte_lpm.c delete_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 9 lines
lib/librte_lpm/rte_lpm.c +1
-1*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = {@@ -993,7 +1094,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,lib/librte_lpm/rte_lpm.c delete_depth_big(struct rte_lpm *lpm, uint32_t · line 1094
Quoted 1 line
lib/librte_lpm/rte_lpm.c +0
-0__atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,Quoted 27 lines
lib/librte_lpm/rte_lpm.c +1
-1__ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +2
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #ifndef _RTE_LPM_H_@@ -20,6 +21,7 @@#include <rte_memory.h> #include <rte_common.h> #include <rte_vect.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {lib/librte_lpm/rte_lpm.h extern "C" { · line 67
Quoted 4 lines
lib/librte_lpm/rte_lpm.h +1
-0@@ -62,6 +64,17 @@ extern "C" {/** Bitmask used to indicate successful lookup */ #define RTE_LPM_LOOKUP_SUCCESS 0x01000000 +/** @internal Default threshold to trigger RCU defer queue reclaimation. */
Quoted 12 lines
lib/librte_lpm/rte_lpm.h extern "C" { · line 76
Quoted 9 lines
lib/librte_lpm/rte_lpm.h +9
-0+#define RTE_LPM_RCU_DQ_RECLAIM_THD 32 + +/** @internal Default RCU defer queue entries to reclaim in one go. */ +#define RTE_LPM_RCU_DQ_RECLAIM_MAX 16 + +/* Create defer queue for reclaim. */ +#define RTE_LPM_QSBR_MODE_DQ 0 +/* Use blocking mode reclaim. No defer queue created. */ +#define RTE_LPM_QSBR_MODE_SYNC 0x01using enums instead of defines?
Will convert to enums in next version.
Quoted 88 lines
Quoted 18 lines
lib/librte_lpm/rte_lpm.h +9
-0+#if RTE_BYTE_ORDER ==RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ __extension__@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ + +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +uint32_t rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +}; + +/** LPM RCU QSBR configuration structure. */ lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 163
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +10
-0+struct rte_lpm_rcu_config { +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +/* Mode of RCU QSBR. RTE_LPM_QSBR_MODE_xxx +* '0' for default: create defer queue for reclaim. +*/ +uint32_t mode; +/* RCU defer queue size. default: lpm->number_tbl8s. */ +uint32_t dq_size; +uint32_t reclaim_thd;/* Threshold to trigger auto reclaim. +* default:
RTE_LPM_RCU_DQ_RECLAIM_TRHD.lib/librte_lpm/rte_lpm.h struct rte_lpm {
Quoted 3 lines
lib/librte_lpm/rte_lpm.h +3
-0+*/ +uint32_t reclaim_max;/* Max entries to reclaim in one go. +* default:
RTE_LPM_RCU_DQ_RECLAIM_MAX.Quoted 20 lines
lib/librte_lpm/rte_lpm.h +13
-0+*/ };/** @@ -179,6 +214,30 @@ rte_lpm_find_existing(const char *name);void rte_lpm_free (struct rte_lpm *lpm); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param cfg +* RCU QSBR configuration +* @param dq +* handler of created RCU QSBR defer queue lib/librte_lpm/rte_lpm.h rte_lpm_find_existing(const char *name); · line 238
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +10
-0+* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct
rte_lpm_rcu_config*cfg,Review
Quoted 6 lines
lib/librte_lpm/rte_lpm.h +2
-0+ struct rte_rcu_qsbr_dq **dq); +/** * Add a rule to the LPM table. * (quoted diff) +0
-0b/lib/librte_lpm/rte_lpm_version.map
lib/librte_lpm/rte_lpm_version.map DPDK_20.0 { · line 29
Quoted 13 lines
index
500f58b80..bfccd7eac100644lib/librte_lpm/rte_lpm_version.map +6
-0@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};--
Regards,
Vladimir
Re: [dpdk-dev] [PATCH v4 1/3] lib/lpm: integrate RCU QSBR
<snip>
Quoted 268 lines
Quoted 51 lines · Ruifeng Wang
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the
Quoted 25 lines
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 ++++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 123 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 ++++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 211 insertions(+), 12 deletions(-)doc/guides/prog_guide/lpm_lib.rst +0
-0@@ -145,6 +145,38 @@ depending on whether we need to move to thenext table or not.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 9 lines
doc/guides/prog_guide/lpm_lib.rst +6
-0Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. +Replacement rule is an existing rule that has the longest prefix +match with the rule to bedeleted, but has smaller depth.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 156
Quoted 3 lines
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +If a replacement rule is found, target tbl24 and tbl8 entries are +updated to have the same depth and next hop value with thereplacement rule.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 159
Quoted 3 lines
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +If no replacement rule can be found, target tbl24 and tbl8 entries +will becleared.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 162
Quoted 3 lines
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +Prefix expansion is performed if the rule's depth is not exactly 24 +bits or32 bits.
Quoted 8 lines
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 165
Quoted 3 lines
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +After deleting a rule, a group of tbl8s that belongs to the same +tbl24 entryare freed in following cases:
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 4 lines
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth nogreater
Quoted 7 lines
than 24.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 6 lines
doc/guides/prog_guide/lpm_lib.rst +6
-0+ +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescentstate.
Quoted 95 lines
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the · line 179
Quoted 4 lines
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +When the LPM is not using RCU, tbl8 group can be freed immediately +even though the readers might be using the tbl8 group entries. This +mightresult in incorrect lookup results.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the
Quoted 5 lines
doc/guides/prog_guide/lpm_lib.rst +5
-0+ +RCU QSBR process is integrated for safe tbl8 group reclaimation. +Application has certain responsibilities while using this feature. +Please refer to resource reclaimation framework of :ref:`RCU +library<RCU_Library>` for more details.
Quoted 55 lines
doc/guides/prog_guide/lpm_lib.rst +1
-0+ Lookup ~~~~~~lib/librte_lpm/Makefile index renamed from lib/librte_lpm/Makefile +0
-0
d682785b6..6f06c5c03100644lib/librte_lpm/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +1
-0@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c +0
-0index
38ab512a4..30f541179100644lib/librte_lpm/rte_lpm.c +8
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -246,12 +247,85 @@ rte_lpm_free(struct rte_lpm *lpm)rte_mcfg_tailq_write_unlock (); +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);rte_free (te); } +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data,unsigned int n) { +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 };uint32_t +tbl8_group_index = *(uint32_t *)data;struct rte_lpm_tbl_entry +*tbl8 = (struct rte_lpm_tbl_entry *)p;lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 272
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +10
-0+ +RTE_SET_USED (n); +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[tbl8_group_index], &zero_tbl8_entry, +__ATOMIC_RELAXED); } + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_lpm_rcu_config*cfg,
Quoted 18 lines
lib/librte_lpm/rte_lpm.c +18
-0+struct rte_rcu_qsbr_dq **dq) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params = {0 }; + +if ((lpm ==NULL ) || (cfg ==NULL )) { rte_errno =EINVAL ;return 1 ; +} + +if (lpm->v) { +rte_errno =EEXIST ; +return 1 ; +} + +if (cfg->mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* No other things to do. */ +}else if (cfg->mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Init QSBR defer queue. */ +snprintf(rcu_dq_name, sizeof(rcu_dq_name), "LPM_RCU_%s",Quoted 8 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 294
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +4
-0+lpm->name); params.name = rcu_dq_name; params.size = cfg->dq_size; +if (params.size ==0 ) params.size = lpm->number_tbl8s; +params.trigger_reclaim_limit = cfg->reclaim_thd;if +(params.trigger_reclaim_limit ==0 )This makes it impossible for a user to configure reclamation triggering with every call. Should we allow it?
Yes, use (
reclaim_thd= 0) to trigger reclamation at each dq enqueue should be a valid case.
Will remove value overriding and take it as is.Quoted 7 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm)
Quoted 6 lines
lib/librte_lpm/rte_lpm.c +4
-0+params.trigger_reclaim_limit = +RTE_LPM_RCU_DQ_RECLAIM_THD ; +params.max_reclaim_size = cfg->reclaim_max;if +(params.max_reclaim_size == 0) params.max_reclaim_size =
RTE_LPM_RCU_DQ_RECLAIM_MAX;Quoted 7 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 301
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+params.esize =sizeof (uint32_t );/* tbl8 group index */ +params.free_fn = __lpm_rcu_qsbr_free_resource; params.p = +lpm->tbl8;I think it's better to pass the LPM pointer here rather than tbl8, for example, in case we decide to add some counters in the future
Use LPM pointer is more extendable.
Will change in next version.Quoted 52 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm)
Quoted 7 lines
lib/librte_lpm/rte_lpm.c +5
-0+params.v = cfg->v; +lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG(ERR, LPM, +"LPM QS defer queue creationfailed\n");
lib/librte_lpm/rte_lpm.c rule_find(struct rte_lpm *lpm, uint32_t
Quoted 18 lines
lib/librte_lpm/rte_lpm.c +14
-0+return 1 ; +} +if (dq) +*dq = lpm->dq; +}else { +rte_errno =EINVAL ; +return 1 ; +} +lpm->rcu_mode = cfg->mode; +lpm->v = cfg->v; + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -394,14 +468,15 @@ rule_find(struct rte_lpm *lpm, uint32_t
ip_masked,uint8_tdepth)lib/librte_lpm/rte_lpm.c rule_find(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth)
Quoted 12 lines
lib/librte_lpm/rte_lpm.c +1
-3* Find, clean and allocate a tbl8. */ static int32_t -tbl8_alloc (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +_tbl8_alloc (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { -tbl8_entry = &tbl8[group_idx *
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];lib/librte_lpm/rte_lpm.c rule_find(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth)
Quoted 3 lines
lib/librte_lpm/rte_lpm.c +3
-0+for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { +tbl8_entry = &lpm->tbl8[group_idx * +
RTE_LPM_TBL8_GROUP_NUM_ENTRIES];Quoted 4 lines
lib/librte_lpm/rte_lpm.c +0
-0/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = { @@ -427,14 +502,40 @@tbl8_alloc(struct
rte_lpm_tbl_entry*tbl8,
uint32_tnumber_tbl8s)Quoted 3 lines
return -ENOSPC; }Quoted 10 lines
lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s) · line 511
Quoted 7 lines
lib/librte_lpm/rte_lpm.c +7
-0+static int32_t +tbl8_alloc (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + +group_idx =_tbl8_alloc (lpm); +if ((group_idx <0 ) && (lpm->dq !=NULL )) {I think (
group_idx==-ENOSPC) will be saferWill change it in next version.
Quoted 8 lines
lib/librte_lpm/rte_lpm.c
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +2
-0+/* If there are no tbl8 groups try to reclaim one. */ if +(rte_rcu_qsbr_dq_reclaim(lpm->dq, 1, NULL, NULL, NULL)== 0)
Quoted 1 line
+group_idx =
_tbl8_alloc(lpm);Quoted 7 lines
I'm not really happy with this approach.
_tbl8_alloc()produces linear scan through the memory to find a freegroup_idxand it is the slowest part ofrte_lpm_add().
Here after reclamation of some group index we need to rescan a memory again to find it. It would be great if there will be some way to return reclaimed elements. Or just to dequeue elements from dq and reclaim them manually.
Quoted 7 lines
I think there is little chance a rescan will be needed. If RCU QSBR defer queue
trigger_reclaim_limitis configured with reasonable value, tbl8 groups will be reclaimed regularly. So defer queue won't get too long.Return reclaimed elements makes API complex. Not sure if it is useful for other use cases.
@Honnappa Nagarahalli, any idea?
I think it is a problem that should be solved in LPM. I had hacked up the allocation scheme to use a ring when I was doing the performance testing. I can send that out if required.
Quoted 205 lines
Quoted 96 lines
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +5
-1+} + +return group_idx; +} + static void -tbl8_free(struct rte_lpm_tbl_entry *tbl8, uint32_t
tbl8_group_start)lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 10 lines
lib/librte_lpm/rte_lpm.c +4
-3+tbl8_free(struct rte_lpm *lpm, uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, -__ATOMIC_RELAXED); +if (!lpm->v) { +/* Set tbl8 group invalid*/ +__atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +4
-0+__ATOMIC_RELAXED); +}else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* Wait for quiescent state change. */ +rte_rcu_qsbr_synchronize(lpm->v,
RTE_QSBR_THRID_INVALID);lib/librte_lpm/rte_lpm.c
Quoted 2 lines
lib/librte_lpm/rte_lpm.c +2
-0+/* Set tbl8 group invalid*/ +__atomic_store(&lpm->tbl8[tbl8_group_start],
&zero_tbl8_entry,lib/librte_lpm/rte_lpm.c tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)
Quoted 4 lines
lib/librte_lpm/rte_lpm.c +4
-0+__ATOMIC_RELAXED); +}else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue(lpm->dq, (void*)&tbl8_group_start);
lib/librte_lpm/rte_lpm.c add_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 5 lines
lib/librte_lpm/rte_lpm.c +1
-0+} }static __rte_noinline int32_t@@ -523,7 +624,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c add_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 9 lines
lib/librte_lpm/rte_lpm.c +1
-0if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ -tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm- number_tbl8s); +tbl8_group_index =tbl8_alloc (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -569,7 +670,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,uint8_tdepth,lib/librte_lpm/rte_lpm.c delete_depth_big(struct rte_lpm *lpm, uint32_t
Quoted 9 lines
lib/librte_lpm/rte_lpm.c +1
-1}/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ -tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm- number_tbl8s); +tbl8_group_index =tbl8_alloc (lpm);if (tbl8_group_index <0 ) {return tbl8_group_index;@@ -977,7 +1078,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t
ip_masked,Quoted 9 lines
lib/librte_lpm/rte_lpm.c +1
-1*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = { @@ -993,7 +1094,7 @@delete_depth_big(struct
rte_lpm*lpm,uint32_t
ip_masked,Quoted 1 line
__atomic_store(&lpm->tbl24[tbl24_index],
&new_tbl24_entry,Quoted 24 lines
__ATOMIC_RELAXED); __atomic_thread_fence(__ATOMIC_RELEASE);lib/librte_lpm/rte_lpm.c +1
-1-tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +2
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #ifndef _RTE_LPM_H_@@ -20,6 +21,7 @@#include <rte_memory.h> #include <rte_common.h> #include <rte_vect.h> +#include <rte_rcu_qsbr.h> Quoted 7 lines
lib/librte_lpm/rte_lpm.h extern "C" { · line 67
Quoted 7 lines
lib/librte_lpm/rte_lpm.h +1
-0#ifdef __cplusplusextern "C" {@@ -62,6 +64,17 @@ extern "C" {/** Bitmask used to indicate successful lookup */ #define RTE_LPM_LOOKUP_SUCCESS 0x01000000 +/** @internal Default threshold to trigger RCU defer queue reclaimation.
Quoted 7 lines
*/
lib/librte_lpm/rte_lpm.h extern "C" { · line 73
Quoted 6 lines
lib/librte_lpm/rte_lpm.h +6
-0+#define RTE_LPM_RCU_DQ_RECLAIM_THD32 + +/** @internal Default RCU defer queue entries to reclaim in one go. +*/ #define RTE_LPM_RCU_DQ_RECLAIM_MAX16 + +/* Create defer queue for reclaim. */ #define
RTE_LPM_QSBR_MODE_DQ0Quoted 5 lines
lib/librte_lpm/rte_lpm.h extern "C" { · line 75
Quoted 2 lines
lib/librte_lpm/rte_lpm.h +2
-0+/* Use blocking mode reclaim. No defer queue created. */ #define +RTE_LPM_QSBR_MODE_SYNC0x01using enums instead of defines?
Will convert to enums in next version.
Quoted 86 lines
Quoted 16 lines
lib/librte_lpm/rte_lpm.h +7
-0+#if RTE_BYTE_ORDER ==RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ __extension__@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ + +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ uint32_t +rcu_mode;/* Blocking, defer queue. */ struct rte_rcu_qsbr_dq *dq;/* +RCU QSBR defer queue. */ }; +lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 161
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +10
-0+/** LPM RCU QSBR configuration structure. */ struct +rte_lpm_rcu_config {struct rte_rcu_qsbr *v;/* RCU QSBR variable. +*/ +/* Mode of RCU QSBR. RTE_LPM_QSBR_MODE_xxx +* '0' for default: create defer queue for reclaim. +*/ +uint32_t mode; +/* RCU defer queue size. default: lpm->number_tbl8s. */ uint32_t +dq_size;uint32_t reclaim_thd;/* Threshold to trigger auto reclaim. +* default:
RTE_LPM_RCU_DQ_RECLAIM_TRHD.lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 164
Quoted 3 lines
lib/librte_lpm/rte_lpm.h +3
-0+*/ +uint32_t reclaim_max;/* Max entries to reclaim in one go. +* default:
RTE_LPM_RCU_DQ_RECLAIM_MAX.Quoted 20 lines
lib/librte_lpm/rte_lpm.h +13
-0+*/ };/** @@ -179,6 +214,30 @@ rte_lpm_find_existing(const char *name);void rte_lpm_free (struct rte_lpm *lpm); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param cfg +* RCU QSBR configuration +* @param dq +* handler of created RCU QSBR defer queue lib/librte_lpm/rte_lpm.h rte_lpm_find_existing(const char *name); · line 238
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +10
-0+* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct
rte_lpm_rcu_config*cfg,Review
Quoted 6 lines
lib/librte_lpm/rte_lpm.h +2
-0+struct rte_rcu_qsbr_dq **dq); +/** * Add a rule to the LPM table. * (quoted diff) +0
-0b/lib/librte_lpm/rte_lpm_version.map
lib/librte_lpm/rte_lpm_version.map DPDK_20.0 { · line 29
Quoted 13 lines
index
500f58b80..bfccd7eac100644lib/librte_lpm/rte_lpm_version.map +6
-0@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { +global: + +rte_lpm_rcu_qsbr_add; +};--
Regards,
Vladimir
PATCH v4 2/3 test/lpm: add LPM RCU integration functional tests
In reply to Ruifeng WangAdd positive and negative tests for API rte_lpm_rcu_qsbr_add. Also test LPM library behavior when RCU QSBR is enabled.
--- app/test/test_lpm.c | 293 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 292 insertions(+), 1 deletion(-)
app/test/test_lpm.c +292 -1
@@ -8,6 +8,7 @@#include <rte_ip.h> #include <rte_lpm.h> +#include <rte_malloc.h> #include "test.h" #include "test_xmmt_ops.h"
@@ -40,6 +41,9 @@ static int32_t test15(void);static int32_t test16 (void );static int32_t test17 (void );static int32_t test18 (void ); +static int32_t test19 (void ); +static int32_t test20 (void ); +static int32_t test21 (void );rte_lpm_test tests[] = {/* Test Cases */
@@ -61,7 +65,10 @@ rte_lpm_test tests[] = { test15, test16, test17, - test18 + test18, + test19, + test20, + test21 };#define MAX_DEPTH 32
@@ -1265,6 +1272,290 @@ test18(void)return PASS ; } +/* +* rte_lpm_rcu_qsbr_add positive and negative tests. +* - Add RCU QSBR variable to LPM +* - Add another RCU QSBR variable to LPM +* - Check LPM attached RCU QSBR variable and FIFO queue +*/ +int32_t +test19 (void ) +{ +struct rte_lpm *lpm =NULL ; +struct rte_lpm_config config; +size_t sz; +struct rte_rcu_qsbr *qsv; +struct rte_rcu_qsbr *qsv2; +int32_t status; +struct rte_lpm_rcu_config rcu_cfg = {0 }; + + config.max_rules =MAX_RULES ; + config.number_tbl8s =NUMBER_TBL8S ; + config.flags =0 ; + + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Create RCU QSBR variable */ + sz =rte_rcu_qsbr_get_memsize (RTE_MAX_LCORE ); + qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (qsv !=NULL ); + + status =rte_rcu_qsbr_init (qsv,RTE_MAX_LCORE ); +TEST_LPM_ASSERT (status ==0 ); + + rcu_cfg.v = qsv; +/* Invalid QSBR mode */ + rcu_cfg.mode =2 ; + status =rte_lpm_rcu_qsbr_add (lpm, &rcu_cfg,NULL ); +TEST_LPM_ASSERT (status !=0 ); + + rcu_cfg.mode =RTE_LPM_QSBR_MODE_DQ ; +/* Attach RCU QSBR to LPM table */ + status =rte_lpm_rcu_qsbr_add (lpm, &rcu_cfg,NULL ); +TEST_LPM_ASSERT (status ==0 ); + +/* Create and attach another RCU QSBR to LPM table */ + qsv2 = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (qsv2 !=NULL ); + + rcu_cfg.v = qsv2; + rcu_cfg.mode =RTE_LPM_QSBR_MODE_SYNC ; + status =rte_lpm_rcu_qsbr_add (lpm, &rcu_cfg,NULL ); +TEST_LPM_ASSERT (status !=0 ); + +TEST_LPM_ASSERT (lpm->dq !=NULL ); + +rte_lpm_free (lpm); +rte_free (qsv); +rte_free (qsv2); + +return PASS ; +} + +/* +* rte_lpm_rcu_qsbr_add DQ mode functional test. +* Reader and writer are in the same thread in this test. +* - Create LPM which supports 1 tbl8 group at max +* - Add RCU QSBR variable to LPM +* - Add a rule with depth=28 (> 24) +* - Register a reader thread (not a real thread) +* - Reader lookup existing rule +* - Writer delete the rule +* - Reader lookup the rule +* - Writer re-add the rule (no available tbl8 group) +* - Reader report quiescent state and unregister +* - Writer re-add the rule +* - Reader lookup the rule +*/ +int32_t +test20 (void ) +{ +struct rte_lpm *lpm =NULL ; +struct rte_lpm_config config; +size_t sz; +struct rte_rcu_qsbr *qsv; +int32_t status; +uint32_t ip, next_hop, next_hop_return; +uint8_t depth; +struct rte_lpm_rcu_config rcu_cfg = {0 }; + + config.max_rules =MAX_RULES ; + config.number_tbl8s =1 ; + config.flags =0 ; + + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Create RCU QSBR variable */ + sz =rte_rcu_qsbr_get_memsize (1 ); + qsv = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (qsv !=NULL ); + + status =rte_rcu_qsbr_init (qsv,1 ); +TEST_LPM_ASSERT (status ==0 ); + + rcu_cfg.v = qsv; + rcu_cfg.mode =RTE_LPM_QSBR_MODE_DQ ; +/* Attach RCU QSBR to LPM table */ + status =rte_lpm_rcu_qsbr_add (lpm, &rcu_cfg,NULL ); +TEST_LPM_ASSERT (status ==0 ); + + ip =RTE_IPV4 (192 ,18 ,100 ,100 ); + depth =28 ; + next_hop =1 ; + status =rte_lpm_add (lpm, ip, depth, next_hop); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (lpm->tbl24[ip>>8 ].valid_group); + +/* Register pseudo reader */ + status =rte_rcu_qsbr_thread_register (qsv,0 ); +TEST_LPM_ASSERT (status ==0 ); +rte_rcu_qsbr_thread_online (qsv,0 ); + + status =rte_lpm_lookup (lpm, ip, &next_hop_return); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (next_hop_return == next_hop); + +/* Writer update */ + status =rte_lpm_delete (lpm, ip, depth); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (!lpm->tbl24[ip>>8 ].valid); + + status =rte_lpm_lookup (lpm, ip, &next_hop_return); +TEST_LPM_ASSERT (status !=0 ); + + status =rte_lpm_add (lpm, ip, depth, next_hop); +TEST_LPM_ASSERT (status !=0 ); + +/* Reader quiescent */ +rte_rcu_qsbr_quiescent (qsv,0 ); + + status =rte_lpm_add (lpm, ip, depth, next_hop); +TEST_LPM_ASSERT (status ==0 ); + +rte_rcu_qsbr_thread_offline (qsv,0 ); + status =rte_rcu_qsbr_thread_unregister (qsv,0 ); +TEST_LPM_ASSERT (status ==0 ); + + status =rte_lpm_lookup (lpm, ip, &next_hop_return); +TEST_LPM_ASSERT (status ==0 ); +TEST_LPM_ASSERT (next_hop_return == next_hop); + +rte_lpm_free (lpm); +rte_free (qsv); + +return PASS ; +} + +static struct rte_lpm *g_lpm; +static struct rte_rcu_qsbr *g_v; +static uint32_t g_ip =RTE_IPV4 (192 ,18 ,100 ,100 ); +static volatile uint8_t writer_done; +/* Report quiescent state interval every 1024 lookups. Larger critical +* sections in reader will result in writer polling multiple times. +*/ +#define QSBR_REPORTING_INTERVAL 1024 +#define WRITER_ITERATIONS 512 + +/* +* Reader thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_reader (void *arg) +{ +int i; +uint32_t next_hop_return =0 ; + +RTE_SET_USED (arg); +/* Register this thread to report quiescent state */ +rte_rcu_qsbr_thread_register (g_v,0 ); +rte_rcu_qsbr_thread_online (g_v,0 ); + +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (g_lpm, g_ip, &next_hop_return); + +/* Update quiescent state */ +rte_rcu_qsbr_quiescent (g_v,0 ); + }while (!writer_done); + +rte_rcu_qsbr_thread_offline (g_v,0 ); +rte_rcu_qsbr_thread_unregister (g_v,0 ); + +return 0 ; +} + +/* +* rte_lpm_rcu_qsbr_add sync mode functional test. +* 1 Reader and 1 writer. They cannot be in the same thread in this test. +* - Create LPM which supports 1 tbl8 group at max +* - Add RCU QSBR variable with sync mode to LPM +* - Register a reader thread. Reader keeps looking up a specific rule. +* - Writer keeps adding and deleting a specific rule with depth=28 (> 24) +*/ +int32_t +test21 (void ) +{ +struct rte_lpm_config config; +size_t sz; +int32_t status; +uint32_t i, next_hop; +uint8_t depth; +struct rte_lpm_rcu_config rcu_cfg = {0 }; + +if (rte_lcore_count () <2 ) { +printf ("Not enough cores for %s, expecting at least 2\n" , + __func__); +return TEST_SKIPPED ; + } + + config.max_rules =MAX_RULES ; + config.number_tbl8s =1 ; + config.flags =0 ; + + g_lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (g_lpm !=NULL ); + +/* Create RCU QSBR variable */ + sz =rte_rcu_qsbr_get_memsize (1 ); + g_v = (struct rte_rcu_qsbr *)rte_zmalloc_socket (NULL , sz, +RTE_CACHE_LINE_SIZE ,SOCKET_ID_ANY ); +TEST_LPM_ASSERT (g_v !=NULL ); + + status =rte_rcu_qsbr_init (g_v,1 ); +TEST_LPM_ASSERT (status ==0 ); + + rcu_cfg.v = g_v; + rcu_cfg.mode =RTE_LPM_QSBR_MODE_SYNC ; +/* Attach RCU QSBR to LPM table */ + status =rte_lpm_rcu_qsbr_add (g_lpm, &rcu_cfg,NULL ); +TEST_LPM_ASSERT (status ==0 ); + + writer_done =0 ; +/* Launch reader thread */ +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , +rte_get_next_lcore (-1 ,1 ,0 )); + + depth =28 ; + next_hop =1 ; + status =rte_lpm_add (g_lpm, g_ip, depth, next_hop); +if (status !=0 ) { +printf ("%s: Failed to add rule\n" , __func__); + goto error; + } + +/* Writer update */ +for (i =0 ; i <WRITER_ITERATIONS ; i++) { + status =rte_lpm_delete (g_lpm, g_ip, depth); +if (status !=0 ) { +printf ("%s: Failed to delete rule at iteration %d\n" , + __func__, i); + goto error; + } + + status =rte_lpm_add (g_lpm, g_ip, depth, next_hop); +if (status !=0 ) { +printf ("%s: Failed to add rule at iteration %d\n" , + __func__, i); + goto error; + } + } + +error: + writer_done =1 ; +/* Wait until reader exited. */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (g_lpm); +rte_free (g_v); + +return (status ==0 ) ?PASS :-1 ; +} +/* * Do all unit tests. */
Signature · 1 line
2.17.1
PATCH v4 3/3 test/lpm: add RCU integration performance tests
In reply to Ruifeng WangFrom: Honnappa Nagarahalli <[email protected]>
Add performance tests for RCU integration. The performance difference with and without RCU integration is very small (~1% to ~2%) on both Arm and x86 platforms.
--- app/test/test_lpm_perf.c | 492 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 489 insertions(+), 3 deletions(-)
app/test/test_lpm_perf.c +489 -3
@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <stdio.h>
@@ -10,12 +11,27 @@#include <rte_cycles.h> #include <rte_random.h> #include <rte_branch_prediction.h> +#include <rte_malloc.h> #include <rte_ip.h> #include <rte_lpm.h> #include "test.h" #include "test_xmmt_ops.h" +struct rte_lpm *lpm; +static struct rte_rcu_qsbr *rv; +static volatile uint8_t writer_done; +static volatile uint32_t thr_id; +static uint64_t gwrite_cycles; +static uint64_t gwrites; +/* LPM APIs are not thread safe, use mutex to provide thread safety */ +static pthread_mutex_t lpm_mutex =PTHREAD_MUTEX_INITIALIZER ; + +/* Report quiescent state interval every 1024 lookups. Larger critical +* sections in reader will result in writer polling multiple times. +*/ +#define QSBR_REPORTING_INTERVAL 1024 +#define TEST_LPM_ASSERT (cond) do { \ if (!(cond)) { \ printf("Error at line %d: \n", __LINE__); \
@@ -24,6 +40,7 @@ } while(0)#define ITERATIONS (1 << 10) +#define RCU_ITERATIONS 10#define BATCH_SIZE (1 << 12)#define BULK_SIZE 32
@@ -35,9 +52,13 @@ struct route_rule { };static struct route_rule large_route_table[MAX_RULE_NUM ]; +/* Route table for routes with depth > 24 */ +struct route_rule large_ldepth_route_table[MAX_RULE_NUM ];static uint32_t num_route_entries; +static uint32_t num_ldepth_route_entries;#define NUM_ROUTE_ENTRIES num_route_entries +#define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entriesenum {IP_CLASS_A ,
@@ -191,7 +212,7 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)uint32_t ip_head_mask;uint32_t rule_num;uint32_t k; -struct route_rule *ptr_rule; +struct route_rule *ptr_rule, *ptr_ldepth_rule;if (ip_class ==IP_CLASS_A ) {/* IP Address class A */ fixed_bit_num =IP_HEAD_BIT_NUM_A ;
@@ -236,10 +257,20 @@ static void generate_random_rule_prefix(uint32_t ip_class, uint8_t depth)*/ start =lrand48 () & mask; ptr_rule = &large_route_table[num_route_entries]; + ptr_ldepth_rule = &large_ldepth_route_table[num_ldepth_route_entries];for (k =0 ; k < rule_num; k++) { ptr_rule->ip = (start << (RTE_LPM_MAX_DEPTH - depth)) | ip_head_mask; ptr_rule->depth = depth; +/* If the depth of the route is more than 24, store it +* in another table as well. +*/ +if (depth >24 ) { + ptr_ldepth_rule->ip = ptr_rule->ip; + ptr_ldepth_rule->depth = ptr_rule->depth; + ptr_ldepth_rule++; + num_ldepth_route_entries++; + } ptr_rule++; start = (start + step) & mask; }
@@ -273,6 +304,7 @@ static void generate_large_route_rule_table(void)uint8_t depth; num_route_entries =0 ; + num_ldepth_route_entries =0 ;memset (large_route_table,0 ,sizeof (large_route_table));for (ip_class =IP_CLASS_A ; ip_class <=IP_CLASS_C ; ip_class++) {
@@ -316,10 +348,460 @@ print_route_distribution(const struct route_rule *table, uint32_t n)printf ("\n" ); } +/* Check condition and return an error if true. */ +static uint16_t enabled_core_ids[RTE_MAX_LCORE ]; +static unsigned int num_cores; + +/* Simple way to allocate thread ids in 0 to RTE_MAX_LCORE space */ +static inline uint32_t +alloc_thread_id (void ) +{ +uint32_t tmp_thr_id; + + tmp_thr_id =__atomic_fetch_add (&thr_id,1 , __ATOMIC_RELAXED); +if (tmp_thr_id >=RTE_MAX_LCORE ) +printf ("Invalid thread id %u\n" , tmp_thr_id); + +return tmp_thr_id; +} + +/* +* Reader thread using rte_lpm data structure without RCU. +*/ +static int +test_lpm_reader (void *arg) +{ +int i; +uint32_t ip_batch[QSBR_REPORTING_INTERVAL ]; +uint32_t next_hop_return =0 ; + +RTE_SET_USED (arg); +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) + ip_batch[i] =rte_rand (); + +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (lpm, ip_batch[i], &next_hop_return); + + }while (!writer_done); + +return 0 ; +} + +/* +* Reader thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_reader (void *arg) +{ +int i; +uint32_t thread_id =alloc_thread_id (); +uint32_t ip_batch[QSBR_REPORTING_INTERVAL ]; +uint32_t next_hop_return =0 ; + +RTE_SET_USED (arg); +/* Register this thread to report quiescent state */ +rte_rcu_qsbr_thread_register (rv, thread_id); +rte_rcu_qsbr_thread_online (rv, thread_id); + +do { +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) + ip_batch[i] =rte_rand (); + +for (i =0 ; i <QSBR_REPORTING_INTERVAL ; i++) +rte_lpm_lookup (lpm, ip_batch[i], &next_hop_return); + +/* Update quiescent state */ +rte_rcu_qsbr_quiescent (rv, thread_id); + }while (!writer_done); + +rte_rcu_qsbr_thread_offline (rv, thread_id); +rte_rcu_qsbr_thread_unregister (rv, thread_id); + +return 0 ; +} + +/* +* Writer thread using rte_lpm data structure with RCU. +*/ +static int +test_lpm_rcu_qsbr_writer (void *arg) +{ +unsigned int i, j, si, ei; +uint64_t begin, total_cycles; +uint8_t core_id = (uint8_t )((uintptr_t )arg); +uint32_t next_hop_add =0xAA ; + +RTE_SET_USED (arg); +/* 2 writer threads are used */ +if (core_id %2 ==0 ) { + si =0 ; + ei =NUM_LDEPTH_ROUTE_ENTRIES /2 ; + }else { + si =NUM_LDEPTH_ROUTE_ENTRIES /2 ; + ei =NUM_LDEPTH_ROUTE_ENTRIES ; + } + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j = si; j < ei; j++) { +pthread_mutex_lock (&lpm_mutex); +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + } +pthread_mutex_unlock (&lpm_mutex); + } + +/* Delete all the entries */ +for (j = si; j < ei; j++) { +pthread_mutex_lock (&lpm_mutex); +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + } +pthread_mutex_unlock (&lpm_mutex); + } + } + + total_cycles =rte_rdtsc_precise () - begin; + +__atomic_fetch_add (&gwrite_cycles, total_cycles, __ATOMIC_RELAXED); +__atomic_fetch_add (&gwrites, +2 *NUM_LDEPTH_ROUTE_ENTRIES *RCU_ITERATIONS , + __ATOMIC_RELAXED); + +return 0 ; +} + +/* +* Functional test: +* 2 writers, rest are readers +*/ +static int +test_lpm_rcu_perf_multi_writer (void ) +{ +struct rte_lpm_config config; +size_t sz; +unsigned int i; +uint16_t core_id; +struct rte_lpm_rcu_config rcu_cfg = {0 }; + +if (rte_lcore_count () <3 ) { +printf ("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n" ); +return TEST_SKIPPED ; + } + + num_cores =0 ; +RTE_LCORE_FOREACH_SLAVE (core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + +printf ("\nPerf test: 2 writers, %d readers, RCU integration enabled\n" , + num_cores -2 ); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Init RCU variable */ + sz =rte_rcu_qsbr_get_memsize (num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc ("rcu0" , sz, +RTE_CACHE_LINE_SIZE ); +rte_rcu_qsbr_init (rv, num_cores); + + rcu_cfg.v = rv; +/* Assign the RCU variable to LPM */ +if (rte_lpm_rcu_qsbr_add (lpm, &rcu_cfg,NULL ) !=0 ) { +printf ("RCU variable assignment failed\n" ); + goto error; + } + + writer_done =0 ; +__atomic_store_n (&gwrite_cycles,0 , __ATOMIC_RELAXED); +__atomic_store_n (&gwrites,0 , __ATOMIC_RELAXED); + +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =2 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , + enabled_core_ids[i]); + +/* Launch writer threads */ +for (i =0 ; i <2 ; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t )i, + enabled_core_ids[i]); + +/* Wait for writer threads */ +for (i =0 ; i <2 ; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +printf ("Total LPM Adds: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %" PRIu64" cycles\n" , +__atomic_load_n (&gwrite_cycles, __ATOMIC_RELAXED) / +__atomic_load_n (&gwrites, __ATOMIC_RELAXED) + ); + +/* Wait and check return value from reader threads */ + writer_done =1 ; +for (i =2 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); +rte_free (rv); + lpm =NULL ; + rv =NULL ; + +/* Test without RCU integration */ +printf ("\nPerf test: 2 writers, %d readers, RCU integration disabled\n" , + num_cores -2 ); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + + writer_done =0 ; +__atomic_store_n (&gwrite_cycles,0 , __ATOMIC_RELAXED); +__atomic_store_n (&gwrites,0 , __ATOMIC_RELAXED); +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =2 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_reader,NULL , + enabled_core_ids[i]); + +/* Launch writer threads */ +for (i =0 ; i <2 ; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_writer, + (void *)(uintptr_t )i, + enabled_core_ids[i]); + +/* Wait for writer threads */ +for (i =0 ; i <2 ; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +printf ("Total LPM Adds: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +2 *ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %" PRIu64" cycles\n" , +__atomic_load_n (&gwrite_cycles, __ATOMIC_RELAXED) / +__atomic_load_n (&gwrites, __ATOMIC_RELAXED) + ); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =2 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); + +return 0 ; + +error: + writer_done =1 ; +/* Wait until all readers have exited */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (lpm); +rte_free (rv); + +return -1 ; +} + +/* +* Functional test: +* Single writer, rest are readers +*/ +static int +test_lpm_rcu_perf (void ) +{ +struct rte_lpm_config config; +uint64_t begin, total_cycles; +size_t sz; +unsigned int i, j; +uint16_t core_id; +uint32_t next_hop_add =0xAA ; +struct rte_lpm_rcu_config rcu_cfg = {0 }; + +if (rte_lcore_count () <2 ) { +printf ("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n" ); +return TEST_SKIPPED ; + } + + num_cores =0 ; +RTE_LCORE_FOREACH_SLAVE (core_id) { + enabled_core_ids[num_cores] = core_id; + num_cores++; + } + +printf ("\nPerf test: 1 writer, %d readers, RCU integration enabled\n" , + num_cores); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + +/* Init RCU variable */ + sz =rte_rcu_qsbr_get_memsize (num_cores); + rv = (struct rte_rcu_qsbr *)rte_zmalloc ("rcu0" , sz, +RTE_CACHE_LINE_SIZE ); +rte_rcu_qsbr_init (rv, num_cores); + + rcu_cfg.v = rv; +/* Assign the RCU variable to LPM */ +if (rte_lpm_rcu_qsbr_add (lpm, &rcu_cfg,NULL ) !=0 ) { +printf ("RCU variable assignment failed\n" ); + goto error; + } + + writer_done =0 ; +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =0 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_rcu_qsbr_reader,NULL , + enabled_core_ids[i]); + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + goto error; + } + +/* Delete all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + goto error; + } + } + total_cycles =rte_rdtsc_precise () - begin; + +printf ("Total LPM Adds: %d\n" ,ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %g cycles\n" , + (double )total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *ITERATIONS )); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =0 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) + goto error; + +rte_lpm_free (lpm); +rte_free (rv); + lpm =NULL ; + rv =NULL ; + +/* Test without RCU integration */ +printf ("\nPerf test: 1 writer, %d readers, RCU integration disabled\n" , + num_cores); + +/* Create LPM table */ + config.max_rules =NUM_LDEPTH_ROUTE_ENTRIES ; + config.number_tbl8s =NUM_LDEPTH_ROUTE_ENTRIES ; + config.flags =0 ; + lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config); +TEST_LPM_ASSERT (lpm !=NULL ); + + writer_done =0 ; +__atomic_store_n (&thr_id,0 , __ATOMIC_SEQ_CST); + +/* Launch reader threads */ +for (i =0 ; i < num_cores; i++) +rte_eal_remote_launch (test_lpm_reader,NULL , + enabled_core_ids[i]); + +/* Measure add/delete. */ + begin =rte_rdtsc_precise (); +for (i =0 ; i <RCU_ITERATIONS ; i++) { +/* Add all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_add (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth, + next_hop_add) !=0 ) { +printf ("Failed to add iteration %d, route# %d\n" , + i, j); + goto error; + } + +/* Delete all the entries */ +for (j =0 ; j <NUM_LDEPTH_ROUTE_ENTRIES ; j++) +if (rte_lpm_delete (lpm, large_ldepth_route_table[j].ip, + large_ldepth_route_table[j].depth) !=0 ) { +printf ("Failed to delete iteration %d, route# %d\n" , + i, j); + goto error; + } + } + total_cycles =rte_rdtsc_precise () - begin; + +printf ("Total LPM Adds: %d\n" ,ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Total LPM Deletes: %d\n" , +ITERATIONS *NUM_LDEPTH_ROUTE_ENTRIES ); +printf ("Average LPM Add/Del: %g cycles\n" , + (double )total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *ITERATIONS )); + + writer_done =1 ; +/* Wait and check return value from reader threads */ +for (i =0 ; i < num_cores; i++) +if (rte_eal_wait_lcore (enabled_core_ids[i]) <0 ) +printf ("Warning: lcore %u not finished.\n" , + enabled_core_ids[i]); + +rte_lpm_free (lpm); + +return 0 ; + +error: + writer_done =1 ; +/* Wait until all readers have exited */ +rte_eal_mp_wait_lcore (); + +rte_lpm_free (lpm); +rte_free (rv); + +return -1 ; +} +static int test_lpm_perf (void ) { -struct rte_lpm *lpm =NULL ;struct rte_lpm_config config; config.max_rules =2000000 ;
@@ -343,7 +825,7 @@ test_lpm_perf(void) lpm =rte_lpm_create (__func__,SOCKET_ID_ANY , &config);TEST_LPM_ASSERT (lpm !=NULL ); -/* Measue add. */ +/* Measure add. */ begin =rte_rdtsc ();for (i =0 ; i <NUM_ROUTE_ENTRIES ; i++) {
@@ -478,6 +960,10 @@ test_lpm_perf(void)rte_lpm_delete_all (lpm);rte_lpm_free (lpm); +test_lpm_rcu_perf (); + +test_lpm_rcu_perf_multi_writer (); +return 0 ; }
Signature · 1 line
2.17.1
v5 cover RCU integration with LPM library
In reply to Ruifeng WangThis patchset integrates RCU QSBR support with LPM library.
Resource reclaimation implementation was splitted from the original series, and has already been part of RCU library. Rework the series to base LPM integration on RCU reclaimation APIs.
New API rte_lpm_rcu_qsbr_add is introduced for application to register a RCU variable that LPM library will use. This provides user the handle to enable RCU that integrated in LPM library.
Functional tests and performance tests are added to cover the integration with RCU.
---
v5:
No default value for reclaim_thd. This allows reclamation triggering with every call. Pass LPM pointer instead of tbl8 as argument of reclaim callback free function. Updated group_idx check at tbl8 allocation.
Use enums instead of defines for different reclamation modes.
RCU QSBR integrated path is inside ALLOW_EXPERIMENTAL_API to avoid ABI change.
v4:
Allow user to configure defer queue: size, reclaim threshold, max entries. Return defer queue handler so user can manually trigger reclaimation. Add blocking mode support. Defer queue will not be created.
Honnappa Nagarahalli (1):
test/lpm: add RCU integration performance tests
Ruifeng Wang (2):
lib/lpm: integrate RCU QSBR
test/lpm: add LPM RCU integration functional tests
app/test/test_lpm.c | 291 ++++++++++++++++- app/test/test_lpm_perf.c | 492 ++++++++++++++++++++++++++++- doc/guides/prog_guide/lpm_lib.rst | 32 ++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 +++++++- lib/librte_lpm/rte_lpm.h | 59 ++++ lib/librte_lpm/rte_lpm_version.map | 6 + 8 files changed, 995 insertions(+), 17 deletions(-)
Signature · 1 line
2.17.1
PATCH v5 1/3 lib/lpm: integrate RCU QSBR
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
--- doc/guides/prog_guide/lpm_lib.rst | 32 +++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 +++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 216 insertions(+), 13 deletions(-)
doc/guides/prog_guide/lpm_lib.rst +32 -0
@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. + Lookup ~~~~~~
lib/librte_lpm/Makefile +1 -1
@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.map
lib/librte_lpm/meson.build +1 -0
@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']
lib/librte_lpm/rte_lpm.c +117 -12
191 lines not on this page · raw message
lib/librte_lpm/rte_lpm.h +59 -0
88 lines not on this page · raw message
lib/librte_lpm/rte_lpm_version.map +6 -0
@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBR
Quoted 50 lines · Ruifeng Wang
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 +++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 +++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 216 insertions(+), 13 deletions(-)doc/guides/prog_guide/lpm_lib.rst +22
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not. · line 179
Quoted 10 lines
doc/guides/prog_guide/lpm_lib.rst +10
-0+ +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. +
Would the lpm6 library benefit from the same? Asking as I do not see much code shared between lpm and lpm6.
[...]
Quoted 12 lines
lib/librte_lpm/rte_lpm.c +1
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h>
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 252
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +4
-1@@ -245,13 +246,84 @@ rte_lpm_free(struct rte_lpm *lpm)TAILQ_REMOVE (lpm_list, te, next);rte_mcfg_tailq_write_unlock (); - +#ifdef ALLOW_EXPERIMENTAL_API +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq); +#endif
All DPDK code under lib/ is compiled with the ALLOW_EXPERIMENTAL_API flag set. There is no need to protect against this flag in rte_lpm.c.
[...]
Quoted 6 lines
lib/librte_lpm/rte_lpm.h +0
-0
lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 152
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +7
-0@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +#ifdef ALLOW_EXPERIMENTAL_API +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +enum rte_lpm_qsbr_mode rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +#endif +};
This is more a comment/question for the lpm maintainers.
Afaics, the rte_lpm structure is exported/public because of lookup which is inlined.
But most of the structure can be hidden and stored in a private structure that would embed the exposed rte_lpm.
The slowpath functions would only have to translate from publicly exposed to internal representation (via container_of).
This patch could do this and be the first step to hide the unneeded exposure of other fields (later/in 20.11 ?).
Thoughts?
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBR
Quoted 119 lines · David Marchand
Quoted 50 lines · Ruifeng Wang
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 +++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 +++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 216 insertions(+), 13 deletions(-)doc/guides/prog_guide/lpm_lib.rst +22
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately.doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not. · line 179
Quoted 10 lines
doc/guides/prog_guide/lpm_lib.rst +10
-0+ +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. +Would the lpm6 library benefit from the same? Asking as I do not see much code shared between lpm and lpm6.
[...]
Quoted 12 lines
lib/librte_lpm/rte_lpm.c +1
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 252
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +4
-1@@ -245,13 +246,84 @@ rte_lpm_free(struct rte_lpm *lpm)TAILQ_REMOVE (lpm_list, te, next);rte_mcfg_tailq_write_unlock (); - +#ifdef ALLOW_EXPERIMENTAL_API +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq); +#endif All DPDK code under
lib/is compiled with theALLOW_EXPERIMENTAL_APIflag set. There is no need to protect against this flag inrte_lpm.c.[...]
Quoted 6 lines
lib/librte_lpm/rte_lpm.h +0
-0lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 152
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +7
-0@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +#ifdef ALLOW_EXPERIMENTAL_API +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +enum rte_lpm_qsbr_mode rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +#endif +};This is more a comment/question for the lpm maintainers.
Afaics, the
rte_lpmstructure is exported/public because of lookup which is inlined.
But most of the structure can be hidden and stored in a private structure that would embed the exposedrte_lpm.
The slowpath functions would only have to translate from publicly exposed to internal representation (viacontainer_of).
Quoted 5 lines
This patch could do this and be the first step to hide the unneeded exposure of other fields (later/in 20.11 ?).
Thoughts?
Hiding as much of the structures as possible is always a good idea, so if that is possible in this patchset I would support such a move.
/Bruce
Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBR
Quoted 123 lines · Bruce Richardson
Quoted 122 lines · David Marchand
Quoted 50 lines · Ruifeng Wang
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 +++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 +++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 216 insertions(+), 13 deletions(-)doc/guides/prog_guide/lpm_lib.rst +22
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately.doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not. · line 179
Quoted 10 lines
doc/guides/prog_guide/lpm_lib.rst +10
-0+ +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. +Would the lpm6 library benefit from the same? Asking as I do not see much code shared between lpm and lpm6.
[...]
Quoted 12 lines
lib/librte_lpm/rte_lpm.c +1
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 252
Quoted 8 lines
lib/librte_lpm/rte_lpm.c +4
-1@@ -245,13 +246,84 @@ rte_lpm_free(struct rte_lpm *lpm)TAILQ_REMOVE (lpm_list, te, next);rte_mcfg_tailq_write_unlock (); - +#ifdef ALLOW_EXPERIMENTAL_API +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq); +#endif All DPDK code under
lib/is compiled with theALLOW_EXPERIMENTAL_APIflag set. There is no need to protect against this flag inrte_lpm.c.[...]
Quoted 6 lines
lib/librte_lpm/rte_lpm.h +0
-0lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 152
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +7
-0@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +#ifdef ALLOW_EXPERIMENTAL_API +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +enum rte_lpm_qsbr_mode rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +#endif +};This is more a comment/question for the lpm maintainers.
Afaics, the
rte_lpmstructure is exported/public because of lookup which is inlined.
But most of the structure can be hidden and stored in a private structure that would embed the exposedrte_lpm.
The slowpath functions would only have to translate from publicly exposed to internal representation (viacontainer_of).This patch could do this and be the first step to hide the unneeded exposure of other fields (later/in 20.11 ?).
Quoted 7 lines
Quoted 2 lines
Thoughts?
Hiding as much of the structures as possible is always a good idea, so if that is possible in this patchset I would support such a move.
/Bruce
Agreed - I acked the change as it doesn't break ABI compatibility. Bruce and David's comments still hold for 20.11+.
Ray K
Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBR
In reply to David MarchandHello Ruifeng,
Quoted 14 lines · David Marchand
Quoted 14 lines
lib/librte_lpm/rte_lpm.c +1
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -245,13 +246,84 @@ rte_lpm_free(struct rte_lpm *lpm)TAILQ_REMOVE (lpm_list, te, next);
Quoted 9 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm) · line 252
Quoted 6 lines
lib/librte_lpm/rte_lpm.c +4
-1rte_mcfg_tailq_write_unlock (); - +#ifdef ALLOW_EXPERIMENTAL_API +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq); +#endif All DPDK code under
lib/is compiled with theALLOW_EXPERIMENTAL_APIflag set. There is no need to protect against this flag inrte_lpm.c.
Please can you look at this?
Thanks.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBR
In reply to David MarchandHi David,
Sorry, I missed tracking of this thread.
Quoted 84 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, June 29, 2020 7:56 PM
To: Ruifeng Wang <[email protected]>; Vladimir Medvedkin <[email protected]>; Bruce Richardson <[email protected]>
Cc: John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Ray Kinsella <[email protected]>; Neil Horman <[email protected]>; dev <[email protected]>; Ananyev, Konstantin <[email protected]>; Honnappa Nagarahalli <[email protected]>; nd <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBRdoc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the
Quoted 26 lines · Ruifeng Wang
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 +++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 +++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 216 insertions(+), 13 deletions(-)doc/guides/prog_guide/lpm_lib.rst +0
-0@@ -145,6 +145,38 @@ depending on whether we need to move to thenext table or not.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 8 lines
doc/guides/prog_guide/lpm_lib.rst +0
-0Prefix expansion is one of the keys of this algorithm, since itimproves the speed dramatically by adding redundancy.
doc/guides/prog_guide/lpm_lib.rst +5
-0+Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement +rule is an existing rule that has the longest prefix match with the rule to bedeleted, but has smaller depth.
doc/guides/prog_guide/lpm_lib.rst
Quoted 3 lines
doc/guides/prog_guide/lpm_lib.rst +3
-0+ +If a replacement rule is found, target tbl24 and tbl8 entries are +updated to have the same depth and next hop value with thereplacement rule.
doc/guides/prog_guide/lpm_lib.rst
Quoted 2 lines
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +If no replacement rule can be found, target tbl24 and tbl8 entries will becleared.
doc/guides/prog_guide/lpm_lib.rst
Quoted 2 lines
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +Prefix expansion is performed if the rule's depth is not exactly 24 bits or32 bits.
doc/guides/prog_guide/lpm_lib.rst
Quoted 2 lines
doc/guides/prog_guide/lpm_lib.rst +2
-0+ +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entryare freed in following cases:
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 4 lines
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greaterthan 24.
doc/guides/prog_guide/lpm_lib.rst depending on whether we need to move to the next table or not.
Quoted 9 lines
doc/guides/prog_guide/lpm_lib.rst +9
-0+ +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately +even though the readers might be using the tbl8 group entries. This mightresult in incorrect lookup results.
doc/guides/prog_guide/lpm_lib.rst
Quoted 4 lines
doc/guides/prog_guide/lpm_lib.rst +4
-0+ +RCU QSBR process is integrated for safe tbl8 group reclaimation. +Application has certain responsibilities while using this feature. +Please refer to resource reclaimation framework of :ref:`RCU library<RCU_Library>` for more details.
Quoted 5 lines
Quoted 1 line
+
Would the lpm6 library benefit from the same? Asking as I do not see much code shared between lpm and lpm6.
Didn't look into lpm6. It may need separate integration with RCU since no shared code between lpm and lpm6 as you mentioned.
Quoted 16 lines
[...]
Quoted 14 lines
lib/librte_lpm/rte_lpm.c index renamed from lib/librte_lpm/rte_lpm.c +0
-0
38ab512a4..41e9c49b8100644lib/librte_lpm/rte_lpm.c +1
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -245,13 +246,84 @@ rte_lpm_free(struct rte_lpm *lpm)TAILQ_REMOVE (lpm_list, te, next);
Quoted 10 lines
lib/librte_lpm/rte_lpm.c rte_lpm_free(struct rte_lpm *lpm)
Quoted 5 lines
lib/librte_lpm/rte_lpm.c +3
-1rte_mcfg_tailq_write_unlock (); - +#ifdef ALLOW_EXPERIMENTAL_API +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq);#endif All DPDK code under
lib/is compiled with theALLOW_EXPERIMENTAL_APIflag set.
There is no need to protect against this flag inrte_lpm.c.
OK, I see. So DPDK libraries will always be compiled with the ALLOW_EXPERIMENTAL_API. It is application's choice to use experimental APIs.
Will update in next version to remove the ALLOW_EXPERIMENTAL_API flag from rte_lpm.c and only keep the one in rte_lpm.h.
Quoted 28 lines
[...]
Quoted 6 lines
lib/librte_lpm/rte_lpm.h index renamed from lib/librte_lpm/rte_lpm.h +0
-0
b9d49ac87..7889f21b3100644lib/librte_lpm/rte_lpm.h struct rte_lpm { · line 152
Quoted 10 lines
lib/librte_lpm/rte_lpm.h +7
-0@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +#ifdef ALLOW_EXPERIMENTAL_API +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +enum rte_lpm_qsbr_mode rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +#endif +};This is more a comment/question for the lpm maintainers.
Afaics, the
rte_lpmstructure is exported/public because of lookup which is inlined.
But most of the structure can be hidden and stored in a private structure that would embed the exposedrte_lpm.
The slowpath functions would only have to translate from publicly exposed to internal representation (viacontainer_of).
Quoted 3 lines
This patch could do this and be the first step to hide the unneeded exposure of other fields (later/in 20.11 ?).
To hide most of the structure is reasonable.
Since it will break ABI, I can do that in 20.11.
Quoted 5 lines
Thoughts?
--
David Marchand
Re: [dpdk-dev] [PATCH v5 1/3] lib/lpm: integrate RCU QSBR
In reply to Ruifeng WangQuoted 390 lines · Ruifeng Wang
Currently, the tbl8 group is freed even though the readers might be using the tbl8 group entries. The freed tbl8 group can be reallocated quickly. This results in incorrect lookup results.
RCU QSBR process is integrated for safe tbl8 group reclaim. Refer to RCU documentation to understand various aspects of integrating RCU library into other libraries.
Signed-off-by Ruifeng Wang <[email protected]>Reviewed-by Honnappa Nagarahalli <[email protected]>--- doc/guides/prog_guide/lpm_lib.rst | 32 +++++++ lib/librte_lpm/Makefile | 2 +- lib/librte_lpm/meson.build | 1 + lib/librte_lpm/rte_lpm.c | 129 ++++++++++++++++++++++++++--- lib/librte_lpm/rte_lpm.h | 59 +++++++++++++ lib/librte_lpm/rte_lpm_version.map | 6 ++ 6 files changed, 216 insertions(+), 13 deletions(-)doc/guides/prog_guide/lpm_lib.rst +32
-0@@ -145,6 +145,38 @@ depending on whether we need to move to the next table or not. Prefix expansion is one of the keys of this algorithm, since it improves the speed dramatically by adding redundancy. +Deletion +~~~~~~~~ + +When deleting a rule, a replacement rule is searched for. Replacement rule is an existing rule that has +the longest prefix match with the rule to be deleted, but has smaller depth. + +If a replacement rule is found, target tbl24 and tbl8 entries are updated to have the same depth and next hop +value with the replacement rule. + +If no replacement rule can be found, target tbl24 and tbl8 entries will be cleared. + +Prefix expansion is performed if the rule's depth is not exactly 24 bits or 32 bits. + +After deleting a rule, a group of tbl8s that belongs to the same tbl24 entry are freed in following cases: + +* All tbl8s in the group are empty . + +* All tbl8s in the group have the same values and with depth no greater than 24. + +Free of tbl8s have different behaviors: + +* If RCU is not used, tbl8s are cleared and reclaimed immediately. + +* If RCU is used, tbl8s are reclaimed when readers are in quiescent state. + +When the LPM is not using RCU, tbl8 group can be freed immediately even though the readers might be using +the tbl8 group entries. This might result in incorrect lookup results. + +RCU QSBR process is integrated for safe tbl8 group reclaimation. Application has certain responsibilities +while using this feature. Please refer to resource reclaimation framework of :ref:`RCU library <RCU_Library>` +for more details. + Lookup ~~~~~~lib/librte_lpm/Makefile +1
-1@@ -8,7 +8,7 @@ LIB = librte_lpm.a CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) -I$(SRCDIR) -LDLIBS += -lrte_eal -lrte_hash +LDLIBS += -lrte_eal -lrte_hash -lrte_rcu EXPORT_MAP := rte_lpm_version.maplib/librte_lpm/meson.build +1
-0@@ -7,3 +7,4 @@ headers = files('rte_lpm.h', 'rte_lpm6.h') # without worrying about which architecture we actually need headers += files('rte_lpm_altivec.h', 'rte_lpm_neon.h', 'rte_lpm_sse.h') deps += ['hash'] +deps += ['rcu']lib/librte_lpm/rte_lpm.c +117
-12@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #include <string.h> @@ -245,13 +246,84 @@ rte_lpm_free(struct rte_lpm *lpm)TAILQ_REMOVE (lpm_list, te, next);rte_mcfg_tailq_write_unlock (); - +#ifdef ALLOW_EXPERIMENTAL_API +if (lpm->dq) +rte_rcu_qsbr_dq_delete (lpm->dq); +#endif rte_free (lpm->tbl8);rte_free (lpm->rules_tbl);rte_free (lpm);rte_free (te); } +static void +__lpm_rcu_qsbr_free_resource (void *p,void *data,unsigned int n) +{ +struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; +uint32_t tbl8_group_index = *(uint32_t *)data; +struct rte_lpm_tbl_entry *tbl8 = ((struct rte_lpm *)p)->tbl8; + +RTE_SET_USED (n); +/* Set tbl8 group invalid */ +__atomic_store (&tbl8[tbl8_group_index], &zero_tbl8_entry, + __ATOMIC_RELAXED); +} + +/* Associate QSBR variable with an LPM object. +*/ +int +rte_lpm_rcu_qsbr_add (struct rte_lpm *lpm,struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq) +{ +char rcu_dq_name[RTE_RCU_QSBR_DQ_NAMESIZE ]; +struct rte_rcu_qsbr_dq_parameters params = {0 }; + +if ((lpm ==NULL ) || (cfg ==NULL )) { + rte_errno =EINVAL ; +return 1 ; + } + +if (lpm->v) { + rte_errno =EEXIST ; +return 1 ; + } + +if (cfg->mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* No other things to do. */ + }else if (cfg->mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Init QSBR defer queue. */ +snprintf (rcu_dq_name,sizeof (rcu_dq_name), +"LPM_RCU_%s" , lpm->name); + params.name = rcu_dq_name; + params.size = cfg->dq_size; +if (params.size ==0 ) + params.size = lpm->number_tbl8s; + params.trigger_reclaim_limit = cfg->reclaim_thd; + params.max_reclaim_size = cfg->reclaim_max; +if (params.max_reclaim_size ==0 ) + params.max_reclaim_size =RTE_LPM_RCU_DQ_RECLAIM_MAX ; + params.esize =sizeof (uint32_t );/* tbl8 group index */ + params.free_fn = __lpm_rcu_qsbr_free_resource; + params.p = lpm; + params.v = cfg->v; + lpm->dq =rte_rcu_qsbr_dq_create (¶ms); +if (lpm->dq ==NULL ) { +RTE_LOG (ERR ,LPM , +"LPM QS defer queue creation failed\n" ); +return 1 ; + } +if (dq) + *dq = lpm->dq; + }else { + rte_errno =EINVAL ; +return 1 ; + } + lpm->rcu_mode = cfg->mode; + lpm->v = cfg->v; + +return 0 ; +} +/* * Adds a rule to the rule table. * @@ -394,14 +466,15 @@ rule_find(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth)* Find, clean and allocate a tbl8. */ static int32_t -tbl8_alloc (struct rte_lpm_tbl_entry *tbl8,uint32_t number_tbl8s) +_tbl8_alloc (struct rte_lpm *lpm) {uint32_t group_idx;/* tbl8 group index. */ struct rte_lpm_tbl_entry *tbl8_entry;/* Scan through tbl8 to find a free (i.e. INVALID) tbl8 group. */ -for (group_idx =0 ; group_idx < number_tbl8s; group_idx++) { - tbl8_entry = &tbl8[group_idx *RTE_LPM_TBL8_GROUP_NUM_ENTRIES ]; +for (group_idx =0 ; group_idx < lpm->number_tbl8s; group_idx++) { + tbl8_entry = &lpm->tbl8[group_idx * +RTE_LPM_TBL8_GROUP_NUM_ENTRIES ];/* If a free tbl8 group is found clean it and set as VALID. */ if (!tbl8_entry->valid_group) { struct rte_lpm_tbl_entry new_tbl8_entry = {@@ -427,14 +500,46 @@ tbl8_alloc(struct rte_lpm_tbl_entry *tbl8, uint32_t number_tbl8s)return -ENOSPC ; } +static int32_t +tbl8_alloc (struct rte_lpm *lpm) +{ +int32_t group_idx;/* tbl8 group index. */ + + group_idx =_tbl8_alloc (lpm); +#ifdef ALLOW_EXPERIMENTAL_API +if ((group_idx == -ENOSPC ) && (lpm->dq !=NULL )) { +/* If there are no tbl8 groups try to reclaim one. */ +if (rte_rcu_qsbr_dq_reclaim (lpm->dq,1 ,NULL ,NULL ,NULL ) ==0 ) + group_idx =_tbl8_alloc (lpm); + } +#endif +return group_idx; +} +static void -tbl8_free (struct rte_lpm_tbl_entry *tbl8,uint32_t tbl8_group_start) +tbl8_free (struct rte_lpm *lpm,uint32_t tbl8_group_start) { -/* Set tbl8 group invalid*/ struct rte_lpm_tbl_entry zero_tbl8_entry = {0 }; - -__atomic_store (&tbl8[tbl8_group_start], &zero_tbl8_entry, +#ifdef ALLOW_EXPERIMENTAL_API +if (!lpm->v) { +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_SYNC ) { +/* Wait for quiescent state change. */ +rte_rcu_qsbr_synchronize (lpm->v,RTE_QSBR_THRID_INVALID ); +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, + __ATOMIC_RELAXED); + }else if (lpm->rcu_mode ==RTE_LPM_QSBR_MODE_DQ ) { +/* Push into QSBR defer queue. */ +rte_rcu_qsbr_dq_enqueue (lpm->dq, (void *)&tbl8_group_start); + } +#else +/* Set tbl8 group invalid*/ +__atomic_store (&lpm->tbl8[tbl8_group_start], &zero_tbl8_entry, __ATOMIC_RELAXED); +#endif }static __rte_noinline int32_t@@ -523,7 +628,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth,if (!lpm->tbl24[tbl24_index].valid) {/* Search for a free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);/* Check tbl8 allocation was successful. */ if (tbl8_group_index <0 ) {@@ -569,7 +674,7 @@ add_depth_big(struct rte_lpm *lpm, uint32_t ip_masked, uint8_t depth, }/* If valid entry but not extended calculate the index into Table8. */ else if (lpm->tbl24[tbl24_index].valid_group ==0 ) {/* Search for free tbl8 group. */ - tbl8_group_index =tbl8_alloc (lpm->tbl8, lpm->number_tbl8s); + tbl8_group_index =tbl8_alloc (lpm);if (tbl8_group_index <0 ) {return tbl8_group_index;@@ -977,7 +1082,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t ip_masked,*/ lpm->tbl24[tbl24_index].valid =0 ;__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }else if (tbl8_recycle_index >-1 ) {/* Update tbl24 entry. */ struct rte_lpm_tbl_entry new_tbl24_entry = {@@ -993,7 +1098,7 @@ delete_depth_big(struct rte_lpm *lpm, uint32_t ip_masked,__atomic_store (&lpm->tbl24[tbl24_index], &new_tbl24_entry, __ATOMIC_RELAXED);__atomic_thread_fence (__ATOMIC_RELEASE); -tbl8_free (lpm->tbl8, tbl8_group_start); +tbl8_free (lpm, tbl8_group_start); }#undef group_idxreturn 0 ;lib/librte_lpm/rte_lpm.h +59
-0@@ -1,5 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * Copyright(c) 2010-2014 Intel Corporation +* Copyright(c) 2020 Arm Limited */ #ifndef _RTE_LPM_H_@@ -20,6 +21,7 @@#include <rte_memory.h> #include <rte_common.h> #include <rte_vect.h> +#include <rte_rcu_qsbr.h> #ifdef __cplusplusextern "C" {@@ -62,6 +64,17 @@ extern "C" {/** Bitmask used to indicate successful lookup */ #define RTE_LPM_LOOKUP_SUCCESS 0x01000000 +/** @internal Default RCU defer queue entries to reclaim in one go. */ +#define RTE_LPM_RCU_DQ_RECLAIM_MAX 16 + +/** RCU reclamation modes */ +enum rte_lpm_qsbr_mode { +/** Create defer queue for reclaim. */ +RTE_LPM_QSBR_MODE_DQ =0 , +/** Use blocking mode reclaim. No defer queue created. */ +RTE_LPM_QSBR_MODE_SYNC +}; +#if RTE_BYTE_ORDER ==RTE_LITTLE_ENDIAN /** @internal Tbl24 entry structure. */ __extension__@@ -130,6 +143,28 @@ struct rte_lpm { __rte_cache_aligned;/**< LPM tbl24 table. */ struct rte_lpm_tbl_entry *tbl8;/**< LPM tbl8 table. */ struct rte_lpm_rule *rules_tbl;/**< LPM rules. */ +#ifdef ALLOW_EXPERIMENTAL_API +/* RCU config. */ +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +enum rte_lpm_qsbr_mode rcu_mode;/* Blocking, defer queue. */ +struct rte_rcu_qsbr_dq *dq;/* RCU QSBR defer queue. */ +#endif +}; + +/** LPM RCU QSBR configuration structure. */ +struct rte_lpm_rcu_config { +struct rte_rcu_qsbr *v;/* RCU QSBR variable. */ +/* Mode of RCU QSBR. RTE_LPM_QSBR_MODE_xxx +* '0' for default: create defer queue for reclaim. +*/ +enum rte_lpm_qsbr_mode mode; +uint32_t dq_size;/* RCU defer queue size. +* default: lpm->number_tbl8s. +*/ +uint32_t reclaim_thd;/* Threshold to trigger auto reclaim. */ +uint32_t reclaim_max;/* Max entries to reclaim in one go. +* default: RTE_LPM_RCU_DQ_RECLAIM_MAX. +*/ };/** @@ -179,6 +214,30 @@ rte_lpm_find_existing(const char *name);void rte_lpm_free (struct rte_lpm *lpm); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Associate RCU QSBR variable with an LPM object. +* +* @param lpm +* the lpm object to add RCU QSBR +* @param cfg +* RCU QSBR configuration +* @param dq +* handler of created RCU QSBR defer queue +* @return +* On success - 0 +* On error - 1 with error code set in rte_errno. +* Possible rte_errno codes are: +* - EINVAL - invalid pointer +* - EEXIST - already added QSBR +* - ENOMEM - memory allocation failure +*/ +__rte_experimental +int rte_lpm_rcu_qsbr_add(struct rte_lpm *lpm, struct rte_lpm_rcu_config *cfg, +struct rte_rcu_qsbr_dq **dq); +/** * Add a rule to the LPM table. * lib/librte_lpm/rte_lpm_version.map +0
-0
lib/librte_lpm/rte_lpm_version.map DPDK_20.0 { · line 29
Quoted 9 lines
lib/librte_lpm/rte_lpm_version.map +6
-0@@ -21,3 +21,9 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + global: + + rte_lpm_rcu_qsbr_add; +};