Skip to contentSkip to conversation

test/lpm: fix cycle calculation in rcu qsbr perf

PATCH Superseded Dharmik Thakkar 5 participants 52 messages Oct 29, 2020 – Nov 5, 2020

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/4 test/lpm: fix cycle calculation in rcu qsbr perf

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 43 ++++++++++++++--------------------------
 1 file changed, 15 insertions(+), 28 deletions(-)
app/test/test_lpm_perf.c +15 -28
index c5a238b9d1e8..0a2d76a983c3 100644
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
-static uint64_t gwrites;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;
 #define NUM_ROUTE_ENTRIES num_route_entries
 #define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries
 
+#define TOTAL_WRITES (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES)
+
 enum {
 	IP_CLASS_A,
 	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
 	/* 2 writer threads are used */
 	if (core_id % 2 == 0) {
 		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	total_cycles = rte_rdtsc_precise() - begin;
 
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
 	return 0;
 }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,13 +544,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) / TOTAL_WRITES
 		);
 
 	/* Wait and check return value from reader threads */
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
 	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -711,11 +700,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -771,11 +759,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
Signature · 1 line
2.17.1
#1 lore raw

PATCH v2 2/4 test/lpm: return error on failure in rcu qsbr perf

Return error if Add/Delete fail in multiwriter perf test

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 4 ++++
 1 file changed, 4 insertions(+)
app/test/test_lpm_perf.c +4 -0
index 0a2d76a983c3..251ea12345ae 100644
@@ -453,6 +453,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route# %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -464,6 +466,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
Signature · 1 line
2.17.1
#2 lore raw

PATCH v2 3/4 test/lpm: remove error checking in rcu qsbr perf

In reply to Dharmik Thakkar

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c +8 -13
index 251ea12345ae..4f15db4f85ee 100644
@@ -554,11 +554,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) / TOTAL_WRITES
 		);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -603,10 +602,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -710,10 +708,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		if (rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -769,11 +766,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
Signature · 1 line
2.17.1
#3 lore raw

PATCH v2 4/4 test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

---
 app/test/test_lpm_perf.c | 338 +++++++++------------------------------
 1 file changed, 73 insertions(+), 265 deletions(-)
app/test/test_lpm_perf.c +73 -265
index 4f15db4f85ee..08312023b661 100644
@@ -430,11 +430,16 @@ test_lpm_rcu_qsbr_writer(void *arg)
 {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == 1) {
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == 2) {
 		si = 0;
 		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
 	} else {
@@ -482,16 +487,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -504,273 +510,76 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) / TOTAL_WRITES
-		);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)(i + j),
+						enabled_core_ids[i]);
 
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
 
@@ -946,9 +755,8 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
 
 	return 0;
 }
Signature · 1 line
2.17.1
#4 lore raw

Re: [dpdk-dev] [PATCH 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Hi Dharmik,

Thanks for the patches, see comments inlined

Quoted 14 lines · Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

---
  app/test/test_lpm_perf.c | 338 +++++++++------------------------------
  1 file changed, 73 insertions(+), 265 deletions(-)
app/test/test_lpm_perf.c +0 -0
index 4f15db4f85ee..08312023b661 100644
@@ -430,11 +430,16 @@ test_lpm_rcu_qsbr_writer(void *arg)
  {

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) · line 437

Quoted 10 lines
app/test/test_lpm_perf.c +3 -3
  	unsigned int i, j, si, ei;
  	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
  	uint32_t next_hop_add = 0xAA;
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == 1) {

Probably it would be better to use enum here instead of 1/2/3?

Quoted 278 lines
app/test/test_lpm_perf.c +42 -209
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == 2) {
  		si = 0;
  		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
  	} else {
@@ -482,16 +487,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
  /*
   * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
   */
  static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
  {
  	struct rte_lpm_config config;
  	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
  	uint16_t core_id;
  	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
  	if (rte_lcore_count() < 3) {
  		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -504,273 +510,76 @@ test_lpm_rcu_perf_multi_writer(void)
  		num_cores++;
  	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) / TOTAL_WRITES
-		);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
  				goto error;
  			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		/* Launch reader threads */

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 558

Quoted 10 lines
app/test/test_lpm_perf.c +6 -3
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,

So now even single writer will acquire a lock for every _add/_delete operation. I don't think it is necessary.

Quoted 87 lines
app/test/test_lpm_perf.c +22 -50
+						(void *)(uintptr_t)(i + j),
+						enabled_core_ids[i]);
 
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
  				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
  	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
  	return 0;
 
@@ -946,9 +755,8 @@ test_lpm_perf(void)
  	rte_lpm_delete_all(lpm);
  	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
 
  	return 0;
  }
Signature · 2 lines
Regards,
Vladimir
#5 lore raw

Re: [dpdk-dev] [PATCH 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 24 lines · Vladimir

Hi Dharmik,

Thanks for the patches, see comments inlined

Quoted 15 lines · Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

---
 app/test/test_lpm_perf.c | 338 +++++++++------------------------------
 1 file changed, 73 insertions(+), 265 deletions(-)
app/test/test_lpm_perf.c +0 -1
index 4f15db4f85ee..08312023b661 100644
@@ -430,11 +430,16 @@ test_lpm_rcu_qsbr_writer(void *arg)
 {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
Quoted 9 lines

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) · line 437

Quoted 6 lines
app/test/test_lpm_perf.c +3 -1
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 -	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == 1) {

Probably it would be better to use enum here instead of 1/2/3?

Yes, I will update the patch.

Quoted 274 lines
Quoted 274 lines
app/test/test_lpm_perf.c +45 -204
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == 2) {
 		si = 0;
 		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
 	} else {
@@ -482,16 +487,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
   /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
   	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -504,273 +510,76 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 -	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) / TOTAL_WRITES
-		);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
 -		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 -	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 -	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 -	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 558

Quoted 6 lines
app/test/test_lpm_perf.c +3 -2
 -	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,

So now even single writer will acquire a lock for every _add/_delete operation. I don't think it is necessary.

Yes, agreed it is not necessary. I wanted to avoid additional if () statement, but I can add it in the new version.

Quoted 85 lines
Quoted 81 lines
app/test/test_lpm_perf.c +22 -47
+						(void *)(uintptr_t)(i + j),
+						enabled_core_ids[i]);
 -	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 -		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
   	return 0;
 @@ -946,9 +755,8 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 -	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
   	return 0;
 }
Signature · 2 lines
Regards,
Vladimir
#7 lore raw
In reply to Dharmik Thakkar
Quoted 3 lines · Dharmik Thakkar

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

To help review this patch, could you provide some more details in the commit log as to what exactly was wrong with the calculation and how this patch fixes things?

Quoted 6 lines
Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
#8 lore raw
Quoted 12 lines · Bruce Richardson
Quoted 3 lines · Dharmik Thakkar

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

To help review this patch, could you provide some more details in the commit log as to what exactly was wrong with the calculation and how this patch fixes things?

I will update the commit message in the next version. Adding it here as well:

Since, rcu qsbr tests run for ‘RCU_ITERATIONS’ and not ‘ITERATIONS’, replace ‘ITERATIONS’ with ‘RCU_ITERATIONS’ for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES. For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro ’TOTAL_WRITES’ and furthermore, ‘g_writes’ has been removed since it is always a fixed value equal to TOTAL_WRITES.

Quoted 6 lines
Quoted 6 lines
Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
#9 lore raw

Hi Dharmik,

Quoted 18 lines · Dharmik Thakkar
Quoted 12 lines · Bruce Richardson
Quoted 3 lines · Dharmik Thakkar

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

To help review this patch, could you provide some more details in the commit log as to what exactly was wrong with the calculation and how this patch fixes things?

I will update the commit message in the next version. Adding it here as well:

Since, rcu qsbr tests run for ‘RCU_ITERATIONS’ and not ‘ITERATIONS’, replace ‘ITERATIONS’ with ‘RCU_ITERATIONS’ for calculating adds, deletes, and cycles.

Quoted 8 lines

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES. For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro ’TOTAL_WRITES’ and furthermore, ‘g_writes’ has been removed since it is always a fixed value equal to TOTAL_WRITES.

Thanks for the clarification. I left a few comments regarding 4-th patch. First 3 patches LGTM, just put more details in the commit message.

Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
Signature · 2 lines
Regards,
Vladimir
#10 lore raw
In reply to Dharmik Thakkar
Quoted 18 lines · Dharmik Thakkar
Quoted 12 lines · Bruce Richardson
Quoted 3 lines · Dharmik Thakkar

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

To help review this patch, could you provide some more details in the commit log as to what exactly was wrong with the calculation and how this patch fixes things?

I will update the commit message in the next version. Adding it here as well:

Since, rcu qsbr tests run for ‘RCU_ITERATIONS’ and not ‘ITERATIONS’, replace ‘ITERATIONS’ with ‘RCU_ITERATIONS’ for calculating adds, deletes, and cycles.

Quoted 8 lines

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES. For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro ’TOTAL_WRITES’ and furthermore, ‘g_writes’ has been removed since it is always a fixed value equal to TOTAL_WRITES.

Thanks for the clear explanation.

#11 lore raw

v2 cover test/lpm: fix rcu qsbr perf test

In reply to Dharmik Thakkar

Fix LPM adds, LPM deletes, and cycle calculation.
Return error if LPM add/delete fails in multi-writer test. Remove redundant error checking for readers.
Combine single and multi threaded test cases to avoid code duplication.
---
v2:
- Add more details about the fix to the commit message
- Replace hard coded values with an enum
- Remove lock acquire/release for single writer

Dharmik Thakkar (4):
test/lpm: fix cycle calculation in rcu qsbr perf test/lpm: return error on failure in rcu qsbr perf test/lpm: remove error checking in rcu qsbr perf test/lpm: avoid code duplication in rcu qsbr perf

 app/test/test_lpm_perf.c | 384 ++++++++++-----------------------------
 1 file changed, 95 insertions(+), 289 deletions(-)
Signature · 1 line
2.17.1
#12 lore raw

PATCH v2 1/4 test/lpm: fix cycle calculation in rcu qsbr perf

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Since, rcu qsbr tests run for 'RCU_ITERATIONS' and not 'ITERATIONS', replace 'ITERATIONS' with 'RCU_ITERATIONS' for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES.
For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro 'TOTAL_WRITES' and furthermore, 'g_writes' has been removed since it is always a fixed value equal to TOTAL_WRITES.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 45 ++++++++++++++--------------------------
 1 file changed, 16 insertions(+), 29 deletions(-)
app/test/test_lpm_perf.c +16 -29
index c5a238b9d1e8..45164b23214b 100644
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
-static uint64_t gwrites;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;
 #define NUM_ROUTE_ENTRIES num_route_entries
 #define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries
 
+#define TOTAL_WRITES (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES)
+
 enum {
 	IP_CLASS_A,
 	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
 	/* 2 writer threads are used */
 	if (core_id % 2 == 0) {
 		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	total_cycles = rte_rdtsc_precise() - begin;
 
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
 	return 0;
 }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,14 +544,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	/* Wait and check return value from reader threads */
 	writer_done = 1;
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
 	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -711,11 +700,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -771,11 +759,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
Signature · 1 line
2.17.1
Reviewed-by on v2 1/4
#13 lore raw

<snip>

Quoted 19 lines

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Since, rcu qsbr tests run for 'RCU_ITERATIONS' and not 'ITERATIONS', replace 'ITERATIONS' with 'RCU_ITERATIONS'
for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES.
For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of
(2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro 'TOTAL_WRITES' and furthermore, 'g_writes' has been removed since it is always a fixed value equal to TOTAL_WRITES.

Quoted 6 lines
Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")

Looks good

Quoted 134 lines
---
 app/test/test_lpm_perf.c | 45 ++++++++++++++--------------------------
 1 file changed, 16 insertions(+), 29 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

c5a238b9d1e8..45164b23214b 100644

app/test/test_lpm_perf.c +0 -0
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;  static volatile uint8_t

writer_done; static volatile uint32_t thr_id; static uint64_t gwrite_cycles; - static uint64_t gwrites;
/* LPM APIs are not thread safe, use mutex to provide thread safety */
static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;

app/test/test_lpm_perf.c +0 -0
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;  #define

NUM_ROUTE_ENTRIES num_route_entries #define
NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries

+#define TOTAL_WRITES (RCU_ITERATIONS *
NUM_LDEPTH_ROUTE_ENTRIES)

app/test/test_lpm_perf.c +1 -4
+
 enum {
 	IP_CLASS_A,
 	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
 	/* 2 writer threads are used */
 	if (core_id % 2 == 0) {
 		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	total_cycles = rte_rdtsc_precise() - begin;
 
 	__atomic_fetch_add(&gwrite_cycles, total_cycles,
 __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES *
 RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
app/test/test_lpm_perf.c +9 -17
 	return 0;
 }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,14 +544,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	/* Wait and check return value from reader threads */
 	writer_done = 1;
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
 	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */ @@ -711,11
+700,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS *

NUM_LDEPTH_ROUTE_ENTRIES);

app/test/test_lpm_perf.c +2 -3
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *

ITERATIONS));

app/test/test_lpm_perf.c +2 -1
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */ @@ -771,11
+759,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS *

NUM_LDEPTH_ROUTE_ENTRIES);

app/test/test_lpm_perf.c +2 -3
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES *

ITERATIONS));

app/test/test_lpm_perf.c +1 -1
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
--

2.17.1

#14 lore raw

PATCH v2 2/4 test/lpm: return error on failure in rcu qsbr perf

In reply to Dharmik Thakkar

Return error if Add/Delete fail in multiwriter perf test

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 4 ++++
 1 file changed, 4 insertions(+)
app/test/test_lpm_perf.c +4 -0
index 45164b23214b..55084816ab91 100644
@@ -453,6 +453,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route# %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -464,6 +466,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
Signature · 1 line
2.17.1
#15 lore raw

Re: [dpdk-dev] [PATCH v2 2/4] test/lpm: return error on failure in rcu qsbr perf

<snip>

Quoted 18 lines

Return error if Add/Delete fail in multiwriter perf test

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 4 ++++
 1 file changed, 4 insertions(+)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

45164b23214b..55084816ab91 100644

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) · line 458

Quoted 6 lines
app/test/test_lpm_perf.c +2 -0
@@ -453,6 +453,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route#
 %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;

Would be good to use the "goto error" method used in this file in other functions.

Quoted 15 lines
app/test/test_lpm_perf.c +0 -0
 			}
 			pthread_mutex_unlock(&lpm_mutex);
app/test/test_lpm_perf.c +2 -0
 		}
@@ -464,6 +466,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route#
 %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;
 			}
 			pthread_mutex_unlock(&lpm_mutex);

}
--
2.17.1

#16 lore raw

Re: [dpdk-dev] [PATCH v2 2/4] test/lpm: return error on failure in rcu qsbr perf

Quoted 19 lines · Honnappa Nagarahalli

<snip>

Quoted 13 lines

Return error if Add/Delete fail in multiwriter perf test

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")

---
app/test/test_lpm_perf.c | 4 ++++

1 file changed, 4 insertions(+)
Quoted 12 lines

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) · line 458

Quoted 11 lines
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

45164b23214b..55084816ab91 100644

app/test/test_lpm_perf.c +2 -0
@@ -453,6 +453,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route#
 %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;

Would be good to use the "goto error" method used in this file in other functions.

Yes, will update in the next version.

Quoted 16 lines
Quoted 16 lines
app/test/test_lpm_perf.c +0 -0
 			}
 			pthread_mutex_unlock(&lpm_mutex);
		}
app/test/test_lpm_perf.c +2 -0
@@ -464,6 +466,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route#
 %d\n",
 					i, j);
+				pthread_mutex_unlock(&lpm_mutex);
+				return -1;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
		}

--
2.17.1

#17 lore raw

PATCH v2 3/4 test/lpm: remove error checking in rcu qsbr perf

In reply to Dharmik Thakkar

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c +8 -13
index 55084816ab91..224c92fa3d65 100644
@@ -554,11 +554,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -603,10 +602,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -710,10 +708,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		if (rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -769,11 +766,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
Signature · 1 line
2.17.1
#18 lore raw

Re: [dpdk-dev] [PATCH v2 3/4] test/lpm: remove error checking in rcu qsbr perf

<snip>

Quoted 46 lines

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

55084816ab91..224c92fa3d65 100644

app/test/test_lpm_perf.c +4 -6
@@ -554,11 +554,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -603,10 +602,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 

app/test/test_lpm_perf.c test_lpm_rcu_perf(void) · line 713

Quoted 9 lines
app/test/test_lpm_perf.c +2 -3
@@ -710,10 +708,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		if (rte_eal_wait_lcore(enabled_core_ids[i]);

^^ Do we need the 'if' statement?

Quoted 19 lines
app/test/test_lpm_perf.c +2 -5
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -769,11 +766,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
--

2.17.1

#19 lore raw

Re: [dpdk-dev] [PATCH v2 3/4] test/lpm: remove error checking in rcu qsbr perf

Quoted 54 lines · Honnappa Nagarahalli

<snip>

Quoted 48 lines

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")

---
app/test/test_lpm_perf.c | 21 ++++++++-------------

1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

55084816ab91..224c92fa3d65 100644

app/test/test_lpm_perf.c +4 -6
@@ -554,11 +554,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -603,10 +602,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -710,10 +708,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf(void) · line 713

Quoted 7 lines
app/test/test_lpm_perf.c +2 -3
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		if (rte_eal_wait_lcore(enabled_core_ids[i]);

^^ Do we need the 'if' statement?

No, will remove in the next version.

Quoted 19 lines
Quoted 19 lines
app/test/test_lpm_perf.c +2 -5
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -769,11 +766,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
--

2.17.1

#20 lore raw

PATCH v2 4/4 test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

---
 app/test/test_lpm_perf.c | 362 ++++++++++-----------------------------
 1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c +91 -271
index 224c92fa3d65..229c835c23f7 100644
@@ -67,6 +67,12 @@ enum {
 	IP_CLASS_C
 };
 
+enum {
+	SINGLE_WRITER = 1,
+	MULTI_WRITER_1,
+	MULTI_WRITER_2
+};
+
 /* struct route_rule_count defines the total number of rules in following a/b/c
  * each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
  * including the ones for private local network.
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)
 {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == SINGLE_WRITER) {
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == MULTI_WRITER_1) {
 		si = 0;
 		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
 	} else {
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route# %d\n",
 					i, j);
-				pthread_mutex_unlock(&lpm_mutex);
+				if (writer_id != SINGLE_WRITER)
+					pthread_mutex_unlock(&lpm_mutex);
 				return -1;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
-				pthread_mutex_unlock(&lpm_mutex);
+				if (writer_id != SINGLE_WRITER)
+					pthread_mutex_unlock(&lpm_mutex);
 				return -1;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 	}
 
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -504,273 +522,76 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)(i + j),
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
 
@@ -946,9 +767,8 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
 
 	return 0;
 }
Signature · 1 line
2.17.1
#21 lore raw

Re: [dpdk-dev] [PATCH v2 4/4] test/lpm: avoid code duplication in rcu qsbr perf

<snip>

Quoted 14 lines

Avoid code duplication by combining single and multi threaded tests

---
 app/test/test_lpm_perf.c | 362 ++++++++++-----------------------------
 1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

224c92fa3d65..229c835c23f7 100644

app/test/test_lpm_perf.c enum { · line 74

Quoted 8 lines
app/test/test_lpm_perf.c +5 -0
@@ -67,6 +67,12 @@ enum {
 	IP_CLASS_C
 };
 
+enum {
+	SINGLE_WRITER = 1,
+	MULTI_WRITER_1,
+	MULTI_WRITER_2
+};

Do we need this? Can we use the number of cores instead?

Quoted 367 lines
app/test/test_lpm_perf.c +1 -0
+
 /* struct route_rule_count defines the total number of rules in following

a/b/c
* each item in a[]/b[]/c[] is the number of common IP address class A/B/C,
not
* including the ones for private local network.

app/test/test_lpm_perf.c +24 -12
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)  {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == SINGLE_WRITER) {
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == MULTI_WRITER_1) {
 		si = 0;
 		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
 	} else {
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route#
 %d\n",
 					i, j);
-				pthread_mutex_unlock(&lpm_mutex);
+				if (writer_id != SINGLE_WRITER)
+
 	pthread_mutex_unlock(&lpm_mutex);
 				return -1;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route#
 %d\n",
 					i, j);
-				pthread_mutex_unlock(&lpm_mutex);
+				if (writer_id != SINGLE_WRITER)
+
 	pthread_mutex_unlock(&lpm_mutex);
 				return -1;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 	}
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest,

expecting at least 3\n"); @@ -504,273 +522,76 @@
test_lpm_rcu_perf_multi_writer(void)

app/test/test_lpm_perf.c +0 -1
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration

enabled\n",

app/test/test_lpm_perf.c +0 -61
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration

disabled\n",

app/test/test_lpm_perf.c +0 -72
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest,

expecting at least 2\n");

app/test/test_lpm_perf.c +0 -49
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +20 -1
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+
	RTE_CACHE_LINE_SIZE);
app/test/test_lpm_perf.c +6 -3
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +5 -21
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration

disabled\n",

app/test/test_lpm_perf.c +4 -10
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 571

Quoted 10 lines
app/test/test_lpm_perf.c +5 -4
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)(i + j),

This can be just 'j'?

Quoted 76 lines
app/test/test_lpm_perf.c +1 -9
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +3 -5
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +4 -3
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +13 -17
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
@@ -946,9 +767,8 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
 
 	return 0;
 }
--

2.17.1

#22 lore raw

Re: [dpdk-dev] [PATCH v2 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 22 lines · Honnappa Nagarahalli

<snip>

Quoted 17 lines

Avoid code duplication by combining single and multi threaded tests

---
app/test/test_lpm_perf.c | 362 ++++++++++-----------------------------

1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

224c92fa3d65..229c835c23f7 100644

app/test/test_lpm_perf.c +0 -0
@@ -67,6 +67,12 @@ enum {
 	IP_CLASS_C
 };
 
Quoted 7 lines

app/test/test_lpm_perf.c enum { · line 74

Quoted 5 lines
app/test/test_lpm_perf.c +5 -0
+enum {
+	SINGLE_WRITER = 1,
+	MULTI_WRITER_1,
+	MULTI_WRITER_2
+};

Do we need this? Can we use the number of cores instead?

There are 3 combinations of writes (adds/deletes):
1. Write all the entries - in case of a single writer
2. Write half of the entries - in case of multiple writers
3. Write remaining half of the entries - in case of multiple writers

So, I think this is required.

Quoted 455 lines
Quoted 367 lines
app/test/test_lpm_perf.c +1 -0
+
 /* struct route_rule_count defines the total number of rules in following

a/b/c
* each item in a[]/b[]/c[] is the number of common IP address class A/B/C,
not
* including the ones for private local network.

app/test/test_lpm_perf.c +18 -8
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)  {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == SINGLE_WRITER) {
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == MULTI_WRITER_1) {
 		si = 0;
 		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
 	} else {
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route#
 %d\n",
 					i, j);
-				pthread_mutex_unlock(&lpm_mutex);
+				if (writer_id != SINGLE_WRITER)
+
 	pthread_mutex_unlock(&lpm_mutex);
 				return -1;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route#
 %d\n",
 					i, j);
-				pthread_mutex_unlock(&lpm_mutex);
+				if (writer_id != SINGLE_WRITER)
+
 	pthread_mutex_unlock(&lpm_mutex);
 				return -1;
			}
app/test/test_lpm_perf.c +2 -1
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
		}
	}
app/test/test_lpm_perf.c +1 -1
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
 * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
 */

static int

app/test/test_lpm_perf.c +1 -1
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
{
	struct rte_lpm_config config;
	size_t sz;
app/test/test_lpm_perf.c +1 -1
-	unsigned int i;
+	unsigned int i, j;
	uint16_t core_id;
	struct rte_lpm_rcu_config rcu_cfg = {0};
app/test/test_lpm_perf.c +1 -0
+	int (*reader_f)(void *arg) = NULL;
	if (rte_lcore_count() < 3) {
		printf("Not enough cores for lpm_rcu_perf_autotest,
expecting at least 3\n"); @@ -504,273 +522,76 @@
test_lpm_rcu_perf_multi_writer(void)
		num_cores++;
	}
app/test/test_lpm_perf.c +0 -1
-	printf("\nPerf test: 2 writers, %d readers, RCU integration

enabled\n",

app/test/test_lpm_perf.c +0 -61
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration

disabled\n",

app/test/test_lpm_perf.c +0 -72
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest,

expecting at least 2\n");

app/test/test_lpm_perf.c +0 -49
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +20 -1
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+
	RTE_CACHE_LINE_SIZE);
app/test/test_lpm_perf.c +6 -0
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
				goto error;
			}
app/test/test_lpm_perf.c +0 -3
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +5 -21
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration

disabled\n",

app/test/test_lpm_perf.c +4 -10
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 571

Quoted 10 lines
app/test/test_lpm_perf.c +5 -4
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)(i + j),

This can be just 'j'?

Quoted 76 lines
app/test/test_lpm_perf.c +1 -9
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +3 -1
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
				goto error;
app/test/test_lpm_perf.c +0 -4
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +4 -3
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +11 -0
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
	}
app/test/test_lpm_perf.c +0 -13
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
	return 0;
app/test/test_lpm_perf.c +2 -4
@@ -946,9 +767,8 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
 
 	return 0;
 }
--

2.17.1

#23 lore raw

Re: [dpdk-dev] [PATCH v2 4/4] test/lpm: avoid code duplication in rcu qsbr perf

<snip>

Quoted 27 lines
Quoted 19 lines
Quoted 19 lines

Avoid code duplication by combining single and multi threaded tests

---
app/test/test_lpm_perf.c | 362
++++++++++-----------------------------

1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c +0 -0

index
224c92fa3d65..229c835c23f7 100644

app/test/test_lpm_perf.c +0 -0
@@ -67,6 +67,12 @@ enum {

IP_CLASS_C
};

Quoted 7 lines

app/test/test_lpm_perf.c enum { · line 74

Quoted 5 lines
app/test/test_lpm_perf.c +5 -0
+enum {
+SINGLE_WRITER = 1,
+MULTI_WRITER_1,
+MULTI_WRITER_2
+};

Do we need this? Can we use the number of cores instead?

Quoted 6 lines

There are 3 combinations of writes (adds/deletes):
1. Write all the entries - in case of a single writer 2. Write half of the entries - in case of multiple writers 3. Write remaining half of the entries - in case of multiple writers

So, I think this is required.

IMO, this is not scalable. Essentially, we need 2 parameters to divide the routes among each writer thread. We need 2 parameters, 1) total number of writers 2) the core ID in the linear space. Creating a structure with these 2 and passing that to the writer thread would be better and scalable.

Quoted 417 lines
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) {

Quoted 8 lines

+
/* struct route_rule_count defines the total number of rules in following a/b/c
* each item in a[]/b[]/c[] is the number of common IP address class
A/B/C, not
* including the ones for private local network.

app/test/test_lpm_perf.c +0 -0
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)  {

unsigned

Quoted 85 lines
Quoted 85 lines

int i, j, si, ei; uint64_t begin, total_cycles; -uint8_t core_id = (uint8_t)((uintptr_t)arg);
+uint8_t writer_id = (uint8_t)((uintptr_t)arg); uint32_t next_hop_add = 0xAA;

app/test/test_lpm_perf.c +7 -2
-/* 2 writer threads are used */
-if (core_id % 2 == 0) {
+/* Single writer (writer_id = 1) */
+if (writer_id == SINGLE_WRITER) {
+si = 0;
+ei = NUM_LDEPTH_ROUTE_ENTRIES;
+}
+/* 2 Writers (writer_id = 2/3)*/
+else if (writer_id == MULTI_WRITER_1) {

si = 0;
ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
} else {

app/test/test_lpm_perf.c +8 -4
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg) for (i = 0;
 i < RCU_ITERATIONS; i++) {
 /* Add all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 large_ldepth_route_table[j].depth,
 next_hop_add) != 0) {
 printf("Failed to add iteration %d, route# %d\n", i, j);
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+
 pthread_mutex_unlock(&lpm_mutex);
 return -1;
 }
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);
 }
 
 /* Delete all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,

large_ldepth_route_table[j].depth) != 0) { printf("Failed to delete iteration %d, route# %d\n", i, j); -pthread_mutex_unlock(&lpm_mutex);

app/test/test_lpm_perf.c +2 -0
+if (writer_id != SINGLE_WRITER)
+

pthread_mutex_unlock(&lpm_mutex);
return -1;
}

app/test/test_lpm_perf.c +2 -1
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);

}
}

app/test/test_lpm_perf.c +1 -1
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
 * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
 */

static int

app/test/test_lpm_perf.c +1 -1
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)

{
struct rte_lpm_config config;
size_t sz;

app/test/test_lpm_perf.c +1 -1
-unsigned int i;
+unsigned int i, j;

uint16_t core_id;
struct rte_lpm_rcu_config rcu_cfg = {0};
+int (*reader_f)(void *arg) = NULL;

if (rte_lcore_count() < 3) {
printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n"); @@ -504,273 +522,76 @@ test_lpm_rcu_perf_multi_writer(void)
num_cores++;
}

Quoted 5 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 5 lines

-printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 46 lines
Quoted 46 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if (rte_lpm_rcu_qsbr_add(lpm,

&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n"); -goto error; -}

app/test/test_lpm_perf.c +0 -22
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i < num_cores;

i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -4
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-rv = NULL;
-
-/* Test without RCU integration */
-printf("\nPerf test: 2 writers, %d readers, RCU integration

disabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 65 lines
Quoted 65 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -21
-TEST_LPM_ASSERT(lpm != NULL);
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL, -enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i < num_cores;

i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -32
-
-rte_lpm_free(lpm);
-
-return 0;
-
-error:
-writer_done = 1;
-/* Wait until all readers have exited */ -rte_eal_mp_wait_lcore();
-
-rte_lpm_free(lpm);
-rte_free(rv);
-
-return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-struct rte_lpm_config config;
-uint64_t begin, total_cycles;
-size_t sz;
-unsigned int i, j;
-uint16_t core_id;
-uint32_t next_hop_add = 0xAA;
-struct rte_lpm_rcu_config rcu_cfg = {0};
-
-if (rte_lcore_count() < 2) {
-printf("Not enough cores for lpm_rcu_perf_autotest, expecting at

least 2\n"); -return TEST_SKIPPED; -}

app/test/test_lpm_perf.c +0 -2
-
-num_cores = 0;
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-RTE_LCORE_FOREACH_WORKER(core_id) {
-enabled_core_ids[num_cores] = core_id; -num_cores++; -}
-
-printf("\nPerf test: 1 writer, %d readers, RCU integration

enabled\n", -num_cores);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 31 lines
Quoted 31 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if (rte_lpm_rcu_qsbr_add(lpm,

&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n"); -goto error; -}

app/test/test_lpm_perf.c +0 -14
-
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if (rte_lpm_add(lpm,

large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth,

app/test/test_lpm_perf.c +1 -2
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route# %d\n", -i, j);
+for (j = 1; j < 3; j++) {
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 534

Quoted 8 lines
app/test/test_lpm_perf.c +8 -0
+if (use_rcu)
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration enabled\n", j, num_cores - j); else
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration disabled\n", j, num_cores - j);
+
+/* Create LPM table */
+config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;

config.number_tbl8s =

Quoted 33 lines
Quoted 33 lines
app/test/test_lpm_perf.c +9 -0
+NUM_LDEPTH_ROUTE_ENTRIES; config.flags = 0; lpm =
+rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+TEST_LPM_ASSERT(lpm != NULL);
+
+/* Init RCU variable */
+if (use_rcu) {
+sz = rte_rcu_qsbr_get_memsize(num_cores);
+rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+

RTE_CACHE_LINE_SIZE);

app/test/test_lpm_perf.c +5 -0
+rte_rcu_qsbr_init(rv, num_cores);
+
+rcu_cfg.v = rv;
+/* Assign the RCU variable to LPM */ if (rte_lpm_rcu_qsbr_add(lpm,
+&rcu_cfg) != 0) { printf("RCU variable assignment failed\n");

goto error;
}

app/test/test_lpm_perf.c +0 -2
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth) != 0) { -printf("Failed to delete iteration %d, route# %d\n", -i, j); -goto error; -} -} -total_cycles = rte_rdtsc_precise() - begin;

app/test/test_lpm_perf.c +2 -1
+reader_f = test_lpm_rcu_qsbr_reader; } else reader_f =
+test_lpm_reader;
 
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del: %g cycles\n", -(double)total_cycles / TOTAL_WRITES);

app/test/test_lpm_perf.c +2 -0
+writer_done = 0;
+__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
Quoted 15 lines
Quoted 12 lines
app/test/test_lpm_perf.c +0 -2
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 0; i < num_cores;

i++) -if (rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -8
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
-rv = NULL;
-
-/* Test without RCU integration */
-printf("\nPerf test: 1 writer, %d readers, RCU integration

disabled\n", -num_cores);

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 3 lines
app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 98 lines
Quoted 10 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +4 -3
-TEST_LPM_ASSERT(lpm != NULL);
+__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+/* Launch reader threads */
+for (i = j; i < num_cores; i++)
+rte_eal_remote_launch(reader_f, NULL,

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 571

Quoted 10 lines
app/test/test_lpm_perf.c +5 -4
+enabled_core_ids[i]);
 
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL,
-enabled_core_ids[i]);
+/* Launch writer threads */
+for (i = 0; i < j; i++)
+rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+(void *)(uintptr_t)(i + j),

This can be just 'j'?

Quoted 76 lines
app/test/test_lpm_perf.c +1 -9
+enabled_core_ids[i]);
 
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-large_ldepth_route_table[j].depth,
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +3 -1
-i, j);
+/* Wait for writer threads */
+for (i = 0; i < j; i++)
+if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)

goto error;

app/test/test_lpm_perf.c +0 -4
-}
 
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-large_ldepth_route_table[j].depth) != 0) {
-printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +4 -3
-i, j);
-goto error;
-}
+printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+__atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +11 -0
+/ TOTAL_WRITES);
+
+writer_done = 1;
+/* Wait until all readers have exited */
+for (i = j; i < num_cores; i++)
+rte_eal_wait_lcore(enabled_core_ids[i]);
+
+rte_lpm_free(lpm);
+rte_free(rv);
+lpm = NULL;
+rv = NULL;

}

app/test/test_lpm_perf.c +0 -13
-total_cycles = rte_rdtsc_precise() - begin;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-printf("Average LPM Add/Del: %g cycles\n",
-(double)total_cycles / TOTAL_WRITES);
-
-writer_done = 1;
-/* Wait until all readers have exited */
-for (i = 0; i < num_cores; i++)
-rte_eal_wait_lcore(enabled_core_ids[i]);
-
-rte_lpm_free(lpm);

return 0;

app/test/test_lpm_perf.c +2 -4
@@ -946,9 +767,8 @@ test_lpm_perf(void)
 rte_lpm_delete_all(lpm);
 rte_lpm_free(lpm);
 
-test_lpm_rcu_perf();
-
-test_lpm_rcu_perf_multi_writer();
+test_lpm_rcu_perf_multi_writer(0);
+test_lpm_rcu_perf_multi_writer(1);
 
 return 0;
 }
--

2.17.1

#24 lore raw

Re: [dpdk-dev] [PATCH v2 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 33 lines · Honnappa Nagarahalli

<snip>

Quoted 27 lines
Quoted 26 lines
Quoted 19 lines

Avoid code duplication by combining single and multi threaded tests

---
app/test/test_lpm_perf.c | 362
++++++++++-----------------------------

1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c +0 -0

index
224c92fa3d65..229c835c23f7 100644

app/test/test_lpm_perf.c +0 -0
@@ -67,6 +67,12 @@ enum {

IP_CLASS_C
};

app/test/test_lpm_perf.c enum { · line 74

Quoted 5 lines
app/test/test_lpm_perf.c +5 -0
+enum {
+SINGLE_WRITER = 1,
+MULTI_WRITER_1,
+MULTI_WRITER_2
+};

Do we need this? Can we use the number of cores instead?

Quoted 8 lines
Quoted 6 lines

There are 3 combinations of writes (adds/deletes):
1. Write all the entries - in case of a single writer 2. Write half of the entries - in case of multiple writers 3. Write remaining half of the entries - in case of multiple writers

So, I think this is required.

IMO, this is not scalable. Essentially, we need 2 parameters to divide the routes among each writer thread. We need 2 parameters, 1) total number of writers 2) the core ID in the linear space. Creating a structure with these 2 and passing that to the writer thread would be better and scalable.

Yes, agreed this is only applicable for 2 writers. Currently, the multi writer test is only limited to a maximum of 2 writers. To support more number of writers, we need something like this (which I believe is in lines with your suggestion):
1. Calculate what each writer will write: single_insert = TOTAL_WRITES / num_writers
2. Pass core ID in linear space as an argument to the writer function: pos_core
3. Calculate si and ei in the writer function: si = pos_core * single_insert; ei = si + single_insert

I can update the patch to enable more than 2 writers.
Do you also suggest we expand the scope of the test to test with more than 2 writers?
This will increase the time for which the test is running (which currently is significant even with 2 writers).

Quoted 417 lines
Quoted 417 lines
Quoted 9 lines

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) {

Quoted 9 lines

+
/* struct route_rule_count defines the total number of rules in following a/b/c
* each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
* including the ones for private local network.

app/test/test_lpm_perf.c +0 -0
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)  {

unsigned

Quoted 85 lines
Quoted 85 lines

int i, j, si, ei; uint64_t begin, total_cycles; -uint8_t core_id = (uint8_t)((uintptr_t)arg);
+uint8_t writer_id = (uint8_t)((uintptr_t)arg); uint32_t next_hop_add = 0xAA;

app/test/test_lpm_perf.c +7 -2
-/* 2 writer threads are used */
-if (core_id % 2 == 0) {
+/* Single writer (writer_id = 1) */
+if (writer_id == SINGLE_WRITER) {
+si = 0;
+ei = NUM_LDEPTH_ROUTE_ENTRIES;
+}
+/* 2 Writers (writer_id = 2/3)*/
+else if (writer_id == MULTI_WRITER_1) {

si = 0;
ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
} else {

app/test/test_lpm_perf.c +8 -4
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg) for (i = 0;
 i < RCU_ITERATIONS; i++) {
 /* Add all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 large_ldepth_route_table[j].depth,
 next_hop_add) != 0) {
 printf("Failed to add iteration %d, route# %d\n", i, j);
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+
 pthread_mutex_unlock(&lpm_mutex);
 return -1;
 }
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);
 }
 
 /* Delete all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,

large_ldepth_route_table[j].depth) != 0) { printf("Failed to delete iteration %d, route# %d\n", i, j); -pthread_mutex_unlock(&lpm_mutex);

app/test/test_lpm_perf.c +2 -0
+if (writer_id != SINGLE_WRITER)
+

pthread_mutex_unlock(&lpm_mutex);
return -1;
}

app/test/test_lpm_perf.c +2 -1
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);

}
}

app/test/test_lpm_perf.c +1 -1
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
 * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
 */

static int

app/test/test_lpm_perf.c +1 -1
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)

{
struct rte_lpm_config config;
size_t sz;

app/test/test_lpm_perf.c +1 -1
-unsigned int i;
+unsigned int i, j;

uint16_t core_id;
struct rte_lpm_rcu_config rcu_cfg = {0};
+int (*reader_f)(void *arg) = NULL;

if (rte_lcore_count() < 3) {
printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n"); @@ -504,273 +522,76 @@ test_lpm_rcu_perf_multi_writer(void)
num_cores++;
}

Quoted 5 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 5 lines

-printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 46 lines
Quoted 46 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if (rte_lpm_rcu_qsbr_add(lpm,

&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n"); -goto error; -}

app/test/test_lpm_perf.c +0 -22
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i < num_cores;

i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -4
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-rv = NULL;
-
-/* Test without RCU integration */
-printf("\nPerf test: 2 writers, %d readers, RCU integration

disabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 65 lines
Quoted 65 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -21
-TEST_LPM_ASSERT(lpm != NULL);
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL, -enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i < num_cores;

i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -32
-
-rte_lpm_free(lpm);
-
-return 0;
-
-error:
-writer_done = 1;
-/* Wait until all readers have exited */ -rte_eal_mp_wait_lcore();
-
-rte_lpm_free(lpm);
-rte_free(rv);
-
-return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-struct rte_lpm_config config;
-uint64_t begin, total_cycles;
-size_t sz;
-unsigned int i, j;
-uint16_t core_id;
-uint32_t next_hop_add = 0xAA;
-struct rte_lpm_rcu_config rcu_cfg = {0};
-
-if (rte_lcore_count() < 2) {
-printf("Not enough cores for lpm_rcu_perf_autotest, expecting at

least 2\n"); -return TEST_SKIPPED; -}

app/test/test_lpm_perf.c +0 -2
-
-num_cores = 0;
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-RTE_LCORE_FOREACH_WORKER(core_id) {
-enabled_core_ids[num_cores] = core_id; -num_cores++; -}
-
-printf("\nPerf test: 1 writer, %d readers, RCU integration

enabled\n", -num_cores);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 31 lines
Quoted 31 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if (rte_lpm_rcu_qsbr_add(lpm,

&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n"); -goto error; -}

app/test/test_lpm_perf.c +0 -14
-
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if (rte_lpm_add(lpm,

large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth,

app/test/test_lpm_perf.c +1 -2
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route# %d\n", -i, j);
+for (j = 1; j < 3; j++) {
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 534

Quoted 8 lines
app/test/test_lpm_perf.c +8 -0
+if (use_rcu)
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration enabled\n", j, num_cores - j); else
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration disabled\n", j, num_cores - j);
+
+/* Create LPM table */
+config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;

config.number_tbl8s =

Quoted 33 lines
Quoted 33 lines
app/test/test_lpm_perf.c +9 -0
+NUM_LDEPTH_ROUTE_ENTRIES; config.flags = 0; lpm =
+rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+TEST_LPM_ASSERT(lpm != NULL);
+
+/* Init RCU variable */
+if (use_rcu) {
+sz = rte_rcu_qsbr_get_memsize(num_cores);
+rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+

RTE_CACHE_LINE_SIZE);

app/test/test_lpm_perf.c +5 -0
+rte_rcu_qsbr_init(rv, num_cores);
+
+rcu_cfg.v = rv;
+/* Assign the RCU variable to LPM */ if (rte_lpm_rcu_qsbr_add(lpm,
+&rcu_cfg) != 0) { printf("RCU variable assignment failed\n");

goto error;
}

app/test/test_lpm_perf.c +0 -2
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth) != 0) { -printf("Failed to delete iteration %d, route# %d\n", -i, j); -goto error; -} -} -total_cycles = rte_rdtsc_precise() - begin;

app/test/test_lpm_perf.c +2 -1
+reader_f = test_lpm_rcu_qsbr_reader; } else reader_f =
+test_lpm_reader;
 
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del: %g cycles\n", -(double)total_cycles / TOTAL_WRITES);

app/test/test_lpm_perf.c +2 -0
+writer_done = 0;
+__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
Quoted 15 lines
Quoted 12 lines
app/test/test_lpm_perf.c +0 -2
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 0; i < num_cores;

i++) -if (rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -8
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
-rv = NULL;
-
-/* Test without RCU integration */
-printf("\nPerf test: 1 writer, %d readers, RCU integration

disabled\n", -num_cores);

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 3 lines
app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 98 lines
Quoted 10 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +4 -3
-TEST_LPM_ASSERT(lpm != NULL);
+__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+/* Launch reader threads */
+for (i = j; i < num_cores; i++)
+rte_eal_remote_launch(reader_f, NULL,

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 571

Quoted 10 lines
app/test/test_lpm_perf.c +5 -4
+enabled_core_ids[i]);
 
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL,
-enabled_core_ids[i]);
+/* Launch writer threads */
+for (i = 0; i < j; i++)
+rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+(void *)(uintptr_t)(i + j),

This can be just 'j'?

Quoted 76 lines
app/test/test_lpm_perf.c +1 -9
+enabled_core_ids[i]);
 
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-large_ldepth_route_table[j].depth,
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +3 -1
-i, j);
+/* Wait for writer threads */
+for (i = 0; i < j; i++)
+if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)

goto error;

app/test/test_lpm_perf.c +0 -4
-}
 
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-large_ldepth_route_table[j].depth) != 0) {
-printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +4 -3
-i, j);
-goto error;
-}
+printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+__atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +11 -0
+/ TOTAL_WRITES);
+
+writer_done = 1;
+/* Wait until all readers have exited */
+for (i = j; i < num_cores; i++)
+rte_eal_wait_lcore(enabled_core_ids[i]);
+
+rte_lpm_free(lpm);
+rte_free(rv);
+lpm = NULL;
+rv = NULL;

}

app/test/test_lpm_perf.c +0 -13
-total_cycles = rte_rdtsc_precise() - begin;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-printf("Average LPM Add/Del: %g cycles\n",
-(double)total_cycles / TOTAL_WRITES);
-
-writer_done = 1;
-/* Wait until all readers have exited */
-for (i = 0; i < num_cores; i++)
-rte_eal_wait_lcore(enabled_core_ids[i]);
-
-rte_lpm_free(lpm);

return 0;

app/test/test_lpm_perf.c +2 -4
@@ -946,9 +767,8 @@ test_lpm_perf(void)
 rte_lpm_delete_all(lpm);
 rte_lpm_free(lpm);
 
-test_lpm_rcu_perf();
-
-test_lpm_rcu_perf_multi_writer();
+test_lpm_rcu_perf_multi_writer(0);
+test_lpm_rcu_perf_multi_writer(1);
 
 return 0;
 }
--

2.17.1

#25 lore raw

Re: [dpdk-dev] [PATCH v2 4/4] test/lpm: avoid code duplication in rcu qsbr perf

<snip>

Quoted 48 lines
Quoted 27 lines
Quoted 27 lines
Quoted 26 lines
Quoted 19 lines

Avoid code duplication by combining single and multi threaded tests

---
app/test/test_lpm_perf.c | 362
++++++++++-----------------------------

1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c +0 -0

index
224c92fa3d65..229c835c23f7 100644

app/test/test_lpm_perf.c +0 -0
@@ -67,6 +67,12 @@ enum {

IP_CLASS_C
};

app/test/test_lpm_perf.c enum { · line 74

Quoted 5 lines
app/test/test_lpm_perf.c +5 -0
+enum {
+SINGLE_WRITER = 1,
+MULTI_WRITER_1,
+MULTI_WRITER_2
+};

Do we need this? Can we use the number of cores instead?

Quoted 7 lines
Quoted 6 lines

There are 3 combinations of writes (adds/deletes):
1. Write all the entries - in case of a single writer 2. Write half of the entries - in case of multiple writers 3. Write remaining half of the entries - in case of multiple writers

So, I think this is required.

IMO, this is not scalable. Essentially, we need 2 parameters to divide the

routes among each writer thread. We need 2 parameters, 1) total number of writers 2) the core ID in the linear space.

Quoted 1 line

Creating a structure with these 2 and passing that to the writer thread

would be better and scalable.

Yes, agreed this is only applicable for 2 writers. Currently, the multi writer test is only limited to a maximum of 2 writers.
To support more number of writers, we need something like this (which I believe is in lines with your suggestion):
1. Calculate what each writer will write: single_insert = TOTAL_WRITES / num_writers 2. Pass core ID in linear space as an argument to the writer
function: pos_core 3. Calculate si and ei in the writer function: si = pos_core * single_insert; ei = si + single_insert

Quoted 5 lines

I can update the patch to enable more than 2 writers.
Do you also suggest we expand the scope of the test to test with more than 2 writers?
This will increase the time for which the test is running (which currently is significant even with 2 writers).

Agree, no to increasing the number of writers. Yes for making the code more generic.

Quoted 410 lines
Quoted 216 lines
Quoted 216 lines
Quoted 10 lines

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) {

Quoted 10 lines

+
/* struct route_rule_count defines the total number of rules in following a/b/c
* each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
* including the ones for private local network.

app/test/test_lpm_perf.c +0 -0
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)  {

unsigned

Quoted 83 lines
Quoted 83 lines

int i, j, si, ei; uint64_t begin, total_cycles; -uint8_t core_id = (uint8_t)((uintptr_t)arg);
+uint8_t writer_id = (uint8_t)((uintptr_t)arg); uint32_t next_hop_add = 0xAA;

app/test/test_lpm_perf.c +4 -2
-/* 2 writer threads are used */
-if (core_id % 2 == 0) {
+/* Single writer (writer_id = 1) */ if (writer_id ==
+SINGLE_WRITER) { si = 0; ei = NUM_LDEPTH_ROUTE_ENTRIES; }
+/* 2 Writers (writer_id = 2/3)*/
+else if (writer_id == MULTI_WRITER_1) {

si = 0;
ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
} else {

app/test/test_lpm_perf.c +8 -4
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg) for (i =
 0; i < RCU_ITERATIONS; i++) {
 /* Add all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 large_ldepth_route_table[j].depth,
 next_hop_add) != 0) {
 printf("Failed to add iteration %d, route# %d\n", i, j);
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+
 pthread_mutex_unlock(&lpm_mutex);
 return -1;
 }
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);
 }
 
 /* Delete all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,

large_ldepth_route_table[j].depth) != 0) { printf("Failed to delete iteration %d, route# %d\n", i, j);

app/test/test_lpm_perf.c +2 -1
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+

pthread_mutex_unlock(&lpm_mutex);
return -1;
}

app/test/test_lpm_perf.c +2 -1
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);

}
}

app/test/test_lpm_perf.c +1 -1
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
 * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
 */

static int

app/test/test_lpm_perf.c +1 -1
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)

{
struct rte_lpm_config config;
size_t sz;

app/test/test_lpm_perf.c +1 -1
-unsigned int i;
+unsigned int i, j;

uint16_t core_id;
struct rte_lpm_rcu_config rcu_cfg = {0};
+int (*reader_f)(void *arg) = NULL;

if (rte_lcore_count() < 3) {
printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n"); @@ -504,273 +522,76 @@ test_lpm_rcu_perf_multi_writer(void)
num_cores++;
}

Quoted 5 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 5 lines

-printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 47 lines
Quoted 47 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if

(rte_lpm_rcu_qsbr_add(lpm,
&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n");

app/test/test_lpm_perf.c +0 -23
-goto error; -}
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i <

num_cores;
i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -3
-
-rte_lpm_free(lpm);
-rte_free(rv);
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-lpm = NULL;
-rv = NULL;
-
-/* Test without RCU integration */ -printf("\nPerf test: 2

writers, %d readers, RCU integration disabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 60 lines
Quoted 60 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -22
-TEST_LPM_ASSERT(lpm != NULL);
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL,
-enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i <

num_cores;
i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -28
-
-rte_lpm_free(lpm);
-
-return 0;
-
-error:
-writer_done = 1;
-/* Wait until all readers have exited */ -rte_eal_mp_wait_lcore();
-
-rte_lpm_free(lpm);
-rte_free(rv);
-
-return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-struct rte_lpm_config config;
-uint64_t begin, total_cycles;
-size_t sz;
-unsigned int i, j;
-uint16_t core_id;
-uint32_t next_hop_add = 0xAA;
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-struct rte_lpm_rcu_config rcu_cfg = {0};
-
-if (rte_lcore_count() < 2) {
-printf("Not enough cores for lpm_rcu_perf_autotest, expecting at

least 2\n"); -return TEST_SKIPPED; -}

app/test/test_lpm_perf.c +0 -3
-
-num_cores = 0;
-RTE_LCORE_FOREACH_WORKER(core_id) { -

enabled_core_ids[num_cores] =

Quoted 184 lines
Quoted 184 lines
Quoted 7 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 7 lines

core_id; -num_cores++; -}

app/test/test_lpm_perf.c +0 -2
-
-printf("\nPerf test: 1 writer, %d readers, RCU integration

enabled\n", -num_cores);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 33 lines
Quoted 33 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if

(rte_lpm_rcu_qsbr_add(lpm,
&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n");

app/test/test_lpm_perf.c +0 -15
-goto error; -}
-
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_add(lpm, large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +1 -3
-large_ldepth_route_table[j].depth,
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route# %d\n", -i, j);
+for (j = 1; j < 3; j++) {
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 534

Quoted 8 lines
app/test/test_lpm_perf.c +8 -0
+if (use_rcu)
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration enabled\n", j, num_cores - j); else
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration disabled\n", j, num_cores - j);
+
+/* Create LPM table */
+config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;

config.number_tbl8s =

Quoted 33 lines
Quoted 33 lines
app/test/test_lpm_perf.c +9 -0
+NUM_LDEPTH_ROUTE_ENTRIES; config.flags = 0; lpm =
+rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+TEST_LPM_ASSERT(lpm != NULL);
+
+/* Init RCU variable */
+if (use_rcu) {
+sz = rte_rcu_qsbr_get_memsize(num_cores);
+rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+

RTE_CACHE_LINE_SIZE);

app/test/test_lpm_perf.c +5 -0
+rte_rcu_qsbr_init(rv, num_cores);
+
+rcu_cfg.v = rv;
+/* Assign the RCU variable to LPM */ if (rte_lpm_rcu_qsbr_add(lpm,
+&rcu_cfg) != 0) { printf("RCU variable assignment failed\n");

goto error;
}

app/test/test_lpm_perf.c +0 -2
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth) != 0) { -printf("Failed to delete iteration %d, route# %d\n", -i, j); -goto error; -} -}

app/test/test_lpm_perf.c +2 -2
-total_cycles = rte_rdtsc_precise() - begin;
+reader_f = test_lpm_rcu_qsbr_reader; } else reader_f =
+test_lpm_reader;
 
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del: %g cycles\n", -(double)total_cycles / TOTAL_WRITES);

app/test/test_lpm_perf.c +2 -0
+writer_done = 0;
+__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
Quoted 15 lines
Quoted 12 lines
app/test/test_lpm_perf.c +0 -2
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 0; i <

num_cores;
i++) -if (rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -7
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
-rv = NULL;
-
-/* Test without RCU integration */ -printf("\nPerf test: 1 writer,

%d readers, RCU integration disabled\n", -num_cores);

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 3 lines
app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 85 lines
Quoted 9 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +3 -3
-TEST_LPM_ASSERT(lpm != NULL);
+__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+/* Launch reader threads */
+for (i = j; i < num_cores; i++)

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 571

Quoted 10 lines
app/test/test_lpm_perf.c +5 -4
+rte_eal_remote_launch(reader_f, NULL, enabled_core_ids[i]);
 
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL,
-enabled_core_ids[i]);
+/* Launch writer threads */
+for (i = 0; i < j; i++)
+rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+(void *)(uintptr_t)(i + j),

This can be just 'j'?

Quoted 64 lines
app/test/test_lpm_perf.c +1 -5
+enabled_core_ids[i]);
 
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_add(lpm, large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +3 -3
-large_ldepth_route_table[j].depth,
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route# %d\n", -i, j);
+/* Wait for writer threads */
+for (i = 0; i < j; i++)
+if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)

goto error;

app/test/test_lpm_perf.c +0 -3
-}
 
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth) != 0) { -printf("Failed to delete iteration %d, route# %d\n", -i, j); -goto error; -}

app/test/test_lpm_perf.c +3 -0
+printf("Total LPM Adds: %d\n", TOTAL_WRITES); printf("Total LPM
+Deletes: %d\n", TOTAL_WRITES); printf("Average LPM Add/Del:
+%"PRIu64" cycles\n", __atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +10 -0
+/ TOTAL_WRITES);
+
+writer_done = 1;
+/* Wait until all readers have exited */ for (i = j; i <
+num_cores; i++) rte_eal_wait_lcore(enabled_core_ids[i]);
+
+rte_lpm_free(lpm);
+rte_free(rv);
+lpm = NULL;
+rv = NULL;

}

app/test/test_lpm_perf.c +0 -3
-total_cycles = rte_rdtsc_precise() - begin;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del: %g cycles\n", -(double)total_cycles / TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 0; i <

num_cores; i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -2
-
-rte_lpm_free(lpm);

return 0;

app/test/test_lpm_perf.c +2 -4
@@ -946,9 +767,8 @@ test_lpm_perf(void) rte_lpm_delete_all(lpm);
 rte_lpm_free(lpm);
 
-test_lpm_rcu_perf();
-
-test_lpm_rcu_perf_multi_writer();
+test_lpm_rcu_perf_multi_writer(0);
+test_lpm_rcu_perf_multi_writer(1);
 
 return 0;
 }
--

2.17.1

#26 lore raw

Re: [dpdk-dev] [PATCH v2 4/4] test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Hi,

Quoted 39 lines · Dharmik Thakkar
Quoted 39 lines · Honnappa Nagarahalli

<snip>

Quoted 33 lines
Quoted 19 lines
Quoted 19 lines

Avoid code duplication by combining single and multi threaded tests

---
app/test/test_lpm_perf.c | 362
++++++++++-----------------------------

1 file changed, 91 insertions(+), 271 deletions(-)
app/test/test_lpm_perf.c +0 -0

index
224c92fa3d65..229c835c23f7 100644

app/test/test_lpm_perf.c +0 -0
@@ -67,6 +67,12 @@ enum {

IP_CLASS_C
};

Quoted 7 lines

app/test/test_lpm_perf.c enum { · line 74

Quoted 5 lines
app/test/test_lpm_perf.c +5 -0
+enum {
+SINGLE_WRITER = 1,
+MULTI_WRITER_1,
+MULTI_WRITER_2
+};

Do we need this? Can we use the number of cores instead?

There are 3 combinations of writes (adds/deletes):
1. Write all the entries - in case of a single writer 2. Write half of the entries - in case of multiple writers 3. Write remaining half of the entries - in case of multiple writers

So, I think this is required.

Quoted 9 lines
Quoted 2 lines

IMO, this is not scalable. Essentially, we need 2 parameters to divide the routes among each writer thread. We need 2 parameters, 1) total number of writers 2) the core ID in the linear space. Creating a structure with these 2 and passing that to the writer thread would be better and scalable.

Yes, agreed this is only applicable for 2 writers. Currently, the multi writer test is only limited to a maximum of 2 writers. To support more number of writers, we need something like this (which I believe is in lines with your suggestion):
1. Calculate what each writer will write: single_insert = TOTAL_WRITES / num_writers
2. Pass core ID in linear space as an argument to the writer function: pos_core
3. Calculate si and ei in the writer function: si = pos_core * single_insert; ei = si + single_insert

Agree to Honnappa suggestion, for me it looks good, better than previous implementation.

Quoted 4 lines

I can update the patch to enable more than 2 writers.
Do you also suggest we expand the scope of the test to test with more than 2 writers?
This will increase the time for which the test is running (which currently is significant even with 2 writers).

I don't see any reason to increase the number of writers more than 2.

Quoted 418 lines
Quoted 417 lines
Quoted 417 lines
Quoted 9 lines

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) {

Quoted 9 lines

+
/* struct route_rule_count defines the total number of rules in following a/b/c
* each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
* including the ones for private local network.

app/test/test_lpm_perf.c +0 -0
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)  {

unsigned

Quoted 85 lines
Quoted 85 lines

int i, j, si, ei; uint64_t begin, total_cycles; -uint8_t core_id = (uint8_t)((uintptr_t)arg);
+uint8_t writer_id = (uint8_t)((uintptr_t)arg); uint32_t next_hop_add = 0xAA;

app/test/test_lpm_perf.c +7 -2
-/* 2 writer threads are used */
-if (core_id % 2 == 0) {
+/* Single writer (writer_id = 1) */
+if (writer_id == SINGLE_WRITER) {
+si = 0;
+ei = NUM_LDEPTH_ROUTE_ENTRIES;
+}
+/* 2 Writers (writer_id = 2/3)*/
+else if (writer_id == MULTI_WRITER_1) {

si = 0;
ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
} else {

app/test/test_lpm_perf.c +8 -4
@@ -447,29 +458,35 @@ test_lpm_rcu_qsbr_writer(void *arg) for (i = 0;
 i < RCU_ITERATIONS; i++) {
 /* Add all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 large_ldepth_route_table[j].depth,
 next_hop_add) != 0) {
 printf("Failed to add iteration %d, route# %d\n", i, j);
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+
 pthread_mutex_unlock(&lpm_mutex);
 return -1;
 }
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);
 }
 
 /* Delete all the entries */
 for (j = si; j < ei; j++) {
-pthread_mutex_lock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_lock(&lpm_mutex);
 if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,

large_ldepth_route_table[j].depth) != 0) { printf("Failed to delete iteration %d, route# %d\n", i, j); -pthread_mutex_unlock(&lpm_mutex);

app/test/test_lpm_perf.c +2 -0
+if (writer_id != SINGLE_WRITER)
+

pthread_mutex_unlock(&lpm_mutex);
return -1;
}

app/test/test_lpm_perf.c +2 -1
-pthread_mutex_unlock(&lpm_mutex);
+if (writer_id != SINGLE_WRITER)
+pthread_mutex_unlock(&lpm_mutex);

}
}

app/test/test_lpm_perf.c +1 -1
@@ -482,16 +499,17 @@ test_lpm_rcu_qsbr_writer(void *arg)
 
 /*
 * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
 */

static int

app/test/test_lpm_perf.c +1 -1
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)

{
struct rte_lpm_config config;
size_t sz;

app/test/test_lpm_perf.c +1 -1
-unsigned int i;
+unsigned int i, j;

uint16_t core_id;
struct rte_lpm_rcu_config rcu_cfg = {0};
+int (*reader_f)(void *arg) = NULL;

if (rte_lcore_count() < 3) {
printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n"); @@ -504,273 +522,76 @@ test_lpm_rcu_perf_multi_writer(void)
num_cores++;
}

Quoted 5 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 5 lines

-printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 46 lines
Quoted 46 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if (rte_lpm_rcu_qsbr_add(lpm,

&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n"); -goto error; -}

app/test/test_lpm_perf.c +0 -22
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i < num_cores;

i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -4
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-rv = NULL;
-
-/* Test without RCU integration */
-printf("\nPerf test: 2 writers, %d readers, RCU integration

disabled\n", -num_cores - 2);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 65 lines
Quoted 65 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -21
-TEST_LPM_ASSERT(lpm != NULL);
-
-writer_done = 0;
-__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 2; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL, -enabled_core_ids[i]);
-
-/* Launch writer threads */
-for (i = 0; i < 2; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-(void *)(uintptr_t)i,
-enabled_core_ids[i]);
-
-/* Wait for writer threads */
-for (i = 0; i < 2; i++)
-if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0) -goto error;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del:
%"PRIu64" cycles\n", -__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) -/ TOTAL_WRITES);

app/test/test_lpm_perf.c +0 -3
-
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 2; i < num_cores;

i++) -rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -32
-
-rte_lpm_free(lpm);
-
-return 0;
-
-error:
-writer_done = 1;
-/* Wait until all readers have exited */ -rte_eal_mp_wait_lcore();
-
-rte_lpm_free(lpm);
-rte_free(rv);
-
-return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-struct rte_lpm_config config;
-uint64_t begin, total_cycles;
-size_t sz;
-unsigned int i, j;
-uint16_t core_id;
-uint32_t next_hop_add = 0xAA;
-struct rte_lpm_rcu_config rcu_cfg = {0};
-
-if (rte_lcore_count() < 2) {
-printf("Not enough cores for lpm_rcu_perf_autotest, expecting at

least 2\n"); -return TEST_SKIPPED; -}

app/test/test_lpm_perf.c +0 -2
-
-num_cores = 0;
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 8 lines
app/test/test_lpm_perf.c +0 -4
-RTE_LCORE_FOREACH_WORKER(core_id) {
-enabled_core_ids[num_cores] = core_id; -num_cores++; -}
-
-printf("\nPerf test: 1 writer, %d readers, RCU integration

enabled\n", -num_cores);

app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 31 lines
Quoted 31 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +0 -9
-TEST_LPM_ASSERT(lpm != NULL);
-
-/* Init RCU variable */
-sz = rte_rcu_qsbr_get_memsize(num_cores);
-rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-RTE_CACHE_LINE_SIZE); -rte_rcu_qsbr_init(rv, num_cores);
-
-rcu_cfg.v = rv;
-/* Assign the RCU variable to LPM */ -if (rte_lpm_rcu_qsbr_add(lpm,

&rcu_cfg) != 0) { -printf("RCU variable assignment failed\n"); -goto error; -}

app/test/test_lpm_perf.c +0 -14
-
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-enabled_core_ids[i]);
-
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if (rte_lpm_add(lpm,

large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth,

app/test/test_lpm_perf.c +1 -2
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route# %d\n", -i, j);
+for (j = 1; j < 3; j++) {
Quoted 8 lines

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 534

Quoted 8 lines
app/test/test_lpm_perf.c +8 -0
+if (use_rcu)
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration enabled\n", j, num_cores - j); else
+printf("\nPerf test: %d writer(s), %d reader(s),"
+       " RCU integration disabled\n", j, num_cores - j);
+
+/* Create LPM table */
+config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;

config.number_tbl8s =

Quoted 33 lines
Quoted 33 lines
app/test/test_lpm_perf.c +9 -0
+NUM_LDEPTH_ROUTE_ENTRIES; config.flags = 0; lpm =
+rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+TEST_LPM_ASSERT(lpm != NULL);
+
+/* Init RCU variable */
+if (use_rcu) {
+sz = rte_rcu_qsbr_get_memsize(num_cores);
+rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+

RTE_CACHE_LINE_SIZE);

app/test/test_lpm_perf.c +5 -0
+rte_rcu_qsbr_init(rv, num_cores);
+
+rcu_cfg.v = rv;
+/* Assign the RCU variable to LPM */ if (rte_lpm_rcu_qsbr_add(lpm,
+&rcu_cfg) != 0) { printf("RCU variable assignment failed\n");

goto error;
}

app/test/test_lpm_perf.c +0 -2
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++) -if

(rte_lpm_delete(lpm, large_ldepth_route_table[j].ip, -large_ldepth_route_table[j].depth) != 0) { -printf("Failed to delete iteration %d, route# %d\n", -i, j); -goto error; -} -} -total_cycles = rte_rdtsc_precise() - begin;

app/test/test_lpm_perf.c +2 -1
+reader_f = test_lpm_rcu_qsbr_reader; } else reader_f =
+test_lpm_reader;
 
-printf("Total LPM Adds: %d\n", TOTAL_WRITES); -printf("Total LPM

Deletes: %d\n", TOTAL_WRITES); -printf("Average LPM Add/Del: %g cycles\n", -(double)total_cycles / TOTAL_WRITES);

app/test/test_lpm_perf.c +2 -0
+writer_done = 0;
+__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
Quoted 15 lines
Quoted 12 lines
app/test/test_lpm_perf.c +0 -2
-writer_done = 1;
-/* Wait until all readers have exited */ -for (i = 0; i < num_cores;

i++) -if (rte_eal_wait_lcore(enabled_core_ids[i]);

app/test/test_lpm_perf.c +0 -8
-
-rte_lpm_free(lpm);
-rte_free(rv);
-lpm = NULL;
-rv = NULL;
-
-/* Test without RCU integration */
-printf("\nPerf test: 1 writer, %d readers, RCU integration

disabled\n", -num_cores);

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void)

Quoted 3 lines
app/test/test_lpm_perf.c +0 -3
-
-/* Create LPM table */
-config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES; -

config.number_tbl8s =

Quoted 98 lines
Quoted 10 lines

NUM_LDEPTH_ROUTE_ENTRIES; -config.flags = 0; -lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);

app/test/test_lpm_perf.c +4 -3
-TEST_LPM_ASSERT(lpm != NULL);
+__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-writer_done = 0;
-__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+/* Launch reader threads */
+for (i = j; i < num_cores; i++)
+rte_eal_remote_launch(reader_f, NULL,

app/test/test_lpm_perf.c test_lpm_rcu_perf_multi_writer(void) · line 571

Quoted 10 lines
app/test/test_lpm_perf.c +5 -4
+enabled_core_ids[i]);
 
-/* Launch reader threads */
-for (i = 0; i < num_cores; i++)
-rte_eal_remote_launch(test_lpm_reader, NULL,
-enabled_core_ids[i]);
+/* Launch writer threads */
+for (i = 0; i < j; i++)
+rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+(void *)(uintptr_t)(i + j),

This can be just 'j'?

Quoted 76 lines
app/test/test_lpm_perf.c +1 -9
+enabled_core_ids[i]);
 
-/* Measure add/delete. */
-begin = rte_rdtsc_precise();
-for (i = 0; i < RCU_ITERATIONS; i++) {
-/* Add all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-large_ldepth_route_table[j].depth,
-next_hop_add) != 0) {
-printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +3 -1
-i, j);
+/* Wait for writer threads */
+for (i = 0; i < j; i++)
+if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)

goto error;

app/test/test_lpm_perf.c +0 -4
-}
 
-/* Delete all the entries */
-for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-large_ldepth_route_table[j].depth) != 0) {
-printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +4 -3
-i, j);
-goto error;
-}
+printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+__atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +11 -0
+/ TOTAL_WRITES);
+
+writer_done = 1;
+/* Wait until all readers have exited */
+for (i = j; i < num_cores; i++)
+rte_eal_wait_lcore(enabled_core_ids[i]);
+
+rte_lpm_free(lpm);
+rte_free(rv);
+lpm = NULL;
+rv = NULL;

}

app/test/test_lpm_perf.c +0 -13
-total_cycles = rte_rdtsc_precise() - begin;
-
-printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-printf("Average LPM Add/Del: %g cycles\n",
-(double)total_cycles / TOTAL_WRITES);
-
-writer_done = 1;
-/* Wait until all readers have exited */
-for (i = 0; i < num_cores; i++)
-rte_eal_wait_lcore(enabled_core_ids[i]);
-
-rte_lpm_free(lpm);

return 0;

app/test/test_lpm_perf.c +2 -4
@@ -946,9 +767,8 @@ test_lpm_perf(void)
 rte_lpm_delete_all(lpm);
 rte_lpm_free(lpm);
 
-test_lpm_rcu_perf();
-
-test_lpm_rcu_perf_multi_writer();
+test_lpm_rcu_perf_multi_writer(0);
+test_lpm_rcu_perf_multi_writer(1);
 
 return 0;
 }
--

2.17.1

Signature · 2 lines
Regards,
Vladimir
#27 lore raw

v3 cover test/lpm: fix rcu qsbr perf test

In reply to Dharmik Thakkar

Fix LPM adds, LPM deletes, and cycle calculation.
Return error if LPM add/delete fails in multi-writer test. Remove redundant error checking for readers.
Combine single and multi threaded test cases to avoid code duplication.
---
v3:
- Add 'goto error'
- Remove unnecessary if statement

v2:
- Add more details about the fix to the commit message
- Replace hard coded values with an enum
- Remove lock acquire/release for single writer

Dharmik Thakkar (4):
test/lpm: fix cycle calculation in rcu qsbr perf test/lpm: return error on failure in rcu qsbr perf test/lpm: remove error checking in rcu qsbr perf test/lpm: avoid code duplication in rcu qsbr perf

 app/test/test_lpm_perf.c | 381 ++++++++++-----------------------------
 1 file changed, 94 insertions(+), 287 deletions(-)
Signature · 1 line
2.17.1
#28 lore raw

PATCH v3 1/4 test/lpm: fix cycle calculation in rcu qsbr perf

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Since, rcu qsbr tests run for 'RCU_ITERATIONS' and not 'ITERATIONS', replace 'ITERATIONS' with 'RCU_ITERATIONS' for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES.
For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro 'TOTAL_WRITES' and furthermore, 'g_writes' has been removed since it is always a fixed value equal to TOTAL_WRITES.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 45 ++++++++++++++--------------------------
 1 file changed, 16 insertions(+), 29 deletions(-)
app/test/test_lpm_perf.c +16 -29
index c5a238b9d1e8..45164b23214b 100644
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
-static uint64_t gwrites;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;
 #define NUM_ROUTE_ENTRIES num_route_entries
 #define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries
 
+#define TOTAL_WRITES (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES)
+
 enum {
 	IP_CLASS_A,
 	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
 	/* 2 writer threads are used */
 	if (core_id % 2 == 0) {
 		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	total_cycles = rte_rdtsc_precise() - begin;
 
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
 	return 0;
 }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,14 +544,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	/* Wait and check return value from reader threads */
 	writer_done = 1;
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
 	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -711,11 +700,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -771,11 +759,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
Signature · 1 line
2.17.1
#29 lore raw

PATCH v3 2/4 test/lpm: return error on failure in rcu qsbr perf

In reply to Dharmik Thakkar

Return error if Add/Delete fail in multiwriter perf test

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 6 ++++++
 1 file changed, 6 insertions(+)
app/test/test_lpm_perf.c +6 -0
index 45164b23214b..fc4c9b60cbbc 100644
@@ -453,6 +453,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route# %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -464,6 +465,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -474,6 +476,10 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
 
 	return 0;
+
+error:
+	pthread_mutex_unlock(&lpm_mutex);
+	return -1;
 }
 
 /*
Signature · 1 line
2.17.1
Reviewed-by on v3 2/4
#30 lore raw

Re: [dpdk-dev] [PATCH v3 2/4] test/lpm: return error on failure in rcu qsbr perf

<snip>

Quoted 9 lines

Return error if Add/Delete fail in multiwriter perf test

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")

Looks good

Quoted 40 lines
---
 app/test/test_lpm_perf.c | 6 ++++++
 1 file changed, 6 insertions(+)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

45164b23214b..fc4c9b60cbbc 100644

app/test/test_lpm_perf.c +1 -0
@@ -453,6 +453,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route#
 %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
app/test/test_lpm_perf.c +1 -0
 		}
@@ -464,6 +465,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route#
 %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
app/test/test_lpm_perf.c +4 -0
 		}
@@ -474,6 +476,10 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	__atomic_fetch_add(&gwrite_cycles, total_cycles,
 __ATOMIC_RELAXED);
 
 	return 0;
+
+error:
+	pthread_mutex_unlock(&lpm_mutex);
+	return -1;
 }
 

/*
--
2.17.1

#31 lore raw

PATCH v3 3/4 test/lpm: remove error checking in rcu qsbr perf

In reply to Dharmik Thakkar

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c +8 -13
index fc4c9b60cbbc..fa6ebc4f7547 100644
@@ -556,11 +556,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -605,10 +604,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -712,10 +710,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -771,11 +768,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
Signature · 1 line
2.17.1
Reviewed-by on v3 3/4
#32 lore raw

Re: [dpdk-dev] [PATCH v3 3/4] test/lpm: remove error checking in rcu qsbr perf

Quoted 16 lines

-----Original Message-----
From: Dharmik Thakkar <[email protected]>
Sent: Monday, November 2, 2020 11:12 PM
To: Bruce Richardson <[email protected]>; Vladimir Medvedkin <[email protected]>; Gavin Hu <[email protected]>; Honnappa Nagarahalli <[email protected]>; Ruifeng Wang <[email protected]>
Cc: [email protected]; nd <[email protected]>; Dharmik Thakkar <[email protected]>; [email protected]
Subject: [PATCH v3 3/4] test/lpm: remove error checking in rcu qsbr perf

Remove redundant error checking for reader threads since they never return error.

Quoted 6 lines
Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")

Looks good

Quoted 64 lines
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

fc4c9b60cbbc..fa6ebc4f7547 100644

app/test/test_lpm_perf.c +8 -14
@@ -556,11 +556,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -605,10 +604,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -712,10 +710,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -771,11 +768,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
--

2.17.1

#33 lore raw

PATCH v3 4/4 test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

---
 app/test/test_lpm_perf.c | 359 ++++++++++-----------------------------
 1 file changed, 89 insertions(+), 270 deletions(-)
app/test/test_lpm_perf.c +89 -270
index fa6ebc4f7547..147801634210 100644
@@ -67,6 +67,12 @@ enum {
 	IP_CLASS_C
 };
 
+enum {
+	SINGLE_WRITER = 1,
+	MULTI_WRITER_1,
+	MULTI_WRITER_2
+};
+
 /* struct route_rule_count defines the total number of rules in following a/b/c
  * each item in a[]/b[]/c[] is the number of common IP address class A/B/C, not
  * including the ones for private local network.
@@ -430,11 +436,16 @@ test_lpm_rcu_qsbr_writer(void *arg)
 {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
+	uint8_t writer_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
+	/* Single writer (writer_id = 1) */
+	if (writer_id == SINGLE_WRITER) {
+		si = 0;
+		ei = NUM_LDEPTH_ROUTE_ENTRIES;
+	}
+	/* 2 Writers (writer_id = 2/3)*/
+	else if (writer_id == MULTI_WRITER_1) {
 		si = 0;
 		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
 	} else {
@@ -447,7 +458,8 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
@@ -455,19 +467,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (writer_id != SINGLE_WRITER)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 	}
 
@@ -478,22 +493,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	return 0;
 
 error:
-	pthread_mutex_unlock(&lpm_mutex);
+	if (writer_id != SINGLE_WRITER)
+		pthread_mutex_unlock(&lpm_mutex);
 	return -1;
 }
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -506,273 +523,76 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)(i + j),
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
 
@@ -948,9 +768,8 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
-
-	test_lpm_rcu_perf_multi_writer();
+	test_lpm_rcu_perf_multi_writer(0);
+	test_lpm_rcu_perf_multi_writer(1);
 
 	return 0;
 }
Signature · 1 line
2.17.1
#34 lore raw

v4 cover test/lpm: fix rcu qsbr perf test

In reply to Dharmik Thakkar

Fix LPM adds, LPM deletes, and cycle calculation.
Return error if LPM add/delete fails in multi-writer test. Return error if single or multi writer test fails
Remove redundant error checking for readers.
Combine single and multi threaded test cases to avoid code duplication.
---
v4:
- Return error if rcu qsbr test fails
- Improve multi writer test to enable more than 2 writers

v3:
- Add 'goto error'
- Remove unnecessary if statement

v2:
- Add more details about the fix to the commit message
- Replace hard coded values with an enum
- Remove lock acquire/release for single writer

Dharmik Thakkar (4):
test/lpm: fix cycle calculation in rcu qsbr perf test/lpm: return error on failure in rcu qsbr perf test/lpm: remove error checking in rcu qsbr perf test/lpm: avoid code duplication in rcu qsbr perf

 app/test/test_lpm_perf.c | 383 ++++++++++-----------------------------
 1 file changed, 91 insertions(+), 292 deletions(-)
Signature · 1 line
2.17.1
#35 lore raw

PATCH v4 1/4 test/lpm: fix cycle calculation in rcu qsbr perf

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Since, rcu qsbr tests run for 'RCU_ITERATIONS' and not 'ITERATIONS', replace 'ITERATIONS' with 'RCU_ITERATIONS' for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES.
For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro 'TOTAL_WRITES' and furthermore, 'g_writes' has been removed since it is always a fixed value equal to TOTAL_WRITES.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 45 ++++++++++++++--------------------------
 1 file changed, 16 insertions(+), 29 deletions(-)
app/test/test_lpm_perf.c +16 -29
index c5a238b9d1e8..45164b23214b 100644
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
-static uint64_t gwrites;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;
 #define NUM_ROUTE_ENTRIES num_route_entries
 #define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries
 
+#define TOTAL_WRITES (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES)
+
 enum {
 	IP_CLASS_A,
 	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
 	/* 2 writer threads are used */
 	if (core_id % 2 == 0) {
 		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	total_cycles = rte_rdtsc_precise() - begin;
 
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
 	return 0;
 }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,14 +544,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	/* Wait and check return value from reader threads */
 	writer_done = 1;
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
 	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -711,11 +700,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -771,11 +759,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
Signature · 1 line
2.17.1
#36 lore raw

PATCH v4 2/4 test/lpm: return error on failure in rcu qsbr perf

In reply to Dharmik Thakkar

Return error if Add/Delete fail in multiwriter perf test

Return error if single or multi writer test fails

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
app/test/test_lpm_perf.c +10 -2
index 45164b23214b..873ecf511c97 100644
@@ -453,6 +453,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route# %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -464,6 +465,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -474,6 +476,10 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
 
 	return 0;
+
+error:
+	pthread_mutex_unlock(&lpm_mutex);
+	return -1;
 }
 
 /*
@@ -947,9 +953,11 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
+	if (test_lpm_rcu_perf() < 0)
+		return -1;
 
-	test_lpm_rcu_perf_multi_writer();
+	if (test_lpm_rcu_perf_multi_writer() < 0)
+		return -1;
 
 	return 0;
 }
Signature · 1 line
2.17.1
#37 lore raw

PATCH v4 3/4 test/lpm: remove error checking in rcu qsbr perf

In reply to Dharmik Thakkar

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c +8 -13
index 873ecf511c97..c8e70ec89ff5 100644
@@ -556,11 +556,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -605,10 +604,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -712,10 +710,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -771,11 +768,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
Signature · 1 line
2.17.1
#38 lore raw

PATCH v4 4/4 test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

Also, enable support for more than 2 writers

---
 app/test/test_lpm_perf.c | 359 +++++++++------------------------------
 1 file changed, 84 insertions(+), 275 deletions(-)
app/test/test_lpm_perf.c +84 -275
index c8e70ec89ff5..a1485e74e77f 100644
@@ -23,6 +23,7 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
+static uint32_t single_insert;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -430,24 +431,21 @@ test_lpm_rcu_qsbr_writer(void *arg)
 {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
+	bool single_writer = (single_insert == NUM_LDEPTH_ROUTE_ENTRIES) ?
+				true : false;
+	uint8_t pos_core = (uint8_t)((uintptr_t)arg);
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
-		si = 0;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-	} else {
-		si = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES;
-	}
+	si = pos_core * single_insert;
+	ei = si + single_insert;
 
 	/* Measure add/delete. */
 	begin = rte_rdtsc_precise();
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
@@ -455,19 +453,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 	}
 
@@ -478,22 +479,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	return 0;
 
 error:
-	pthread_mutex_unlock(&lpm_mutex);
+	if (!single_writer)
+		pthread_mutex_unlock(&lpm_mutex);
 	return -1;
 }
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -506,273 +509,79 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Calculate writes by each writer */
+		single_insert = NUM_LDEPTH_ROUTE_ENTRIES / j;
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)i,
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
 
@@ -948,10 +757,10 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	if (test_lpm_rcu_perf() < 0)
+	if (test_lpm_rcu_perf_multi_writer(0) < 0)
 		return -1;
 
-	if (test_lpm_rcu_perf_multi_writer() < 0)
+	if (test_lpm_rcu_perf_multi_writer(1) < 0)
 		return -1;
 
 	return 0;
Signature · 1 line
2.17.1
Reviewed-by on v4 4/4
#39 lore raw

Re: [dpdk-dev] [PATCH v4 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 12 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Dharmik Thakkar
Sent: Tuesday, November 3, 2020 4:24 PM
To: Bruce Richardson <[email protected]>; Vladimir Medvedkin <[email protected]>
Cc: [email protected]; nd <[email protected]>; Dharmik Thakkar <[email protected]>
Subject: [dpdk-dev] [PATCH v4 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 6 lines

Avoid code duplication by combining single and multi threaded tests

Also, enable support for more than 2 writers

Looks good

Quoted 467 lines
---
 app/test/test_lpm_perf.c | 359 +++++++++------------------------------
 1 file changed, 84 insertions(+), 275 deletions(-)
app/test/test_lpm_perf.c index renamed from app/test/test_lpm_perf.c +0 -0

c8e70ec89ff5..a1485e74e77f 100644

app/test/test_lpm_perf.c +2 -1
@@ -23,6 +23,7 @@ static struct rte_rcu_qsbr *rv;  static volatile uint8_t
 writer_done;  static volatile uint32_t thr_id;  static uint64_t gwrite_cycles;
+static uint32_t single_insert;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -430,24 +431,21 @@ test_lpm_rcu_qsbr_writer(void *arg)  {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
+	bool single_writer = (single_insert ==

NUM_LDEPTH_ROUTE_ENTRIES) ?

app/test/test_lpm_perf.c +12 -12
+				true : false;
+	uint8_t pos_core = (uint8_t)((uintptr_t)arg);
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
-		si = 0;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-	} else {
-		si = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES;
-	}
+	si = pos_core * single_insert;
+	ei = si + single_insert;
 
 	/* Measure add/delete. */
 	begin = rte_rdtsc_precise();
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
@@ -455,19 +453,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm,
 large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route#
 %d\n",
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
app/test/test_lpm_perf.c +6 -4
 	}
@@ -478,22 +479,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	return 0;
 
 error:
-	pthread_mutex_unlock(&lpm_mutex);
+	if (!single_writer)
+		pthread_mutex_unlock(&lpm_mutex);
 	return -1;
 }
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest,

expecting at least 3\n"); @@ -506,273 +509,79 @@
test_lpm_rcu_perf_multi_writer(void)

app/test/test_lpm_perf.c +0 -1
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration

enabled\n",

app/test/test_lpm_perf.c +0 -61
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration

disabled\n",

app/test/test_lpm_perf.c +0 -72
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest,

expecting at least 2\n");

app/test/test_lpm_perf.c +0 -49
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +0 -7
-					i, j);
-				goto error;
-			}
-
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +23 -1
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Calculate writes by each writer */
+		single_insert = NUM_LDEPTH_ROUTE_ENTRIES / j;
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+
	RTE_CACHE_LINE_SIZE);
app/test/test_lpm_perf.c +11 -17
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration

disabled\n",

app/test/test_lpm_perf.c +10 -23
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)i,
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +3 -5
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm,

large_ldepth_route_table[j].ip,

app/test/test_lpm_perf.c +0 -2
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route#

%d\n",

app/test/test_lpm_perf.c +4 -3
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles,

__ATOMIC_RELAXED)

app/test/test_lpm_perf.c +13 -16
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
@@ -948,10 +757,10 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	if (test_lpm_rcu_perf() < 0)
+	if (test_lpm_rcu_perf_multi_writer(0) < 0)
 		return -1;
 
-	if (test_lpm_rcu_perf_multi_writer() < 0)
+	if (test_lpm_rcu_perf_multi_writer(1) < 0)
 		return -1;
 
 	return 0;
--

2.17.1

#40 lore raw

Re: [dpdk-dev] [PATCH v4 4/4] test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Hi Thakkar,

Quoted 33 lines · Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

Also, enable support for more than 2 writers

---
  app/test/test_lpm_perf.c | 359 +++++++++------------------------------
  1 file changed, 84 insertions(+), 275 deletions(-)
app/test/test_lpm_perf.c +4 -2
index c8e70ec89ff5..a1485e74e77f 100644
@@ -23,6 +23,7 @@ static struct rte_rcu_qsbr *rv;
  static volatile uint8_t writer_done;
  static volatile uint32_t thr_id;
  static uint64_t gwrite_cycles;
+static uint32_t single_insert;
  /* LPM APIs are not thread safe, use mutex to provide thread safety */
  static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -430,24 +431,21 @@ test_lpm_rcu_qsbr_writer(void *arg)
  {
  	unsigned int i, j, si, ei;
  	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
  	uint32_t next_hop_add = 0xAA;
+	bool single_writer = (single_insert == NUM_LDEPTH_ROUTE_ENTRIES) ?
+				true : false;
+	uint8_t pos_core = (uint8_t)((uintptr_t)arg);
 
-	/* 2 writer threads are used */

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) · line 441

Quoted 10 lines
app/test/test_lpm_perf.c +2 -7
-	if (core_id % 2 == 0) {
-		si = 0;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-	} else {
-		si = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES;
-	}
+	si = pos_core * single_insert;
+	ei = si + single_insert;
 

In this case, given that you are doing
"single_insert = NUM_LDEPTH_ROUTE_ENTRIES / j;"
below, the number of ldepth_routes must be a multiple of the number of writers, so some number of routes can be skipped in the opposite case. Consider something like:

number_of_writers = j;
...
si = (pos_core * NUM_LDEPTH_ROUTE_ENTRIES)/number_of_writers; ei = ((pos_core + 1 ) * NUM_LDEPTH_ROUTE_ENTRIES)/number_of_writers;

Quoted 416 lines
app/test/test_lpm_perf.c +78 -266
  	/* Measure add/delete. */
  	begin = rte_rdtsc_precise();
  	for (i = 0; i < RCU_ITERATIONS; i++) {
  		/* Add all the entries */
  		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
  			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
  					large_ldepth_route_table[j].depth,
  					next_hop_add) != 0) {
@@ -455,19 +453,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
  					i, j);
  				goto error;
  			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
  		}
 
  		/* Delete all the entries */
  		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
  			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
  				large_ldepth_route_table[j].depth) != 0) {
  				printf("Failed to delete iteration %d, route# %d\n",
  					i, j);
  				goto error;
  			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
  		}
  	}
 
@@ -478,22 +479,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
  	return 0;
 
  error:
-	pthread_mutex_unlock(&lpm_mutex);
+	if (!single_writer)
+		pthread_mutex_unlock(&lpm_mutex);
  	return -1;
  }
 
  /*
   * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
   */
  static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
  {
  	struct rte_lpm_config config;
  	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
  	uint16_t core_id;
  	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
  	if (rte_lcore_count() < 3) {
  		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -506,273 +509,79 @@ test_lpm_rcu_perf_multi_writer(void)
  		num_cores++;
  	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Calculate writes by each writer */
+		single_insert = NUM_LDEPTH_ROUTE_ENTRIES / j;
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
  				goto error;
  			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)i,
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
  				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
  	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
  	return 0;
 
@@ -948,10 +757,10 @@ test_lpm_perf(void)
  	rte_lpm_delete_all(lpm);
  	rte_lpm_free(lpm);
 
-	if (test_lpm_rcu_perf() < 0)
+	if (test_lpm_rcu_perf_multi_writer(0) < 0)
  		return -1;
 
-	if (test_lpm_rcu_perf_multi_writer() < 0)
+	if (test_lpm_rcu_perf_multi_writer(1) < 0)
  		return -1;
 
  	return 0;
Signature · 2 lines
Regards,
Vladimir
#41 lore raw

Re: [dpdk-dev] [PATCH v4 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 49 lines · Vladimir

Hi Thakkar,

Quoted 27 lines · Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests Also, enable support for more than 2 writers

---
 app/test/test_lpm_perf.c | 359 +++++++++------------------------------
 1 file changed, 84 insertions(+), 275 deletions(-)
app/test/test_lpm_perf.c +1 -0
index c8e70ec89ff5..a1485e74e77f 100644
@@ -23,6 +23,7 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
+static uint32_t single_insert;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 @@ -430,24 +431,21 @@ test_lpm_rcu_qsbr_writer(void *arg)

{

app/test/test_lpm_perf.c +3 -1
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
+	bool single_writer = (single_insert == NUM_LDEPTH_ROUTE_ENTRIES) ?
+				true : false;
+	uint8_t pos_core = (uint8_t)((uintptr_t)arg);

app/test/test_lpm_perf.c test_lpm_rcu_qsbr_writer(void *arg) · line 440

Quoted 10 lines
app/test/test_lpm_perf.c +2 -7
 -	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
-		si = 0;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-	} else {
-		si = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES;
-	}
+	si = pos_core * single_insert;
+	ei = si + single_insert;

In this case, given that you are doing
"single_insert = NUM_LDEPTH_ROUTE_ENTRIES / j;"
below, the number of ldepth_routes must be a multiple of the number of writers, so some number of routes can be skipped in the opposite case. Consider something like:

Quoted 5 lines

number_of_writers = j;
...
si = (pos_core * NUM_LDEPTH_ROUTE_ENTRIES)/number_of_writers; ei = ((pos_core + 1 ) * NUM_LDEPTH_ROUTE_ENTRIES)/number_of_writers;

Yes, agreed some routes can be skipped. I will update the patch with the above changes. Thanks!

Quoted 402 lines
Quoted 398 lines
app/test/test_lpm_perf.c +8 -4
 	/* Measure add/delete. */
 	begin = rte_rdtsc_precise();
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
@@ -455,19 +453,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
   		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (!single_writer)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 	}
 @@ -478,22 +479,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	return 0;

error:

app/test/test_lpm_perf.c +70 -252
-	pthread_mutex_unlock(&lpm_mutex);
+	if (!single_writer)
+		pthread_mutex_unlock(&lpm_mutex);
 	return -1;
 }
   /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
   	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -506,273 +509,79 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 -	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		/* Calculate writes by each writer */
+		single_insert = NUM_LDEPTH_ROUTE_ENTRIES / j;
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
 -	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 -	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 -	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 -	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 -	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)i,
+						enabled_core_ids[i]);
 -	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 -		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
   	return 0;
 @@ -948,10 +757,10 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 -	if (test_lpm_rcu_perf() < 0)
+	if (test_lpm_rcu_perf_multi_writer(0) < 0)
 		return -1;
 -	if (test_lpm_rcu_perf_multi_writer() < 0)
+	if (test_lpm_rcu_perf_multi_writer(1) < 0)
 		return -1;
   	return 0;
Signature · 2 lines
Regards,
Vladimir
#42 lore raw

v5 cover test/lpm: fix rcu qsbr perf test

In reply to Dharmik Thakkar

Fix LPM adds, LPM deletes, and cycle calculation.
Return error if LPM add/delete fails in multi-writer test. Return error if single or multi writer test fails
Remove redundant error checking for readers.
Combine single and multi threaded test cases to avoid code duplication.
---
v5:
- Update logic for routes inserted by each writer in multi writer test to avoid possibility of routes getting skipped

v4:
- Return error if rcu qsbr test fails
- Improve multi writer test to enable more than 2 writers

v3:
- Add 'goto error'
- Remove unnecessary if statement

v2:
- Add more details about the fix to the commit message
- Replace hard coded values with an enum
- Remove lock acquire/release for single writer

Dharmik Thakkar (4):
test/lpm: fix cycle calculation in rcu qsbr perf test/lpm: return error on failure in rcu qsbr perf test/lpm: remove error checking in rcu qsbr perf test/lpm: avoid code duplication in rcu qsbr perf

 app/test/test_lpm_perf.c | 380 +++++++++------------------------------
 1 file changed, 88 insertions(+), 292 deletions(-)
Signature · 1 line
2.17.1
#43 lore raw

PATCH v5 1/4 test/lpm: fix cycle calculation in rcu qsbr perf

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Since, rcu qsbr tests run for 'RCU_ITERATIONS' and not 'ITERATIONS', replace 'ITERATIONS' with 'RCU_ITERATIONS' for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES.
For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro 'TOTAL_WRITES' and furthermore, 'g_writes' has been removed since it is always a fixed value equal to TOTAL_WRITES.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 45 ++++++++++++++--------------------------
 1 file changed, 16 insertions(+), 29 deletions(-)
app/test/test_lpm_perf.c +16 -29
index c5a238b9d1e8..45164b23214b 100644
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
-static uint64_t gwrites;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;
 #define NUM_ROUTE_ENTRIES num_route_entries
 #define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries
 
+#define TOTAL_WRITES (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES)
+
 enum {
 	IP_CLASS_A,
 	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
 	/* 2 writer threads are used */
 	if (core_id % 2 == 0) {
 		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	total_cycles = rte_rdtsc_precise() - begin;
 
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
 	return 0;
 }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,14 +544,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	/* Wait and check return value from reader threads */
 	writer_done = 1;
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
 	writer_done = 0;
 	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
 	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
 		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -711,11 +700,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
@@ -771,11 +759,10 @@ test_lpm_rcu_perf(void)
 	}
 	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
 	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
 	/* Wait and check return value from reader threads */
Signature · 1 line
2.17.1
Acked-by on v5 1/4
#44 lore raw
Quoted 144 lines · Dharmik Thakkar

Fix incorrect calculations for LPM adds, LPM deletes, and average cycles in RCU QSBR perf tests

Since, rcu qsbr tests run for 'RCU_ITERATIONS' and not 'ITERATIONS', replace 'ITERATIONS' with 'RCU_ITERATIONS' for calculating adds, deletes, and cycles.

Also, for multi-writer perf test, each writer only writes half of NUM_LDEPTH_ROUTE_ENTRIES.
For 2 writers, total adds (or deletes) should be (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES) instead of (2 * RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES).

Since, for both the single and multi writer tests, total adds/deletes is equal to (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES), this has been replaced with a macro 'TOTAL_WRITES' and furthermore, 'g_writes' has been removed since it is always a fixed value equal to TOTAL_WRITES.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
  app/test/test_lpm_perf.c | 45 ++++++++++++++--------------------------
  1 file changed, 16 insertions(+), 29 deletions(-)
app/test/test_lpm_perf.c +13 -26
index c5a238b9d1e8..45164b23214b 100644
@@ -23,7 +23,6 @@ static struct rte_rcu_qsbr *rv;
  static volatile uint8_t writer_done;
  static volatile uint32_t thr_id;
  static uint64_t gwrite_cycles;
-static uint64_t gwrites;
  /* LPM APIs are not thread safe, use mutex to provide thread safety */
  static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -60,6 +59,8 @@ static uint32_t num_ldepth_route_entries;
  #define NUM_ROUTE_ENTRIES num_route_entries
  #define NUM_LDEPTH_ROUTE_ENTRIES num_ldepth_route_entries
 
+#define TOTAL_WRITES (RCU_ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES)
+
  enum {
  	IP_CLASS_A,
  	IP_CLASS_B,
@@ -432,7 +433,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
  	uint8_t core_id = (uint8_t)((uintptr_t)arg);
  	uint32_t next_hop_add = 0xAA;
 
-	RTE_SET_USED(arg);
  	/* 2 writer threads are used */
  	if (core_id % 2 == 0) {
  		si = 0;
@@ -472,9 +472,6 @@ test_lpm_rcu_qsbr_writer(void *arg)
  	total_cycles = rte_rdtsc_precise() - begin;
 
  	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
-	__atomic_fetch_add(&gwrites,
-			2 * NUM_LDEPTH_ROUTE_ENTRIES * RCU_ITERATIONS,
-			__ATOMIC_RELAXED);
 
  	return 0;
  }
@@ -528,7 +525,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
  	writer_done = 0;
  	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
 
  	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
@@ -548,14 +544,11 @@ test_lpm_rcu_perf_multi_writer(void)
  		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
  			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
  	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
  	/* Wait and check return value from reader threads */
  	writer_done = 1;
@@ -581,7 +574,6 @@ test_lpm_rcu_perf_multi_writer(void)
 
  	writer_done = 0;
  	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&gwrites, 0, __ATOMIC_RELAXED);
  	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
  	/* Launch reader threads */
@@ -600,14 +592,11 @@ test_lpm_rcu_perf_multi_writer(void)
  		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
  			goto error;
 
-	printf("Total LPM Adds: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		2 * ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
  	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED) /
-			__atomic_load_n(&gwrites, __ATOMIC_RELAXED)
-		);
+		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+		/ TOTAL_WRITES);
 
  	writer_done = 1;
  	/* Wait and check return value from reader threads */
@@ -711,11 +700,10 @@ test_lpm_rcu_perf(void)
  	}
  	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
  	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
  	writer_done = 1;
  	/* Wait and check return value from reader threads */
@@ -771,11 +759,10 @@ test_lpm_rcu_perf(void)
  	}
  	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);

app/test/test_lpm_perf.c test_lpm_rcu_perf(void) · line 768

Quoted 10 lines
app/test/test_lpm_perf.c +3 -3
-	printf("Total LPM Deletes: %d\n",
-		ITERATIONS * NUM_LDEPTH_ROUTE_ENTRIES);
+	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
  	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / (NUM_LDEPTH_ROUTE_ENTRIES * ITERATIONS));
+		(double)total_cycles / TOTAL_WRITES);
 
  	writer_done = 1;
  	/* Wait and check return value from reader threads */
Signature · 2 lines
Regards,
Vladimir
#45 lore raw

PATCH v5 2/4 test/lpm: return error on failure in rcu qsbr perf

In reply to Dharmik Thakkar

Return error if Add/Delete fail in multiwriter perf test

Return error if single or multi writer test fails

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)
app/test/test_lpm_perf.c +10 -2
index 45164b23214b..873ecf511c97 100644
@@ -453,6 +453,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					next_hop_add) != 0) {
 				printf("Failed to add iteration %d, route# %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -464,6 +465,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
+				goto error;
 			}
 			pthread_mutex_unlock(&lpm_mutex);
 		}
@@ -474,6 +476,10 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
 
 	return 0;
+
+error:
+	pthread_mutex_unlock(&lpm_mutex);
+	return -1;
 }
 
 /*
@@ -947,9 +953,11 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	test_lpm_rcu_perf();
+	if (test_lpm_rcu_perf() < 0)
+		return -1;
 
-	test_lpm_rcu_perf_multi_writer();
+	if (test_lpm_rcu_perf_multi_writer() < 0)
+		return -1;
 
 	return 0;
 }
Signature · 1 line
2.17.1
Acked-by on v5 2/4
#46 lore raw

Re: [dpdk-dev] [PATCH v5 2/4] test/lpm: return error on failure in rcu qsbr perf

Quoted 51 lines · Dharmik Thakkar

Return error if Add/Delete fail in multiwriter perf test

Return error if single or multi writer test fails

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
  app/test/test_lpm_perf.c | 12 ++++++++++--
  1 file changed, 10 insertions(+), 2 deletions(-)
app/test/test_lpm_perf.c +6 -0
index 45164b23214b..873ecf511c97 100644
@@ -453,6 +453,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
  					next_hop_add) != 0) {
  				printf("Failed to add iteration %d, route# %d\n",
  					i, j);
+				goto error;
  			}
  			pthread_mutex_unlock(&lpm_mutex);
  		}
@@ -464,6 +465,7 @@ test_lpm_rcu_qsbr_writer(void *arg)
  				large_ldepth_route_table[j].depth) != 0) {
  				printf("Failed to delete iteration %d, route# %d\n",
  					i, j);
+				goto error;
  			}
  			pthread_mutex_unlock(&lpm_mutex);
  		}
@@ -474,6 +476,10 @@ test_lpm_rcu_qsbr_writer(void *arg)
  	__atomic_fetch_add(&gwrite_cycles, total_cycles, __ATOMIC_RELAXED);
 
  	return 0;
+
+error:
+	pthread_mutex_unlock(&lpm_mutex);
+	return -1;
  }
 
  /*
@@ -947,9 +953,11 @@ test_lpm_perf(void)
  	rte_lpm_delete_all(lpm);
  	rte_lpm_free(lpm);
 

app/test/test_lpm_perf.c test_lpm_perf(void) · line 963

Quoted 10 lines
app/test/test_lpm_perf.c +4 -2
-	test_lpm_rcu_perf();
+	if (test_lpm_rcu_perf() < 0)
+		return -1;
 
-	test_lpm_rcu_perf_multi_writer();
+	if (test_lpm_rcu_perf_multi_writer() < 0)
+		return -1;
 
  	return 0;
  }
Signature · 2 lines
Regards,
Vladimir
#47 lore raw

PATCH v5 3/4 test/lpm: remove error checking in rcu qsbr perf

In reply to Dharmik Thakkar

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
 app/test/test_lpm_perf.c | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c +8 -13
index 873ecf511c97..c8e70ec89ff5 100644
@@ -556,11 +556,10 @@ test_lpm_rcu_perf_multi_writer(void)
 		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
 		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
 	writer_done = 1;
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -605,10 +604,9 @@ test_lpm_rcu_perf_multi_writer(void)
 		/ TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
@@ -712,10 +710,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 	rte_free(rv);
@@ -771,11 +768,9 @@ test_lpm_rcu_perf(void)
 		(double)total_cycles / TOTAL_WRITES);
 
 	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
 	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
 	rte_lpm_free(lpm);
 
Signature · 1 line
2.17.1
Acked-by on v5 3/4
#48 lore raw

Re: [dpdk-dev] [PATCH v5 3/4] test/lpm: remove error checking in rcu qsbr perf

Quoted 63 lines · Dharmik Thakkar

Remove redundant error checking for reader threads since they never return error.

Fixes eff30b59cc2e ("test/lpm: add RCU performance tests")
---
  app/test/test_lpm_perf.c | 21 ++++++++-------------
  1 file changed, 8 insertions(+), 13 deletions(-)
app/test/test_lpm_perf.c +6 -9
index 873ecf511c97..c8e70ec89ff5 100644
@@ -556,11 +556,10 @@ test_lpm_rcu_perf_multi_writer(void)
  		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
  		/ TOTAL_WRITES);
 
-	/* Wait and check return value from reader threads */
  	writer_done = 1;
+	/* Wait until all readers have exited */
  	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
  	rte_lpm_free(lpm);
  	rte_free(rv);
@@ -605,10 +604,9 @@ test_lpm_rcu_perf_multi_writer(void)
  		/ TOTAL_WRITES);
 
  	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
  	for (i = 2; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
  	rte_lpm_free(lpm);
 
@@ -712,10 +710,9 @@ test_lpm_rcu_perf(void)
  		(double)total_cycles / TOTAL_WRITES);
 
  	writer_done = 1;
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
  	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
  	rte_lpm_free(lpm);
  	rte_free(rv);
@@ -771,11 +768,9 @@ test_lpm_rcu_perf(void)
  		(double)total_cycles / TOTAL_WRITES);
 
  	writer_done = 1;

app/test/test_lpm_perf.c test_lpm_rcu_perf(void) · line 776

Quoted 10 lines
app/test/test_lpm_perf.c +2 -4
-	/* Wait and check return value from reader threads */
+	/* Wait until all readers have exited */
  	for (i = 0; i < num_cores; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			printf("Warning: lcore %u not finished.\n",
-				enabled_core_ids[i]);
+		rte_eal_wait_lcore(enabled_core_ids[i]);
 
  	rte_lpm_free(lpm);
 
Signature · 2 lines
Regards,
Vladimir
#49 lore raw

PATCH v5 4/4 test/lpm: avoid code duplication in rcu qsbr perf

In reply to Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

Also, enable support for more than 2 writers

---
 app/test/test_lpm_perf.c | 356 +++++++++------------------------------
 1 file changed, 81 insertions(+), 275 deletions(-)
app/test/test_lpm_perf.c +81 -275
index c8e70ec89ff5..2bed00d0648f 100644
@@ -23,6 +23,7 @@ static struct rte_rcu_qsbr *rv;
 static volatile uint8_t writer_done;
 static volatile uint32_t thr_id;
 static uint64_t gwrite_cycles;
+static uint32_t num_writers;
 /* LPM APIs are not thread safe, use mutex to provide thread safety */
 static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -430,24 +431,19 @@ test_lpm_rcu_qsbr_writer(void *arg)
 {
 	unsigned int i, j, si, ei;
 	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
 	uint32_t next_hop_add = 0xAA;
+	uint8_t pos_core = (uint8_t)((uintptr_t)arg);
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
-		si = 0;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-	} else {
-		si = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES;
-	}
+	si = (pos_core * NUM_LDEPTH_ROUTE_ENTRIES) / num_writers;
+	ei = ((pos_core + 1) * NUM_LDEPTH_ROUTE_ENTRIES) / num_writers;
 
 	/* Measure add/delete. */
 	begin = rte_rdtsc_precise();
 	for (i = 0; i < RCU_ITERATIONS; i++) {
 		/* Add all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
 					large_ldepth_route_table[j].depth,
 					next_hop_add) != 0) {
@@ -455,19 +451,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 
 		/* Delete all the entries */
 		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_lock(&lpm_mutex);
 			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
 				large_ldepth_route_table[j].depth) != 0) {
 				printf("Failed to delete iteration %d, route# %d\n",
 					i, j);
 				goto error;
 			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_unlock(&lpm_mutex);
 		}
 	}
 
@@ -478,22 +477,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
 	return 0;
 
 error:
-	pthread_mutex_unlock(&lpm_mutex);
+	if (num_writers > 1)
+		pthread_mutex_unlock(&lpm_mutex);
 	return -1;
 }
 
 /*
  * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
  */
 static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
 {
 	struct rte_lpm_config config;
 	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
 	uint16_t core_id;
 	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
 	if (rte_lcore_count() < 3) {
 		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -506,273 +507,78 @@ test_lpm_rcu_perf_multi_writer(void)
 		num_cores++;
 	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		num_writers = j;
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
 				goto error;
 			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)i,
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
 				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
 	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
 	return 0;
 
@@ -948,10 +754,10 @@ test_lpm_perf(void)
 	rte_lpm_delete_all(lpm);
 	rte_lpm_free(lpm);
 
-	if (test_lpm_rcu_perf() < 0)
+	if (test_lpm_rcu_perf_multi_writer(0) < 0)
 		return -1;
 
-	if (test_lpm_rcu_perf_multi_writer() < 0)
+	if (test_lpm_rcu_perf_multi_writer(1) < 0)
 		return -1;
 
 	return 0;
Signature · 1 line
2.17.1
Acked-by on v5 4/4
#50 lore raw

Re: [dpdk-dev] [PATCH v5 4/4] test/lpm: avoid code duplication in rcu qsbr perf

Quoted 447 lines · Dharmik Thakkar

Avoid code duplication by combining single and multi threaded tests

Also, enable support for more than 2 writers

---
  app/test/test_lpm_perf.c | 356 +++++++++------------------------------
  1 file changed, 81 insertions(+), 275 deletions(-)
app/test/test_lpm_perf.c +79 -273
index c8e70ec89ff5..2bed00d0648f 100644
@@ -23,6 +23,7 @@ static struct rte_rcu_qsbr *rv;
  static volatile uint8_t writer_done;
  static volatile uint32_t thr_id;
  static uint64_t gwrite_cycles;
+static uint32_t num_writers;
  /* LPM APIs are not thread safe, use mutex to provide thread safety */
  static pthread_mutex_t lpm_mutex = PTHREAD_MUTEX_INITIALIZER;
 
@@ -430,24 +431,19 @@ test_lpm_rcu_qsbr_writer(void *arg)
  {
  	unsigned int i, j, si, ei;
  	uint64_t begin, total_cycles;
-	uint8_t core_id = (uint8_t)((uintptr_t)arg);
  	uint32_t next_hop_add = 0xAA;
+	uint8_t pos_core = (uint8_t)((uintptr_t)arg);
 
-	/* 2 writer threads are used */
-	if (core_id % 2 == 0) {
-		si = 0;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-	} else {
-		si = NUM_LDEPTH_ROUTE_ENTRIES / 2;
-		ei = NUM_LDEPTH_ROUTE_ENTRIES;
-	}
+	si = (pos_core * NUM_LDEPTH_ROUTE_ENTRIES) / num_writers;
+	ei = ((pos_core + 1) * NUM_LDEPTH_ROUTE_ENTRIES) / num_writers;
 
  	/* Measure add/delete. */
  	begin = rte_rdtsc_precise();
  	for (i = 0; i < RCU_ITERATIONS; i++) {
  		/* Add all the entries */
  		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_lock(&lpm_mutex);
  			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
  					large_ldepth_route_table[j].depth,
  					next_hop_add) != 0) {
@@ -455,19 +451,22 @@ test_lpm_rcu_qsbr_writer(void *arg)
  					i, j);
  				goto error;
  			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_unlock(&lpm_mutex);
  		}
 
  		/* Delete all the entries */
  		for (j = si; j < ei; j++) {
-			pthread_mutex_lock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_lock(&lpm_mutex);
  			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
  				large_ldepth_route_table[j].depth) != 0) {
  				printf("Failed to delete iteration %d, route# %d\n",
  					i, j);
  				goto error;
  			}
-			pthread_mutex_unlock(&lpm_mutex);
+			if (num_writers > 1)
+				pthread_mutex_unlock(&lpm_mutex);
  		}
  	}
 
@@ -478,22 +477,24 @@ test_lpm_rcu_qsbr_writer(void *arg)
  	return 0;
 
  error:
-	pthread_mutex_unlock(&lpm_mutex);
+	if (num_writers > 1)
+		pthread_mutex_unlock(&lpm_mutex);
  	return -1;
  }
 
  /*
   * Functional test:
- * 2 writers, rest are readers
+ * 1/2 writers, rest are readers
   */
  static int
-test_lpm_rcu_perf_multi_writer(void)
+test_lpm_rcu_perf_multi_writer(uint8_t use_rcu)
  {
  	struct rte_lpm_config config;
  	size_t sz;
-	unsigned int i;
+	unsigned int i, j;
  	uint16_t core_id;
  	struct rte_lpm_rcu_config rcu_cfg = {0};
+	int (*reader_f)(void *arg) = NULL;
 
  	if (rte_lcore_count() < 3) {
  		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 3\n");
@@ -506,273 +507,78 @@ test_lpm_rcu_perf_multi_writer(void)
  		num_cores++;
  	}
 
-	printf("\nPerf test: 2 writers, %d readers, RCU integration enabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
-
-	/* Test without RCU integration */
-	printf("\nPerf test: 2 writers, %d readers, RCU integration disabled\n",
-		num_cores - 2);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	writer_done = 0;
-	__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Launch writer threads */
-	for (i = 0; i < 2; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
-					(void *)(uintptr_t)i,
-					enabled_core_ids[i]);
-
-	/* Wait for writer threads */
-	for (i = 0; i < 2; i++)
-		if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
-			goto error;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %"PRIu64" cycles\n",
-		__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
-		/ TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 2; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-
-	return 0;
-
-error:
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	rte_eal_mp_wait_lcore();
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-
-	return -1;
-}
-
-/*
- * Functional test:
- * Single writer, rest are readers
- */
-static int
-test_lpm_rcu_perf(void)
-{
-	struct rte_lpm_config config;
-	uint64_t begin, total_cycles;
-	size_t sz;
-	unsigned int i, j;
-	uint16_t core_id;
-	uint32_t next_hop_add = 0xAA;
-	struct rte_lpm_rcu_config rcu_cfg = {0};
-
-	if (rte_lcore_count() < 2) {
-		printf("Not enough cores for lpm_rcu_perf_autotest, expecting at least 2\n");
-		return TEST_SKIPPED;
-	}
-
-	num_cores = 0;
-	RTE_LCORE_FOREACH_WORKER(core_id) {
-		enabled_core_ids[num_cores] = core_id;
-		num_cores++;
-	}
-
-	printf("\nPerf test: 1 writer, %d readers, RCU integration enabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
-
-	/* Init RCU variable */
-	sz = rte_rcu_qsbr_get_memsize(num_cores);
-	rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
-						RTE_CACHE_LINE_SIZE);
-	rte_rcu_qsbr_init(rv, num_cores);
-
-	rcu_cfg.v = rv;
-	/* Assign the RCU variable to LPM */
-	if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
-		printf("RCU variable assignment failed\n");
-		goto error;
-	}
-
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
-
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_rcu_qsbr_reader, NULL,
-					enabled_core_ids[i]);
-
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
-
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
+	for (j = 1; j < 3; j++) {
+		if (use_rcu)
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration enabled\n", j, num_cores - j);
+		else
+			printf("\nPerf test: %d writer(s), %d reader(s),"
+			       " RCU integration disabled\n", j, num_cores - j);
+
+		num_writers = j;
+
+		/* Create LPM table */
+		config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
+		config.flags = 0;
+		lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
+		TEST_LPM_ASSERT(lpm != NULL);
+
+		/* Init RCU variable */
+		if (use_rcu) {
+			sz = rte_rcu_qsbr_get_memsize(num_cores);
+			rv = (struct rte_rcu_qsbr *)rte_zmalloc("rcu0", sz,
+							RTE_CACHE_LINE_SIZE);
+			rte_rcu_qsbr_init(rv, num_cores);
+
+			rcu_cfg.v = rv;
+			/* Assign the RCU variable to LPM */
+			if (rte_lpm_rcu_qsbr_add(lpm, &rcu_cfg) != 0) {
+				printf("RCU variable assignment failed\n");
  				goto error;
  			}
-	}
-	total_cycles = rte_rdtsc_precise() - begin;
 
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
+			reader_f = test_lpm_rcu_qsbr_reader;
+		} else
+			reader_f = test_lpm_reader;
 
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
-	rte_free(rv);
-	lpm = NULL;
-	rv = NULL;
+		writer_done = 0;
+		__atomic_store_n(&gwrite_cycles, 0, __ATOMIC_RELAXED);
 
-	/* Test without RCU integration */
-	printf("\nPerf test: 1 writer, %d readers, RCU integration disabled\n",
-		num_cores);
-
-	/* Create LPM table */
-	config.max_rules = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.number_tbl8s = NUM_LDEPTH_ROUTE_ENTRIES;
-	config.flags = 0;
-	lpm = rte_lpm_create(__func__, SOCKET_ID_ANY, &config);
-	TEST_LPM_ASSERT(lpm != NULL);
+		__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
 
-	writer_done = 0;
-	__atomic_store_n(&thr_id, 0, __ATOMIC_SEQ_CST);
+		/* Launch reader threads */
+		for (i = j; i < num_cores; i++)
+			rte_eal_remote_launch(reader_f, NULL,
+						enabled_core_ids[i]);
 
-	/* Launch reader threads */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_remote_launch(test_lpm_reader, NULL,
-					enabled_core_ids[i]);
+		/* Launch writer threads */
+		for (i = 0; i < j; i++)
+			rte_eal_remote_launch(test_lpm_rcu_qsbr_writer,
+						(void *)(uintptr_t)i,
+						enabled_core_ids[i]);
 
-	/* Measure add/delete. */
-	begin = rte_rdtsc_precise();
-	for (i = 0; i < RCU_ITERATIONS; i++) {
-		/* Add all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_add(lpm, large_ldepth_route_table[j].ip,
-					large_ldepth_route_table[j].depth,
-					next_hop_add) != 0) {
-				printf("Failed to add iteration %d, route# %d\n",
-					i, j);
+		/* Wait for writer threads */
+		for (i = 0; i < j; i++)
+			if (rte_eal_wait_lcore(enabled_core_ids[i]) < 0)
  				goto error;
-			}
 
-		/* Delete all the entries */
-		for (j = 0; j < NUM_LDEPTH_ROUTE_ENTRIES; j++)
-			if (rte_lpm_delete(lpm, large_ldepth_route_table[j].ip,
-				large_ldepth_route_table[j].depth) != 0) {
-				printf("Failed to delete iteration %d, route# %d\n",
-					i, j);
-				goto error;
-			}
+		printf("Total LPM Adds: %d\n", TOTAL_WRITES);
+		printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
+		printf("Average LPM Add/Del: %"PRIu64" cycles\n",
+			__atomic_load_n(&gwrite_cycles, __ATOMIC_RELAXED)
+			/ TOTAL_WRITES);
+
+		writer_done = 1;
+		/* Wait until all readers have exited */
+		for (i = j; i < num_cores; i++)
+			rte_eal_wait_lcore(enabled_core_ids[i]);
+
+		rte_lpm_free(lpm);
+		rte_free(rv);
+		lpm = NULL;
+		rv = NULL;
  	}
-	total_cycles = rte_rdtsc_precise() - begin;
-
-	printf("Total LPM Adds: %d\n", TOTAL_WRITES);
-	printf("Total LPM Deletes: %d\n", TOTAL_WRITES);
-	printf("Average LPM Add/Del: %g cycles\n",
-		(double)total_cycles / TOTAL_WRITES);
-
-	writer_done = 1;
-	/* Wait until all readers have exited */
-	for (i = 0; i < num_cores; i++)
-		rte_eal_wait_lcore(enabled_core_ids[i]);
-
-	rte_lpm_free(lpm);
 
  	return 0;
 
@@ -948,10 +754,10 @@ test_lpm_perf(void)
  	rte_lpm_delete_all(lpm);
  	rte_lpm_free(lpm);

app/test/test_lpm_perf.c test_lpm_perf(void) · line 763

Quoted 10 lines
app/test/test_lpm_perf.c +2 -2
 
-	if (test_lpm_rcu_perf() < 0)
+	if (test_lpm_rcu_perf_multi_writer(0) < 0)
  		return -1;
 
-	if (test_lpm_rcu_perf_multi_writer() < 0)
+	if (test_lpm_rcu_perf_multi_writer(1) < 0)
  		return -1;
 
  	return 0;
Signature · 2 lines
Regards,
Vladimir
#51 lore raw

Re: [dpdk-dev] [PATCH v5 0/4] test/lpm: fix rcu qsbr perf test

In reply to Dharmik Thakkar
Quoted 6 lines · Dharmik Thakkar

Fix LPM adds, LPM deletes, and cycle calculation.
Return error if LPM add/delete fails in multi-writer test. Return error if single or multi writer test fails
Remove redundant error checking for readers.
Combine single and multi threaded test cases to avoid code duplication.

Series applied, thanks Dharmik.

Signature · 1 line
David Marchand