Skip to contentSkip to conversation

selftests/resctrl: increase L3 cache test coverage

PATCH Superseded Richard Cheng 2 participants 8 messages Jun 8 – Aug 10

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover selftests/resctrl: increase L3 cache test coverage

The resctrl selftests check L3 cache isolation (L3_CAT) and occupancy accuracy (CMT), but some basic L3 behaviour is untested: that an allocation actually bounds a group's cache use, that invalid CBMs are rejected, and that info/L3/bit_usage reports allocations correctly.

Add three self-contained tests for these. Each gates on the feature it needs, so it runs wherever that feature exists and skips otherwise. They were exercised on an arm64 MPAM platform.

Richard Cheng (3):
selftests/resctrl: Add L3_CAT_OCCUP test to verify CAT bounds
occupancy
selftests/resctrl: Add L3_CAT_VALIDATE to check invalid CBMs selftests/resctrl: Add L3_BIT_USAGE to check bit_usage tracks
allocation

 tools/testing/selftests/resctrl/cat_test.c    | 433 ++++++++++++++++++
 tools/testing/selftests/resctrl/cmt_test.c    |   3 -
 tools/testing/selftests/resctrl/resctrl.h     |   6 +
 .../testing/selftests/resctrl/resctrl_tests.c |   3 +
 4 files changed, 442 insertions(+), 3 deletions(-)
Signature · 1 line
2.43.0
#1 lore raw

PATCH 1/3 selftests/resctrl: Add L3_CAT_OCCUP test to verify CAT bounds occupancy

L3_CAT needs a CPU-exclusive cache portion, so it's skipped when MPAM reports every CBM bit as shareable, leaving L3 allocation untested. CMT only checks occupancy accuracy, not that a CBM actually limits it.

L3_CAT_OCCUP gives a group a small CBM, run a workload spanning the whole cache, and check every occupancy sample stays within the allocation. An unenforced CBM would instead let occupancy grow to the full cache.

Move CON_MON_LCC_OCCUP_PATH to resctrl.h to share it with CMT.

---
 tools/testing/selftests/resctrl/cat_test.c    | 201 ++++++++++++++++++
 tools/testing/selftests/resctrl/cmt_test.c    |   3 -
 tools/testing/selftests/resctrl/resctrl.h     |   4 +
 .../testing/selftests/resctrl/resctrl_tests.c |   1 +
 4 files changed, 206 insertions(+), 3 deletions(-)
tools/testing/selftests/resctrl/cat_test.c +201 -0
index f00b622c1460..16a947f1ed16 100644
@@ -402,3 +402,204 @@ struct resctrl_test l2_noncont_cat_test = {
 	.feature_check = noncont_cat_feature_check,
 	.run_test = noncont_cat_run_test,
 };
+
+/*
+ * L3_CAT_OCCUP - Verify that a CAT allocation bounds cache occupancy.
+ *
+ * Unlike L3_CAT (which measures interference between groups and needs an
+ * exclusive cache portion), this test gives a control group a strict subset
+ * of the CBM, then runs a benchmark whose buffer spans the *whole* cache -
+ * i.e. much larger than the allocation. With CAT enforced, the group can
+ * only keep its allocated portion resident, so llc_occupancy settles near
+ * the allocation size. Without enforcement occupancy would instead climb
+ * towards the full cache. This works even when all CBM bits are shareable
+ * (where L3_CAT is skipped).
+ */
+#define CAT_OCCUP_RESULT_FILE		"result_cat_occup"
+#define CAT_OCCUP_NUM_OF_RUNS		5
+
+static int cat_occup_cpu;
+
+static int cat_occup_init(const struct resctrl_val_param *param, int domain_id)
+{
+	char schemata[64];
+
+	sprintf(llc_occup_path, CON_MON_LCC_OCCUP_PATH, RESCTRL_PATH,
+		param->ctrlgrp, domain_id);
+
+	/*
+	 * Confine the benchmark to the allocated portion *before* it starts
+	 * filling (resctrl_val() calls init() before forking the benchmark),
+	 * so occupancy reflects the restricted CBM from the first sample.
+	 */
+	snprintf(schemata, sizeof(schemata), "%lx", param->mask);
+
+	return write_schemata(param->ctrlgrp, schemata, cat_occup_cpu, "L3");
+}
+
+static int cat_occup_setup(const struct resctrl_test *test,
+			   const struct user_params *uparams,
+			   struct resctrl_val_param *p)
+{
+	if (p->num_of_runs >= CAT_OCCUP_NUM_OF_RUNS)
+		return END_OF_TESTS;
+
+	p->num_of_runs++;
+
+	return 0;
+}
+
+static int cat_occup_measure(const struct user_params *uparams,
+			     struct resctrl_val_param *param, pid_t bm_pid)
+{
+	sleep(1);
+	return measure_llc_resctrl(param->filename, bm_pid);
+}
+
+static int cat_occup_check_results(struct resctrl_val_param *param,
+				   size_t alloc_span, size_t cache_size,
+				   int no_of_bits)
+{
+	char *token_array[8], temp[512];
+	unsigned long occu, max_occu = 0, ceiling, floor;
+	int runs = 0;
+	int fail = 0;
+	FILE *fp;
+
+	/*
+	 * Check every sample, not an average: CAT is a hard limit, so a single
+	 * sample above the allocation is a real violation that an average
+	 * could mask.
+	 */
+	ceiling = alloc_span + (cache_size - alloc_span) / 2;
+	floor = alloc_span / 2;
+
+	ksft_print_msg("Checking for pass/fail\n");
+	fp = fopen(param->filename, "r");
+	if (!fp) {
+		ksft_perror("Error in opening file");
+
+		return -1;
+	}
+
+	while (fgets(temp, sizeof(temp), fp)) {
+		char *token = strtok(temp, ":\t");
+		int fields = 0;
+
+		while (token) {
+			token_array[fields++] = token;
+			token = strtok(NULL, ":\t");
+		}
+
+		/* Field 3 is the resctrl-reported llc_occupancy value. */
+		occu = strtoul(token_array[3], NULL, 0);
+		runs++;
+
+		if (occu > max_occu)
+			max_occu = occu;
+
+		if (occu > ceiling) {
+			ksft_print_msg("Fail: run %d occupancy %lu exceeds ceiling %lu\n",
+				       runs, occu, ceiling);
+			fail = 1;
+		}
+	}
+	fclose(fp);
+
+	if (!runs) {
+		ksft_print_msg("No occupancy samples collected\n");
+		return -1;
+	}
+
+	if (max_occu < floor) {
+		ksft_print_msg("Fail: peak occupancy %lu never reached floor %lu\n",
+			       max_occu, floor);
+		fail = 1;
+	}
+
+	ksft_print_msg("%s CAT confines occupancy to the allocated %d-bit portion\n",
+		       fail ? "Fail:" : "Pass:", no_of_bits);
+	ksft_print_msg("occupancy=%lu alloc=%zu full=%zu ceiling=%lu floor=%lu\n",
+		       max_occu, alloc_span, cache_size, ceiling, floor);
+
+	return fail;
+}
+
+static void cat_occup_test_cleanup(void)
+{
+	remove(CAT_OCCUP_RESULT_FILE);
+}
+
+static int cat_occup_run_test(const struct resctrl_test *test,
+			      const struct user_params *uparams)
+{
+	struct fill_buf_param fill_buf = {};
+	unsigned long cache_total_size = 0;
+	unsigned long full_mask;
+	int count_of_bits;
+	size_t alloc_span;
+	int n, ret;
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = get_cache_size(uparams->cpu, test->resource, &cache_total_size);
+	if (ret)
+		return ret;
+	ksft_print_msg("Cache size :%lu\n", cache_total_size);
+
+	count_of_bits = count_bits(full_mask);
+
+	/*
+	 * Allocate a strict subset of the cache so the benchmark buffer
+	 * is larger than the allocation and CAT has something to enforce.
+	 */
+	n = uparams->bits ? : count_of_bits / 2;
+	if (n < 1 || n >= count_of_bits) {
+		ksft_print_msg("Invalid number of CBM bits %d, expected 1 to %d\n",
+			       n, count_of_bits - 1);
+		return -1;
+	}
+
+	struct resctrl_val_param param = {
+		.ctrlgrp	= "c1",
+		.filename	= CAT_OCCUP_RESULT_FILE,
+		.mask		= ~(full_mask << n) & full_mask,
+		.num_of_runs	= 0,
+		.init		= cat_occup_init,
+		.setup		= cat_occup_setup,
+		.measure	= cat_occup_measure,
+	};
+
+	alloc_span = cache_portion_size(cache_total_size, param.mask, full_mask);
+
+	/* Benchmark buffer spans the full cache: larger than the allocation. */
+	fill_buf.buf_size = cache_total_size;
+	fill_buf.memflush = uparams->fill_buf ? uparams->fill_buf->memflush : true;
+	param.fill_buf = &fill_buf;
+	cat_occup_cpu = uparams->cpu;
+
+	remove(param.filename);
+
+	ret = resctrl_val(test, uparams, &param);
+	if (ret)
+		return ret;
+
+	return cat_occup_check_results(&param, alloc_span, cache_total_size, n);
+}
+
+static bool cat_occup_feature_check(const struct resctrl_test *test)
+{
+	return test_resource_feature_check(test) &&
+	       resctrl_mon_feature_exists("L3_MON", "llc_occupancy");
+}
+
+struct resctrl_test l3_cat_occup_test = {
+	.name = "L3_CAT_OCCUP",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = cat_occup_feature_check,
+	.run_test = cat_occup_run_test,
+	.cleanup = cat_occup_test_cleanup,
+};
tools/testing/selftests/resctrl/cmt_test.c +0 -3
index d09e693dc739..ef51daa8061a 100644
@@ -16,9 +16,6 @@
 #define MAX_DIFF		2000000
 #define MAX_DIFF_PERCENT	15
 
-#define CON_MON_LCC_OCCUP_PATH		\
-	"%s/%s/mon_data/mon_L3_%02d/llc_occupancy"
-
 static int cmt_init(const struct resctrl_val_param *param, int domain_id)
 {
 	sprintf(llc_occup_path, CON_MON_LCC_OCCUP_PATH, RESCTRL_PATH,
tools/testing/selftests/resctrl/resctrl.h +4 -0
index afe635b6e48d..ce3abf0bdac2 100644
@@ -31,6 +31,9 @@
 #define PHYS_ID_PATH		"/sys/devices/system/cpu/cpu"
 #define INFO_PATH		"/sys/fs/resctrl/info"
 
+#define CON_MON_LCC_OCCUP_PATH		\
+	"%s/%s/mon_data/mon_L3_%02d/llc_occupancy"
+
 /*
  * CPU vendor IDs
  *
@@ -244,6 +247,7 @@ extern struct resctrl_test mbm_test;
 extern struct resctrl_test mba_test;
 extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
+extern struct resctrl_test l3_cat_occup_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
index dbcd5eea9fbc..324a60818aa1 100644
@@ -19,6 +19,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&mba_test,
 	&cmt_test,
 	&l3_cat_test,
+	&l3_cat_occup_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };
Signature · 1 line
2.43.0
maintainer
#2 lore raw

Re: [PATCH 1/3] selftests/resctrl: Add L3_CAT_OCCUP test to verify CAT bounds occupancy

Hi Richard,

Quoted 3 lines · Richard Cheng

L3_CAT needs a CPU-exclusive cache portion, so it's skipped when MPAM reports every CBM bit as shareable, leaving L3 allocation untested. CMT only checks occupancy accuracy, not that a CBM actually limits it.

hmmm ... CMT test ensures that the LLC occupancy is within a % of the size of the cache allocation. To me this implies that it indeed tests that the CBM limits the allocation, no?

Quoted 21 lines

L3_CAT_OCCUP gives a group a small CBM, run a workload spanning the whole cache, and check every occupancy sample stays within the allocation. An unenforced CBM would instead let occupancy grow to the full cache.

Move CON_MON_LCC_OCCUP_PATH to resctrl.h to share it with CMT.

---
 tools/testing/selftests/resctrl/cat_test.c    | 201 ++++++++++++++++++
 tools/testing/selftests/resctrl/cmt_test.c    |   3 -
 tools/testing/selftests/resctrl/resctrl.h     |   4 +
 .../testing/selftests/resctrl/resctrl_tests.c |   1 +
 4 files changed, 206 insertions(+), 3 deletions(-)
tools/testing/selftests/resctrl/cat_test.c +0 -0
index f00b622c1460..16a947f1ed16 100644

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 410

Quoted 9 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
@@ -402,3 +402,204 @@ struct resctrl_test l2_noncont_cat_test = {
 	.feature_check = noncont_cat_feature_check,
 	.run_test = noncont_cat_run_test,
 };
+
+/*
+ * L3_CAT_OCCUP - Verify that a CAT allocation bounds cache occupancy.
+ *
+ * Unlike L3_CAT (which measures interference between groups and needs an
+ * exclusive cache portion), this test gives a control group a strict subset

Please let comment just refer to what this test does. These comments are unlikely to be updated if L3_CAT ever changes.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 416

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+ * of the CBM, then runs a benchmark whose buffer spans the *whole* cache -
+ * i.e. much larger than the allocation. With CAT enforced, the group can
+ * only keep its allocated portion resident, so llc_occupancy settles near
+ * the allocation size. Without enforcement occupancy would instead climb
+ * towards the full cache. This works even when all CBM bits are shareable
+ * (where L3_CAT is skipped).

hmmm ... these statements state as fact what really depends on system load and interference that this test make no attempt to avoid.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 423

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+ */
+#define CAT_OCCUP_RESULT_FILE		"result_cat_occup"
+#define CAT_OCCUP_NUM_OF_RUNS		5
+
+static int cat_occup_cpu;
+
+static int cat_occup_init(const struct resctrl_val_param *param, int domain_id)

Please take a look at recent resctrl selftest changes that provides more data to the init() that will avoid the cat_occup_cpu global. Even so, why not just use cmt_init() that reduces interference from rest of system to improve chances of workload's cache occupancy to match its cache allocation?

Quoted 17 lines
tools/testing/selftests/resctrl/cat_test.c +17 -0
+{
+	char schemata[64];
+
+	sprintf(llc_occup_path, CON_MON_LCC_OCCUP_PATH, RESCTRL_PATH,
+		param->ctrlgrp, domain_id);
+
+	/*
+	 * Confine the benchmark to the allocated portion *before* it starts
+	 * filling (resctrl_val() calls init() before forking the benchmark),
+	 * so occupancy reflects the restricted CBM from the first sample.
+	 */
+	snprintf(schemata, sizeof(schemata), "%lx", param->mask);
+
+	return write_schemata(param->ctrlgrp, schemata, cat_occup_cpu, "L3");
+}
+
+static int cat_occup_setup(const struct resctrl_test *test,

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 450

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+			   const struct user_params *uparams,
+			   struct resctrl_val_param *p)
+{
+	if (p->num_of_runs >= CAT_OCCUP_NUM_OF_RUNS)
+		return END_OF_TESTS;
+
+	p->num_of_runs++;
+
+	return 0;
+}

cmt_setup()?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 457

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+
+static int cat_occup_measure(const struct user_params *uparams,
+			     struct resctrl_val_param *param, pid_t bm_pid)
+{
+	sleep(1);
+	return measure_llc_resctrl(param->filename, bm_pid);
+}

cmt_measure()?

This test has a lot in common with the existing CMT test since it duplicates cmt_setup(), cmt_measure(), and cmt_feature_check(). From what I can tell it can use cmt_init() also. Could the implementation be simplified by instead considering it a CMT test, move code to cmt_test.c, and avoid all this duplication?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 464

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+
+static int cat_occup_check_results(struct resctrl_val_param *param,
+				   size_t alloc_span, size_t cache_size,
+				   int no_of_bits)
+{
+	char *token_array[8], temp[512];
+	unsigned long occu, max_occu = 0, ceiling, floor;

Please follow kernel coding style (throughout this series) by, in this example, using reverse-fir tree ordering.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 470

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+	int runs = 0;
+	int fail = 0;
+	FILE *fp;
+
+	/*
+	 * Check every sample, not an average: CAT is a hard limit, so a single

hmmm ... "is a hard limit" does not match what the code does

Quoted 24 lines
tools/testing/selftests/resctrl/cat_test.c +24 -0
+	 * sample above the allocation is a real violation that an average
+	 * could mask.
+	 */
+	ceiling = alloc_span + (cache_size - alloc_span) / 2;
+	floor = alloc_span / 2;
+
+	ksft_print_msg("Checking for pass/fail\n");
+	fp = fopen(param->filename, "r");
+	if (!fp) {
+		ksft_perror("Error in opening file");
+
+		return -1;
+	}
+
+	while (fgets(temp, sizeof(temp), fp)) {
+		char *token = strtok(temp, ":\t");
+		int fields = 0;
+
+		while (token) {
+			token_array[fields++] = token;
+			token = strtok(NULL, ":\t");
+		}
+
+		/* Field 3 is the resctrl-reported llc_occupancy value. */

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 504

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+		occu = strtoul(token_array[3], NULL, 0);
+		runs++;
+
+		if (occu > max_occu)
+			max_occu = occu;
+
+		if (occu > ceiling) {
+			ksft_print_msg("Fail: run %d occupancy %lu exceeds ceiling %lu\n",
+				       runs, occu, ceiling);
+			fail = 1;

KSFT_FAIL is available to avoid using magic numbers.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 516

Quoted 12 lines
tools/testing/selftests/resctrl/cat_test.c +12 -0
+		}
+	}
+	fclose(fp);
+
+	if (!runs) {
+		ksft_print_msg("No occupancy samples collected\n");
+		return -1;
+	}
+
+	if (max_occu < floor) {
+		ksft_print_msg("Fail: peak occupancy %lu never reached floor %lu\n",
+			       max_occu, floor);

I think this test should be dropped. We cannot control the environments in which the tests are run and legitimate interference may cause this test to fail without it meaning that there is a bug in resctrl.

Quoted 54 lines
tools/testing/selftests/resctrl/cat_test.c +54 -0
+		fail = 1;
+	}
+
+	ksft_print_msg("%s CAT confines occupancy to the allocated %d-bit portion\n",
+		       fail ? "Fail:" : "Pass:", no_of_bits);
+	ksft_print_msg("occupancy=%lu alloc=%zu full=%zu ceiling=%lu floor=%lu\n",
+		       max_occu, alloc_span, cache_size, ceiling, floor);
+
+	return fail;
+}
+
+static void cat_occup_test_cleanup(void)
+{
+	remove(CAT_OCCUP_RESULT_FILE);
+}
+
+static int cat_occup_run_test(const struct resctrl_test *test,
+			      const struct user_params *uparams)
+{
+	struct fill_buf_param fill_buf = {};
+	unsigned long cache_total_size = 0;
+	unsigned long full_mask;
+	int count_of_bits;
+	size_t alloc_span;
+	int n, ret;
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = get_cache_size(uparams->cpu, test->resource, &cache_total_size);
+	if (ret)
+		return ret;
+	ksft_print_msg("Cache size :%lu\n", cache_total_size);
+
+	count_of_bits = count_bits(full_mask);
+
+	/*
+	 * Allocate a strict subset of the cache so the benchmark buffer
+	 * is larger than the allocation and CAT has something to enforce.
+	 */
+	n = uparams->bits ? : count_of_bits / 2;
+	if (n < 1 || n >= count_of_bits) {
+		ksft_print_msg("Invalid number of CBM bits %d, expected 1 to %d\n",
+			       n, count_of_bits - 1);
+		return -1;
+	}
+
+	struct resctrl_val_param param = {
+		.ctrlgrp	= "c1",
+		.filename	= CAT_OCCUP_RESULT_FILE,
+		.mask		= ~(full_mask << n) & full_mask,
+		.num_of_runs	= 0,
+		.init		= cat_occup_init,

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 580

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+		.setup		= cat_occup_setup,
+		.measure	= cat_occup_measure,
+	};
+
+	alloc_span = cache_portion_size(cache_total_size, param.mask, full_mask);
+
+	/* Benchmark buffer spans the full cache: larger than the allocation. */
+	fill_buf.buf_size = cache_total_size;
+	fill_buf.memflush = uparams->fill_buf ? uparams->fill_buf->memflush : true;
+	param.fill_buf = &fill_buf;

This prevents usage of user provided benchmark. Please compare with cmt_run_test() initialization. You can find more details about how the workload and parameters are communicated in e958c21e2ede ("selftests/resctrl: Make benchmark parameter passing robust")

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 590

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+	cat_occup_cpu = uparams->cpu;
+
+	remove(param.filename);
+
+	ret = resctrl_val(test, uparams, &param);
+	if (ret)
+		return ret;
+
+	return cat_occup_check_results(&param, alloc_span, cache_total_size, n);
+}

Fundamentally this looks like a duplicate of cmt_run_test()? Only differences I see is the size of the buffer and how the test results are checked for pass/fail. Looking at the pass/fail I do not see a big difference with what the CMT test tests. I do not see what value this tests add beyond what the CMT test already provides.

Quoted 56 lines
tools/testing/selftests/resctrl/cat_test.c +15 -0
+
+static bool cat_occup_feature_check(const struct resctrl_test *test)
+{
+	return test_resource_feature_check(test) &&
+	       resctrl_mon_feature_exists("L3_MON", "llc_occupancy");
+}
+
+struct resctrl_test l3_cat_occup_test = {
+	.name = "L3_CAT_OCCUP",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = cat_occup_feature_check,
+	.run_test = cat_occup_run_test,
+	.cleanup = cat_occup_test_cleanup,
+};
tools/testing/selftests/resctrl/cmt_test.c +0 -3
index d09e693dc739..ef51daa8061a 100644
@@ -16,9 +16,6 @@
 #define MAX_DIFF		2000000
 #define MAX_DIFF_PERCENT	15
 
-#define CON_MON_LCC_OCCUP_PATH		\
-	"%s/%s/mon_data/mon_L3_%02d/llc_occupancy"
-
 static int cmt_init(const struct resctrl_val_param *param, int domain_id)
 {
 	sprintf(llc_occup_path, CON_MON_LCC_OCCUP_PATH, RESCTRL_PATH,
tools/testing/selftests/resctrl/resctrl.h +4 -0
index afe635b6e48d..ce3abf0bdac2 100644
@@ -31,6 +31,9 @@
 #define PHYS_ID_PATH		"/sys/devices/system/cpu/cpu"
 #define INFO_PATH		"/sys/fs/resctrl/info"
 
+#define CON_MON_LCC_OCCUP_PATH		\
+	"%s/%s/mon_data/mon_L3_%02d/llc_occupancy"
+
 /*
  * CPU vendor IDs
  *
@@ -244,6 +247,7 @@ extern struct resctrl_test mbm_test;
 extern struct resctrl_test mba_test;
 extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
+extern struct resctrl_test l3_cat_occup_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
tools/testing/selftests/resctrl/resctrl_tests.c +0 -0
index dbcd5eea9fbc..324a60818aa1 100644

tools/testing/selftests/resctrl/resctrl_tests.c static struct resctrl_test *resctrl_tests[] = { · line 25

Quoted 7 lines
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
@@ -19,6 +19,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&mba_test,
 	&cmt_test,
 	&l3_cat_test,
+	&l3_cat_occup_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };

Reinette

#3 lore raw

Re: [PATCH 1/3] selftests/resctrl: Add L3_CAT_OCCUP test to verify CAT bounds occupancy

Quoted 2 lines · Reinette Chatre

Hi Richard,

Hi Reinette,

Thanks for the review !

Quoted 9 lines · Richard Cheng
Quoted 3 lines

L3_CAT needs a CPU-exclusive cache portion, so it's skipped when MPAM reports every CBM bit as shareable, leaving L3 allocation untested. CMT only checks occupancy accuracy, not that a CBM actually limits it.

hmmm ... CMT test ensures that the LLC occupancy is within a % of the size of the cache allocation. To me this implies that it indeed tests that the CBM limits the allocation, no?

Yes you're correct on this. I based the patch on older version of the CMT test. After rebasing, I see the current CMT already uses a workload larger than the allocation and configures the root group with the complementary CBM.

It now covers what I wanted to test.

I think we're good to drop this patch in v2.

Quoted 317 lines
Quoted 21 lines

L3_CAT_OCCUP gives a group a small CBM, run a workload spanning the whole cache, and check every occupancy sample stays within the allocation. An unenforced CBM would instead let occupancy grow to the full cache.

Move CON_MON_LCC_OCCUP_PATH to resctrl.h to share it with CMT.

---
 tools/testing/selftests/resctrl/cat_test.c    | 201 ++++++++++++++++++
 tools/testing/selftests/resctrl/cmt_test.c    |   3 -
 tools/testing/selftests/resctrl/resctrl.h     |   4 +
 .../testing/selftests/resctrl/resctrl_tests.c |   1 +
 4 files changed, 206 insertions(+), 3 deletions(-)
tools/testing/selftests/resctrl/cat_test.c +0 -0
index f00b622c1460..16a947f1ed16 100644

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 410

Quoted 9 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
@@ -402,3 +402,204 @@ struct resctrl_test l2_noncont_cat_test = {
 	.feature_check = noncont_cat_feature_check,
 	.run_test = noncont_cat_run_test,
 };
+
+/*
+ * L3_CAT_OCCUP - Verify that a CAT allocation bounds cache occupancy.
+ *
+ * Unlike L3_CAT (which measures interference between groups and needs an
+ * exclusive cache portion), this test gives a control group a strict subset

Please let comment just refer to what this test does. These comments are unlikely to be updated if L3_CAT ever changes.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 416

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+ * of the CBM, then runs a benchmark whose buffer spans the *whole* cache -
+ * i.e. much larger than the allocation. With CAT enforced, the group can
+ * only keep its allocated portion resident, so llc_occupancy settles near
+ * the allocation size. Without enforcement occupancy would instead climb
+ * towards the full cache. This works even when all CBM bits are shareable
+ * (where L3_CAT is skipped).

hmmm ... these statements state as fact what really depends on system load and interference that this test make no attempt to avoid.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 423

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+ */
+#define CAT_OCCUP_RESULT_FILE		"result_cat_occup"
+#define CAT_OCCUP_NUM_OF_RUNS		5
+
+static int cat_occup_cpu;
+
+static int cat_occup_init(const struct resctrl_val_param *param, int domain_id)

Please take a look at recent resctrl selftest changes that provides more data to the init() that will avoid the cat_occup_cpu global. Even so, why not just use cmt_init() that reduces interference from rest of system to improve chances of workload's cache occupancy to match its cache allocation?

Quoted 17 lines
tools/testing/selftests/resctrl/cat_test.c +17 -0
+{
+	char schemata[64];
+
+	sprintf(llc_occup_path, CON_MON_LCC_OCCUP_PATH, RESCTRL_PATH,
+		param->ctrlgrp, domain_id);
+
+	/*
+	 * Confine the benchmark to the allocated portion *before* it starts
+	 * filling (resctrl_val() calls init() before forking the benchmark),
+	 * so occupancy reflects the restricted CBM from the first sample.
+	 */
+	snprintf(schemata, sizeof(schemata), "%lx", param->mask);
+
+	return write_schemata(param->ctrlgrp, schemata, cat_occup_cpu, "L3");
+}
+
+static int cat_occup_setup(const struct resctrl_test *test,

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 450

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+			   const struct user_params *uparams,
+			   struct resctrl_val_param *p)
+{
+	if (p->num_of_runs >= CAT_OCCUP_NUM_OF_RUNS)
+		return END_OF_TESTS;
+
+	p->num_of_runs++;
+
+	return 0;
+}

cmt_setup()?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 457

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+
+static int cat_occup_measure(const struct user_params *uparams,
+			     struct resctrl_val_param *param, pid_t bm_pid)
+{
+	sleep(1);
+	return measure_llc_resctrl(param->filename, bm_pid);
+}

cmt_measure()?

This test has a lot in common with the existing CMT test since it duplicates cmt_setup(), cmt_measure(), and cmt_feature_check(). From what I can tell it can use cmt_init() also. Could the implementation be simplified by instead considering it a CMT test, move code to cmt_test.c, and avoid all this duplication?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 464

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+
+static int cat_occup_check_results(struct resctrl_val_param *param,
+				   size_t alloc_span, size_t cache_size,
+				   int no_of_bits)
+{
+	char *token_array[8], temp[512];
+	unsigned long occu, max_occu = 0, ceiling, floor;

Please follow kernel coding style (throughout this series) by, in this example, using reverse-fir tree ordering.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 470

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+	int runs = 0;
+	int fail = 0;
+	FILE *fp;
+
+	/*
+	 * Check every sample, not an average: CAT is a hard limit, so a single

hmmm ... "is a hard limit" does not match what the code does

Quoted 24 lines
tools/testing/selftests/resctrl/cat_test.c +24 -0
+	 * sample above the allocation is a real violation that an average
+	 * could mask.
+	 */
+	ceiling = alloc_span + (cache_size - alloc_span) / 2;
+	floor = alloc_span / 2;
+
+	ksft_print_msg("Checking for pass/fail\n");
+	fp = fopen(param->filename, "r");
+	if (!fp) {
+		ksft_perror("Error in opening file");
+
+		return -1;
+	}
+
+	while (fgets(temp, sizeof(temp), fp)) {
+		char *token = strtok(temp, ":\t");
+		int fields = 0;
+
+		while (token) {
+			token_array[fields++] = token;
+			token = strtok(NULL, ":\t");
+		}
+
+		/* Field 3 is the resctrl-reported llc_occupancy value. */

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 504

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+		occu = strtoul(token_array[3], NULL, 0);
+		runs++;
+
+		if (occu > max_occu)
+			max_occu = occu;
+
+		if (occu > ceiling) {
+			ksft_print_msg("Fail: run %d occupancy %lu exceeds ceiling %lu\n",
+				       runs, occu, ceiling);
+			fail = 1;

KSFT_FAIL is available to avoid using magic numbers.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 516

Quoted 12 lines
tools/testing/selftests/resctrl/cat_test.c +12 -0
+		}
+	}
+	fclose(fp);
+
+	if (!runs) {
+		ksft_print_msg("No occupancy samples collected\n");
+		return -1;
+	}
+
+	if (max_occu < floor) {
+		ksft_print_msg("Fail: peak occupancy %lu never reached floor %lu\n",
+			       max_occu, floor);

I think this test should be dropped. We cannot control the environments in which the tests are run and legitimate interference may cause this test to fail without it meaning that there is a bug in resctrl.

Quoted 54 lines
tools/testing/selftests/resctrl/cat_test.c +54 -0
+		fail = 1;
+	}
+
+	ksft_print_msg("%s CAT confines occupancy to the allocated %d-bit portion\n",
+		       fail ? "Fail:" : "Pass:", no_of_bits);
+	ksft_print_msg("occupancy=%lu alloc=%zu full=%zu ceiling=%lu floor=%lu\n",
+		       max_occu, alloc_span, cache_size, ceiling, floor);
+
+	return fail;
+}
+
+static void cat_occup_test_cleanup(void)
+{
+	remove(CAT_OCCUP_RESULT_FILE);
+}
+
+static int cat_occup_run_test(const struct resctrl_test *test,
+			      const struct user_params *uparams)
+{
+	struct fill_buf_param fill_buf = {};
+	unsigned long cache_total_size = 0;
+	unsigned long full_mask;
+	int count_of_bits;
+	size_t alloc_span;
+	int n, ret;
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = get_cache_size(uparams->cpu, test->resource, &cache_total_size);
+	if (ret)
+		return ret;
+	ksft_print_msg("Cache size :%lu\n", cache_total_size);
+
+	count_of_bits = count_bits(full_mask);
+
+	/*
+	 * Allocate a strict subset of the cache so the benchmark buffer
+	 * is larger than the allocation and CAT has something to enforce.
+	 */
+	n = uparams->bits ? : count_of_bits / 2;
+	if (n < 1 || n >= count_of_bits) {
+		ksft_print_msg("Invalid number of CBM bits %d, expected 1 to %d\n",
+			       n, count_of_bits - 1);
+		return -1;
+	}
+
+	struct resctrl_val_param param = {
+		.ctrlgrp	= "c1",
+		.filename	= CAT_OCCUP_RESULT_FILE,
+		.mask		= ~(full_mask << n) & full_mask,
+		.num_of_runs	= 0,
+		.init		= cat_occup_init,

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 580

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+		.setup		= cat_occup_setup,
+		.measure	= cat_occup_measure,
+	};
+
+	alloc_span = cache_portion_size(cache_total_size, param.mask, full_mask);
+
+	/* Benchmark buffer spans the full cache: larger than the allocation. */
+	fill_buf.buf_size = cache_total_size;
+	fill_buf.memflush = uparams->fill_buf ? uparams->fill_buf->memflush : true;
+	param.fill_buf = &fill_buf;

This prevents usage of user provided benchmark. Please compare with cmt_run_test() initialization. You can find more details about how the workload and parameters are communicated in e958c21e2ede ("selftests/resctrl: Make benchmark parameter passing robust")

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l2_noncont_cat_test = { · line 590

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+	cat_occup_cpu = uparams->cpu;
+
+	remove(param.filename);
+
+	ret = resctrl_val(test, uparams, &param);
+	if (ret)
+		return ret;
+
+	return cat_occup_check_results(&param, alloc_span, cache_total_size, n);
+}

Fundamentally this looks like a duplicate of cmt_run_test()? Only differences I see is the size of the buffer and how the test results are checked for pass/fail. Looking at the pass/fail I do not see a big difference with what the CMT test tests. I do not see what value this tests add beyond what the CMT test already provides.

Quoted 51 lines
tools/testing/selftests/resctrl/cat_test.c +15 -0
+
+static bool cat_occup_feature_check(const struct resctrl_test *test)
+{
+	return test_resource_feature_check(test) &&
+	       resctrl_mon_feature_exists("L3_MON", "llc_occupancy");
+}
+
+struct resctrl_test l3_cat_occup_test = {
+	.name = "L3_CAT_OCCUP",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = cat_occup_feature_check,
+	.run_test = cat_occup_run_test,
+	.cleanup = cat_occup_test_cleanup,
+};
tools/testing/selftests/resctrl/cmt_test.c +0 -3
index d09e693dc739..ef51daa8061a 100644
@@ -16,9 +16,6 @@
 #define MAX_DIFF		2000000
 #define MAX_DIFF_PERCENT	15
 
-#define CON_MON_LCC_OCCUP_PATH		\
-	"%s/%s/mon_data/mon_L3_%02d/llc_occupancy"
-
 static int cmt_init(const struct resctrl_val_param *param, int domain_id)
 {
 	sprintf(llc_occup_path, CON_MON_LCC_OCCUP_PATH, RESCTRL_PATH,
tools/testing/selftests/resctrl/resctrl.h +4 -0
index afe635b6e48d..ce3abf0bdac2 100644
@@ -31,6 +31,9 @@
 #define PHYS_ID_PATH		"/sys/devices/system/cpu/cpu"
 #define INFO_PATH		"/sys/fs/resctrl/info"
 
+#define CON_MON_LCC_OCCUP_PATH		\
+	"%s/%s/mon_data/mon_L3_%02d/llc_occupancy"
+
 /*
  * CPU vendor IDs
  *
@@ -244,6 +247,7 @@ extern struct resctrl_test mbm_test;
 extern struct resctrl_test mba_test;
 extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
+extern struct resctrl_test l3_cat_occup_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
Quoted 14 lines

tools/testing/selftests/resctrl/resctrl_tests.c static struct resctrl_test *resctrl_tests[] = { · line 25

Quoted 12 lines
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
index dbcd5eea9fbc..324a60818aa1 100644
@@ -19,6 +19,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&mba_test,
 	&cmt_test,
 	&l3_cat_test,
+	&l3_cat_occup_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };

Reinette

Agreed for the above comments, I'll drop this change for v2. Thanks for pointing this out.

Best regards,
Richard Cheng.

#4 lore raw

PATCH 2/3 selftests/resctrl: Add L3_CAT_VALIDATE to check invalid CBMs

In reply to Richard Cheng

A CAT allocation is a CBM, and the kernel must refuse masks that can't describe a valid allocation. Nothing in the suite check this.

Implement L3_CAT_VALIDATE, first confirm a valid full mask is accepted, then write an empty mask and a mask with bits outside cbm_mask and confirm each is refused. A mask below min_cbm_bits is also checked where the platform needs more than one bit.

---
 tools/testing/selftests/resctrl/cat_test.c    | 79 +++++++++++++++++++
 tools/testing/selftests/resctrl/resctrl.h     |  1 +
 .../testing/selftests/resctrl/resctrl_tests.c |  1 +
 3 files changed, 81 insertions(+)
tools/testing/selftests/resctrl/cat_test.c +79 -0
index 16a947f1ed16..d236988916d9 100644
@@ -603,3 +603,82 @@ struct resctrl_test l3_cat_occup_test = {
 	.run_test = cat_occup_run_test,
 	.cleanup = cat_occup_test_cleanup,
 };
+
+/*
+ * L3_CAT_VALIDATE - Verify the kernel rejects invalid L3 CBM writes.
+ *
+ * CAT allocation accepts a Capacity Bit Mask (CBM). The kernel must reject
+ * masks that cannot represent a valid allocation: an empty mask, a mask with
+ * bits outside cbm_mask, or (where the platform requires it) a mask with
+ * fewer than min_cbm_bits bits. A rejected schemata write makes
+ * write_schemata() return non-zero, which is the signal this test checks.
+ */
+static int cat_validate_run_test(const struct resctrl_test *test,
+				 const struct user_params *uparams)
+{
+	unsigned long full_mask, bad_mask;
+	unsigned int min_cbm_bits, start;
+	char schemata[64];
+	int count_of_bits, ret;
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = resource_info_unsigned_get(test->resource, "min_cbm_bits",
+					 &min_cbm_bits);
+	if (ret)
+		return ret;
+
+	count_of_bits = count_bits(full_mask);
+
+	/* A valid full CBM must be accepted */
+	snprintf(schemata, sizeof(schemata), "%lx", full_mask);
+	if (write_schemata("", schemata, uparams->cpu, test->resource)) {
+		ksft_print_msg("Valid CBM 0x%lx was rejected\n", full_mask);
+		return 1;
+	}
+
+	/* An empty mask must be rejected. */
+	if (!write_schemata("", "0", uparams->cpu, test->resource)) {
+		ksft_print_msg("Empty CBM was accepted, must be rejected\n");
+		return 1;
+	}
+
+	/* A mask with a bit outside cbm_mask must be rejected. */
+	bad_mask = full_mask | (1UL << count_of_bits);
+	snprintf(schemata, sizeof(schemata), "%lx", bad_mask);
+	if (!write_schemata("", schemata, uparams->cpu, test->resource)) {
+		ksft_print_msg("Out-of-range CBM 0x%lx was accepted, must be rejected\n",
+			       bad_mask);
+		return 1;
+	}
+
+	/*
+	 * A mask with fewer than min_cbm_bits bits must be rejected. When
+	 * min_cbm_bits is 1 the only sub-minimum mask is the empty mask
+	 * This case only applies when the platform requires more contiguous bits.
+	 */
+	if (min_cbm_bits > 1) {
+		count_contiguous_bits(full_mask, &start);
+		bad_mask = create_bit_mask(start, min_cbm_bits - 1);
+		snprintf(schemata, sizeof(schemata), "%lx", bad_mask);
+		if (!write_schemata("", schemata, uparams->cpu, test->resource)) {
+			ksft_print_msg("CBM 0x%lx with too few bits was accepted\n",
+				       bad_mask);
+			return 1;
+		}
+	}
+
+	ksft_print_msg("Pass: CAT rejects invalid CBM writes\n");
+
+	return 0;
+}
+
+struct resctrl_test l3_cat_validate_test = {
+	.name = "L3_CAT_VALIDATE",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = test_resource_feature_check,
+	.run_test = cat_validate_run_test,
+};
tools/testing/selftests/resctrl/resctrl.h +1 -0
index ce3abf0bdac2..e2e3cf7833bc 100644
@@ -248,6 +248,7 @@ extern struct resctrl_test mba_test;
 extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
 extern struct resctrl_test l3_cat_occup_test;
+extern struct resctrl_test l3_cat_validate_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
index 324a60818aa1..66739e96f33c 100644
@@ -20,6 +20,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&cmt_test,
 	&l3_cat_test,
 	&l3_cat_occup_test,
+	&l3_cat_validate_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };
Signature · 1 line
2.43.0
maintainer
#5 lore raw

Re: [PATCH 2/3] selftests/resctrl: Add L3_CAT_VALIDATE to check invalid CBMs

Hi Richard,

Thank you very much. This is useful.

Quoted 4 lines · Richard Cheng

A CAT allocation is a CBM, and the kernel must refuse masks that can't describe a valid allocation. Nothing in the suite check this.

Implement L3_CAT_VALIDATE, first confirm a valid full mask is accepted,

How about "L3_CBM_VALIDATE"?

Quoted 2 lines

then write an empty mask and a mask with bits outside cbm_mask and confirm each is refused. A mask below min_cbm_bits is also checked where

The empty mask is not always a valid test (more below).

Quoted 1 line

the platform needs more than one bit.

(nit: please write changelog in imperative tone)

Quoted 17 lines
---
 tools/testing/selftests/resctrl/cat_test.c    | 79 +++++++++++++++++++
 tools/testing/selftests/resctrl/resctrl.h     |  1 +
 .../testing/selftests/resctrl/resctrl_tests.c |  1 +
 3 files changed, 81 insertions(+)
tools/testing/selftests/resctrl/cat_test.c +1 -0
index 16a947f1ed16..d236988916d9 100644
@@ -603,3 +603,82 @@ struct resctrl_test l3_cat_occup_test = {
 	.run_test = cat_occup_run_test,
 	.cleanup = cat_occup_test_cleanup,
 };
+

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 616

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+/*
+ * L3_CAT_VALIDATE - Verify the kernel rejects invalid L3 CBM writes.
+ *
+ * CAT allocation accepts a Capacity Bit Mask (CBM). The kernel must reject
+ * masks that cannot represent a valid allocation: an empty mask, a mask with
+ * bits outside cbm_mask, or (where the platform requires it) a mask with
+ * fewer than min_cbm_bits bits. A rejected schemata write makes
+ * write_schemata() return non-zero, which is the signal this test checks.
+ */
+static int cat_validate_run_test(const struct resctrl_test *test,

All tests in this file are "cat" tests so this function name could be more specific. How about "cbm_validate_run_test()"?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 622

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+				 const struct user_params *uparams)
+{
+	unsigned long full_mask, bad_mask;
+	unsigned int min_cbm_bits, start;
+	char schemata[64];
+	int count_of_bits, ret;

(reverse fir please)

Quoted 7 lines
tools/testing/selftests/resctrl/cat_test.c +7 -0
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = resource_info_unsigned_get(test->resource, "min_cbm_bits",
+					 &min_cbm_bits);

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 639

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+	if (ret)
+		return ret;
+
+	count_of_bits = count_bits(full_mask);
+
+	/* A valid full CBM must be accepted */
+	snprintf(schemata, sizeof(schemata), "%lx", full_mask);
+	if (write_schemata("", schemata, uparams->cpu, test->resource)) {
+		ksft_print_msg("Valid CBM 0x%lx was rejected\n", full_mask);
+		return 1;

KSFT_FAIL?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 642

Quoted 3 lines
tools/testing/selftests/resctrl/cat_test.c +3 -0
+	}
+
+	/* An empty mask must be rejected. */

Some systems do support an empty mask (AMD) so this needs to depend on min_cbm_bits being larger than zero.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 654

Quoted 12 lines
tools/testing/selftests/resctrl/cat_test.c +12 -0
+	if (!write_schemata("", "0", uparams->cpu, test->resource)) {
+		ksft_print_msg("Empty CBM was accepted, must be rejected\n");
+		return 1;
+	}
+
+	/* A mask with a bit outside cbm_mask must be rejected. */
+	bad_mask = full_mask | (1UL << count_of_bits);
+	snprintf(schemata, sizeof(schemata), "%lx", bad_mask);
+	if (!write_schemata("", schemata, uparams->cpu, test->resource)) {
+		ksft_print_msg("Out-of-range CBM 0x%lx was accepted, must be rejected\n",
+			       bad_mask);
+		return 1;

KSFT_FAIL?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 663

Quoted 9 lines
tools/testing/selftests/resctrl/cat_test.c +9 -0
+	}
+
+	/*
+	 * A mask with fewer than min_cbm_bits bits must be rejected. When
+	 * min_cbm_bits is 1 the only sub-minimum mask is the empty mask
+	 * This case only applies when the platform requires more contiguous bits.
+	 */
+	if (min_cbm_bits > 1) {
+		count_contiguous_bits(full_mask, &start);

The return value should be checked here to ensure that the create_bit_mask() below is valid.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_occup_test = { · line 669

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+		bad_mask = create_bit_mask(start, min_cbm_bits - 1);
+		snprintf(schemata, sizeof(schemata), "%lx", bad_mask);
+		if (!write_schemata("", schemata, uparams->cpu, test->resource)) {
+			ksft_print_msg("CBM 0x%lx with too few bits was accepted\n",
+				       bad_mask);
+			return 1;

KSFT_FAIL?

Quoted 32 lines
tools/testing/selftests/resctrl/cat_test.c +15 -0
+		}
+	}
+
+	ksft_print_msg("Pass: CAT rejects invalid CBM writes\n");
+
+	return 0;
+}
+
+struct resctrl_test l3_cat_validate_test = {
+	.name = "L3_CAT_VALIDATE",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = test_resource_feature_check,
+	.run_test = cat_validate_run_test,
+};
tools/testing/selftests/resctrl/resctrl.h +1 -0
index ce3abf0bdac2..e2e3cf7833bc 100644
@@ -248,6 +248,7 @@ extern struct resctrl_test mba_test;
 extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
 extern struct resctrl_test l3_cat_occup_test;
+extern struct resctrl_test l3_cat_validate_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
tools/testing/selftests/resctrl/resctrl_tests.c +0 -0
index 324a60818aa1..66739e96f33c 100644

tools/testing/selftests/resctrl/resctrl_tests.c static struct resctrl_test *resctrl_tests[] = { · line 26

Quoted 7 lines
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
@@ -20,6 +20,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&cmt_test,
 	&l3_cat_test,
 	&l3_cat_occup_test,
+	&l3_cat_validate_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };

Thank you.

Reinette

#6 lore raw

PATCH 3/3 selftests/resctrl: Add L3_BIT_USAGE to check bit_usage tracks allocation

In reply to Richard Cheng

info/L3/bit_usage annotates every cache portion as used by software ('X or 'S') or not ('H' or '0'). Nothing in the suite checks it's reported correctly.

Implement L3_BIT_USAGE to set the root group's CBM to a few masks and confirm that, for every portion, bit_usage marks it software-used exactly when it is in the CBM. With only the root group present this must track the mask bit-for-bit.

---
 tools/testing/selftests/resctrl/cat_test.c    | 153 ++++++++++++++++++
 tools/testing/selftests/resctrl/resctrl.h     |   1 +
 .../testing/selftests/resctrl/resctrl_tests.c |   1 +
 3 files changed, 155 insertions(+)
tools/testing/selftests/resctrl/cat_test.c +153 -0
index d236988916d9..62af8ac98109 100644
@@ -682,3 +682,156 @@ struct resctrl_test l3_cat_validate_test = {
 	.feature_check = test_resource_feature_check,
 	.run_test = cat_validate_run_test,
 };
+
+/*
+ * L3_BIT_USAGE - Verify info/L3/bit_usage reflects the allocation.
+ *
+ * bit_usage annotates each cache portion: 'X'/'S' mean a portion is used by
+ * software, 'H'/'0' mean it is not. With only the root group present, a
+ * portion is software-used exactly when it is in the root CBM, so bit_usage
+ * must track the CBM bit-for-bit.
+ */
+#define BIT_USAGE_LEN	256
+
+static bool bit_usage_sw_used(char c)
+{
+	return c == 'X' || c == 'S';
+}
+
+static bool bit_usage_not_used(char c)
+{
+	return c == 'H' || c == '0';
+}
+
+static int bit_usage_for_domain(const char *resource, int domain_id,
+				char *out, size_t len)
+{
+	char path[1024], raw[BIT_USAGE_LEN], *tok, *save;
+	FILE *fp;
+
+	snprintf(path, sizeof(path), "%s/%s/bit_usage", INFO_PATH, resource);
+	fp = fopen(path, "r");
+	if (!fp) {
+		ksft_perror("Error opening bit_usage");
+		return -1;
+	}
+	if (!fgets(raw, sizeof(raw), fp)) {
+		ksft_perror("Error reading bit_usage");
+		fclose(fp);
+		return -1;
+	}
+	fclose(fp);
+
+	/* bit_usage is "id=chars;id=chars;..."; return the chars for domain_id. */
+	for (tok = strtok_r(raw, ";\n", &save); tok; tok = strtok_r(NULL, ";\n", &save)) {
+		char *eq = strchr(tok, '=');
+
+		if (!eq)
+			continue;
+		*eq = '\0';
+		if (atoi(tok) == domain_id) {
+			snprintf(out, len, "%s", eq + 1);
+			return 0;
+		}
+	}
+
+	ksft_print_msg("No bit_usage entry for domain %d\n", domain_id);
+	return -1;
+}
+
+static int bit_usage_check_mask(const struct resctrl_test *test, int cpu,
+				int domain_id, unsigned long mask,
+				int count_of_bits)
+{
+	char schemata[64], usage[BIT_USAGE_LEN];
+	int i, ret;
+
+	snprintf(schemata, sizeof(schemata), "%lx", mask);
+	ret = write_schemata("", schemata, cpu, test->resource);
+	if (ret) {
+		ksft_print_msg("Failed to set CBM 0x%lx\n", mask);
+		return ret;
+	}
+
+	ret = bit_usage_for_domain(test->resource, domain_id, usage, sizeof(usage));
+	if (ret)
+		return ret;
+
+	if (strlen(usage) != count_of_bits) {
+		ksft_print_msg("bit_usage \"%s\" has %zu chars, expected %d\n",
+			       usage, strlen(usage), count_of_bits);
+		return 1;
+	}
+
+	/* bit_usage prints the highest portion first, so usage[0] is bit N-1. */
+	for (i = 0; i < count_of_bits; i++) {
+		int bit = count_of_bits - 1 - i;
+		bool in_cbm = (mask >> bit) & 1;
+		char c = usage[i];
+
+		if (!bit_usage_sw_used(c) && !bit_usage_not_used(c)) {
+			ksft_print_msg("Unexpected bit_usage char '%c' for CBM 0x%lx\n",
+				       c, mask);
+			return 1;
+		}
+		if (in_cbm != bit_usage_sw_used(c)) {
+			ksft_print_msg("CBM 0x%lx portion %d shows '%c', %s allocation\n",
+				       mask, bit, c, in_cbm ? "in" : "not in");
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+static int bit_usage_run_test(const struct resctrl_test *test,
+			      const struct user_params *uparams)
+{
+	unsigned long full_mask, masks[3];
+	char schemata[64];
+	int count_of_bits, domain_id, i, ret;
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = get_domain_id(test->resource, uparams->cpu, &domain_id);
+	if (ret < 0)
+		return ret;
+
+	count_of_bits = count_bits(full_mask);
+
+	masks[0] = full_mask;					/* every portion */
+	masks[1] = create_bit_mask(0, count_of_bits / 2);	/* low half */
+	masks[2] = full_mask & ~masks[1];			/* high half */
+
+	for (i = 0; i < 3; i++) {
+		ret = bit_usage_check_mask(test, uparams->cpu, domain_id,
+					   masks[i], count_of_bits);
+		if (ret)
+			break;
+	}
+
+	/* Restore the root group to the full CBM. */
+	snprintf(schemata, sizeof(schemata), "%lx", full_mask);
+	write_schemata("", schemata, uparams->cpu, test->resource);
+
+	if (!ret)
+		ksft_print_msg("Pass: bit_usage reflects the allocation\n");
+
+	return ret;
+}
+
+static bool bit_usage_feature_check(const struct resctrl_test *test)
+{
+	return test_resource_feature_check(test) &&
+	       resource_info_file_exists(test->resource, "bit_usage");
+}
+
+struct resctrl_test l3_bit_usage_test = {
+	.name = "L3_BIT_USAGE",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = bit_usage_feature_check,
+	.run_test = bit_usage_run_test,
+};
tools/testing/selftests/resctrl/resctrl.h +1 -0
index e2e3cf7833bc..270af2b26ba7 100644
@@ -249,6 +249,7 @@ extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
 extern struct resctrl_test l3_cat_occup_test;
 extern struct resctrl_test l3_cat_validate_test;
+extern struct resctrl_test l3_bit_usage_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
index 66739e96f33c..e622928a2d7d 100644
@@ -21,6 +21,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&l3_cat_test,
 	&l3_cat_occup_test,
 	&l3_cat_validate_test,
+	&l3_bit_usage_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };
Signature · 1 line
2.43.0
maintainer
#7 lore raw

Re: [PATCH 3/3] selftests/resctrl: Add L3_BIT_USAGE to check bit_usage tracks allocation

Hi Richard,

Thank you very much for creating this test.

Quoted 26 lines · Richard Cheng

info/L3/bit_usage annotates every cache portion as used by software ('X or 'S') or not ('H' or '0'). Nothing in the suite checks it's reported correctly.

Implement L3_BIT_USAGE to set the root group's CBM to a few masks and confirm that, for every portion, bit_usage marks it software-used exactly when it is in the CBM. With only the root group present this must track the mask bit-for-bit.

---
 tools/testing/selftests/resctrl/cat_test.c    | 153 ++++++++++++++++++
 tools/testing/selftests/resctrl/resctrl.h     |   1 +
 .../testing/selftests/resctrl/resctrl_tests.c |   1 +
 3 files changed, 155 insertions(+)
tools/testing/selftests/resctrl/cat_test.c +2 -0
index d236988916d9..62af8ac98109 100644
@@ -682,3 +682,156 @@ struct resctrl_test l3_cat_validate_test = {
 	.feature_check = test_resource_feature_check,
 	.run_test = cat_validate_run_test,
 };
+
+/*

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 696

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+ * L3_BIT_USAGE - Verify info/L3/bit_usage reflects the allocation.
+ *
+ * bit_usage annotates each cache portion: 'X'/'S' mean a portion is used by
+ * software, 'H'/'0' mean it is not. With only the root group present, a
+ * portion is software-used exactly when it is in the root CBM, so bit_usage
+ * must track the CBM bit-for-bit.
+ */
+#define BIT_USAGE_LEN	256
+
+static bool bit_usage_sw_used(char c)

There is a bit of redundancy here with the multiple appearances of "use". How about "bit_sw_used()" or "bit_used_by_sw()" or ...?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 701

Quoted 5 lines
tools/testing/selftests/resctrl/cat_test.c +5 -0
+{
+	return c == 'X' || c == 'S';
+}
+
+static bool bit_usage_not_used(char c)

(same comment about multiple "use" appearances)

Quoted 20 lines
tools/testing/selftests/resctrl/cat_test.c +20 -0
+{
+	return c == 'H' || c == '0';
+}
+
+static int bit_usage_for_domain(const char *resource, int domain_id,
+				char *out, size_t len)
+{
+	char path[1024], raw[BIT_USAGE_LEN], *tok, *save;
+	FILE *fp;
+
+	snprintf(path, sizeof(path), "%s/%s/bit_usage", INFO_PATH, resource);
+	fp = fopen(path, "r");
+	if (!fp) {
+		ksft_perror("Error opening bit_usage");
+		return -1;
+	}
+	if (!fgets(raw, sizeof(raw), fp)) {
+		ksft_perror("Error reading bit_usage");
+		fclose(fp);
+		return -1;

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 731

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+	}
+	fclose(fp);
+
+	/* bit_usage is "id=chars;id=chars;..."; return the chars for domain_id. */
+	for (tok = strtok_r(raw, ";\n", &save); tok; tok = strtok_r(NULL, ";\n", &save)) {
+		char *eq = strchr(tok, '=');
+
+		if (!eq)
+			continue;
+		*eq = '\0';

Above looks like a re-implementation of strtok?

Quoted 31 lines
tools/testing/selftests/resctrl/cat_test.c +31 -0
+		if (atoi(tok) == domain_id) {
+			snprintf(out, len, "%s", eq + 1);
+			return 0;
+		}
+	}
+
+	ksft_print_msg("No bit_usage entry for domain %d\n", domain_id);
+	return -1;
+}
+
+static int bit_usage_check_mask(const struct resctrl_test *test, int cpu,
+				int domain_id, unsigned long mask,
+				int count_of_bits)
+{
+	char schemata[64], usage[BIT_USAGE_LEN];
+	int i, ret;
+
+	snprintf(schemata, sizeof(schemata), "%lx", mask);
+	ret = write_schemata("", schemata, cpu, test->resource);
+	if (ret) {
+		ksft_print_msg("Failed to set CBM 0x%lx\n", mask);
+		return ret;
+	}
+
+	ret = bit_usage_for_domain(test->resource, domain_id, usage, sizeof(usage));
+	if (ret)
+		return ret;
+
+	if (strlen(usage) != count_of_bits) {
+		ksft_print_msg("bit_usage \"%s\" has %zu chars, expected %d\n",
+			       usage, strlen(usage), count_of_bits);

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 772

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+		return 1;
+	}
+
+	/* bit_usage prints the highest portion first, so usage[0] is bit N-1. */
+	for (i = 0; i < count_of_bits; i++) {
+		int bit = count_of_bits - 1 - i;
+		bool in_cbm = (mask >> bit) & 1;
+		char c = usage[i];
+
+		if (!bit_usage_sw_used(c) && !bit_usage_not_used(c)) {

Could this be simplified (by avoiding double negative) to

		if (bit_usage_sw_used(c) && bit_usage_not_used(c)) ?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 773

Quoted 1 line
tools/testing/selftests/resctrl/cat_test.c +1 -0
+			ksft_print_msg("Unexpected bit_usage char '%c' for CBM 0x%lx\n",

I think "Conflicting" instead of "Unexpected" would be more specific about what the issue is.

Quoted 9 lines
tools/testing/selftests/resctrl/cat_test.c +9 -0
+				       c, mask);
+			return 1;
+		}
+		if (in_cbm != bit_usage_sw_used(c)) {
+			ksft_print_msg("CBM 0x%lx portion %d shows '%c', %s allocation\n",
+				       mask, bit, c, in_cbm ? "in" : "not in");
+			return 1;
+		}
+	}

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 792

Quoted 10 lines
tools/testing/selftests/resctrl/cat_test.c +10 -0
+
+	return 0;
+}
+
+static int bit_usage_run_test(const struct resctrl_test *test,
+			      const struct user_params *uparams)
+{
+	unsigned long full_mask, masks[3];
+	char schemata[64];
+	int count_of_bits, domain_id, i, ret;

(reverse fir)

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 800

Quoted 8 lines
tools/testing/selftests/resctrl/cat_test.c +8 -0
+
+	ret = get_full_cbm(test->resource, &full_mask);
+	if (ret)
+		return ret;
+
+	ret = get_domain_id(test->resource, uparams->cpu, &domain_id);
+	if (ret < 0)
+		return ret;

Since this function does not use domain ID, could this be simplified by letting bit_usage_check_mask() determine the domain ID itself?

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 806

Quoted 6 lines
tools/testing/selftests/resctrl/cat_test.c +6 -0
+
+	count_of_bits = count_bits(full_mask);
+
+	masks[0] = full_mask;					/* every portion */
+	masks[1] = create_bit_mask(0, count_of_bits / 2);	/* low half */
+	masks[2] = full_mask & ~masks[1];			/* high half */

Please do not use tail comments. Instead move comments above the line they apply to.

tools/testing/selftests/resctrl/cat_test.c struct resctrl_test l3_cat_validate_test = { · line 817

Quoted 11 lines
tools/testing/selftests/resctrl/cat_test.c +11 -0
+
+	for (i = 0; i < 3; i++) {
+		ret = bit_usage_check_mask(test, uparams->cpu, domain_id,
+					   masks[i], count_of_bits);
+		if (ret)
+			break;
+	}
+
+	/* Restore the root group to the full CBM. */
+	snprintf(schemata, sizeof(schemata), "%lx", full_mask);
+	write_schemata("", schemata, uparams->cpu, test->resource);

It is not necessary to restore the state - resctrl fs is unmounted after every test.

Quoted 37 lines
tools/testing/selftests/resctrl/cat_test.c +20 -0
+
+	if (!ret)
+		ksft_print_msg("Pass: bit_usage reflects the allocation\n");
+
+	return ret;
+}
+
+static bool bit_usage_feature_check(const struct resctrl_test *test)
+{
+	return test_resource_feature_check(test) &&
+	       resource_info_file_exists(test->resource, "bit_usage");
+}
+
+struct resctrl_test l3_bit_usage_test = {
+	.name = "L3_BIT_USAGE",
+	.group = "CAT",
+	.resource = "L3",
+	.feature_check = bit_usage_feature_check,
+	.run_test = bit_usage_run_test,
+};
tools/testing/selftests/resctrl/resctrl.h +1 -0
index e2e3cf7833bc..270af2b26ba7 100644
@@ -249,6 +249,7 @@ extern struct resctrl_test cmt_test;
 extern struct resctrl_test l3_cat_test;
 extern struct resctrl_test l3_cat_occup_test;
 extern struct resctrl_test l3_cat_validate_test;
+extern struct resctrl_test l3_bit_usage_test;
 extern struct resctrl_test l3_noncont_cat_test;
 extern struct resctrl_test l2_noncont_cat_test;
 
tools/testing/selftests/resctrl/resctrl_tests.c +0 -0
index 66739e96f33c..e622928a2d7d 100644

tools/testing/selftests/resctrl/resctrl_tests.c static struct resctrl_test *resctrl_tests[] = { · line 27

Quoted 7 lines
tools/testing/selftests/resctrl/resctrl_tests.c +1 -0
@@ -21,6 +21,7 @@ static struct resctrl_test *resctrl_tests[] = {
 	&l3_cat_test,
 	&l3_cat_occup_test,
 	&l3_cat_validate_test,
+	&l3_bit_usage_test,
 	&l3_noncont_cat_test,
 	&l2_noncont_cat_test,
 };

Reinette