Skip to contentSkip to conversation

kernel/irq: Add irqbalance01

PATCH Superseded Richard Palethorpe 4 participants 7 messages Aug 24, 2021 – Sep 9, 2021

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 kernel/irq: Add irqbalance01

Add first test specifically targeting interrupts and IRQ management.

This includes some comments inline because I think the parsing code is unavoidably confusing.

Note on the CPU mask parsing; there is already some code for parsing and manipulating bitmaps in the LTP. However it is absurdly complicated and we don't need actual bitmaps. In fact an array of bytes is more flexible.

---
 runtest/irq                         |   1 +
 testcases/kernel/irq/.gitignore     |   1 +
 testcases/kernel/irq/Makefile       |   9 +
 testcases/kernel/irq/irqbalance01.c | 288 ++++++++++++++++++++++++++++
 4 files changed, 299 insertions(+)
 create mode 100644 runtest/irq
 create mode 100644 testcases/kernel/irq/.gitignore
 create mode 100644 testcases/kernel/irq/Makefile
 create mode 100644 testcases/kernel/irq/irqbalance01.c
runtest/irq new file +1 -0
new file mode 100644
index 000000000..56d0d23c8
@@ -0,0 +1 @@
+irqbalance01 irqbalance01
testcases/kernel/irq/.gitignore new file +1 -0
new file mode 100644
index 000000000..8ed69a99c
@@ -0,0 +1 @@
+irqbalance01
testcases/kernel/irq/Makefile new file +9 -0
new file mode 100644
index 000000000..085e06fac
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+top_srcdir		?= ../../..
+
+include $(top_srcdir)/include/mk/testcases.mk
+
+INSTALL_TARGETS		:= *.sh
+
+include $(top_srcdir)/include/mk/generic_leaf_target.mk
testcases/kernel/irq/irqbalance01.c new file +288 -0

288 lines not on this page · raw message

Signature · 1 line
2.31.1
#1 lore raw

PATCH 1/1 kernel/irq: Add irqbalance01

Hi Richie,

LGTM, only few nits found bellow (easily fixed before merge)

...

testcases/kernel/irq/.gitignore line 1

Quoted 3 lines

+++ b/testcases/kernel/irq/.gitignore

testcases/kernel/irq/.gitignore new file +1 -0
@@ -0,0 +1 @@
+irqbalance01

/irqbalance01

testcases/kernel/irq/Makefile line 7

Quoted 13 lines
testcases/kernel/irq/Makefile new file +7 -0
new file mode 100644
index 000000000..085e06fac
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+top_srcdir		?= ../../..
+
+include $(top_srcdir)/include/mk/testcases.mk
+
+INSTALL_TARGETS		:= *.sh

This should be removed (probably copy paste error

Quoted 20 lines
testcases/kernel/irq/Makefile new file +2 -0
+
+include $(top_srcdir)/include/mk/generic_leaf_target.mk
testcases/kernel/irq/irqbalance01.c new file +12 -0
new file mode 100644
index 000000000..0a476839c
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 SUSE LLC <[email protected]> */
+/*\
+ * [Description]
+ *
+ * Check that something (e.g. irqbalance daemon) is performing IRQ
+ * load balancing.
+ *
+ * On most systems userland needs to set /proc/irq/$IRQ/smp_affinity
+ * to prevent many IRQs being delivered to the same CPU.
+ *
+ * Note some drivers and IRQ controllers will distribute IRQs

testcases/kernel/irq/irqbalance01.c line 22

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+ * evenly. Some systems will have housekeeping CPUs configured. Some
+ * IRQs can not be masked etc. So this test is not appropriate for all
+ * scenarios.
+ *
+ * Furthermore, exactly how IRQs should be distributed is a
+ * performance and/or security issue. This is only a generic smoke
+ * test. It will hopefully detect misconfigured systems and total
+ * balancing failures which are often silent errors.
+ *
+ * Heuristic: Evidence of Change

Add blank new line here to get better docparse formatting:

testcases/kernel/irq/irqbalance01.c line 24

Quoted 2 lines
testcases/kernel/irq/irqbalance01.c new file +2 -0
+ * 1. Find IRQs with a non-zero count
+ * 2. Check if they are now disallowed

LGTM. It'd be interesting to hear opinion of IRQ subsystem maintainer Thomas Gleixner.

testcases/kernel/irq/irqbalance01.c line 26

Quoted 2 lines
testcases/kernel/irq/irqbalance01.c new file +2 -0
+ *
+ * There are two sources of information we need to parse:

Also here.

testcases/kernel/irq/irqbalance01.c line 34

Quoted 8 lines
testcases/kernel/irq/irqbalance01.c new file +8 -0
+ * 1. /proc/interrupts
+ * 2. /proc/irq/$IRQ/smp_affinity
+ *
+ * We get the active IRQs and CPUs from /proc/interrupts. It also
+ * contains the per-CPU IRQ counts and info we do not care about.
+ *
+ * We get the IRQ masks from each active IRQ's smp_affinity file. This
+ * is a bitmask written out in hexidecimal format. It shows which CPUs

^ hexadecimal

testcases/kernel/irq/irqbalance01.c line 35

Quoted 1 line
testcases/kernel/irq/irqbalance01.c new file +1 -0
+ * an IRQ may be recieved by.

^ received

Quoted 27 lines
testcases/kernel/irq/irqbalance01.c new file +27 -0
+ */
+
+#include <stdlib.h>
+
+#include "tst_test.h"
+#include "tst_safe_stdio.h"
+#include "tst_safe_file_at.h"
+
+enum affinity {
+	ALLOW,
+	DENY,
+};
+
+static unsigned int *irq_stats;
+static enum affinity *irq_affinity;
+
+static unsigned int nr_cpus;
+static unsigned int nr_irqs;
+static unsigned int *irq_ids;
+
+static void collect_irq_info(void)
+{
+	char *buf = NULL, *c, *first_row;
+	char path[PATH_MAX];
+	size_t size = 1024;
+	size_t ret, row, col;
+	long acc;

testcases/kernel/irq/irqbalance01.c line 72

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+	unsigned int cpu_total, mask_pos;
+	int fd = SAFE_OPEN("/proc/interrupts", O_RDONLY);
+
+	nr_cpus = 0;
+	nr_irqs = 0;
+
+	do {
+		size *= 2;
+		buf = SAFE_REALLOC(buf, size);
+		SAFE_LSEEK(fd, SEEK_SET, 0);
		SAFE_LSEEK(fd, 0, SEEK_SET);

(it works only because SEEK_SET is also 0)

testcases/kernel/irq/irqbalance01.c line 74

Quoted 2 lines
testcases/kernel/irq/irqbalance01.c new file +2 -0
+		ret = SAFE_READ(0, fd, buf, size - 1);
+	} while (ret >= size - 1);

...

The rest LGTM.

Kind regards,
Petr

#2 lore raw

PATCH 1/1 kernel/irq: Add irqbalance01

Hello Petr,

Quoted 3 lines · Petr Vorel

Hi Richie,

LGTM, only few nits found bellow (easily fixed before merge)

Thanks, but I also changed the printf to tst_printf which should be added in the bpf_prog05 patch set. So I can post V2 after that is merged.

Quoted 53 lines

...

testcases/kernel/irq/.gitignore line 1

Quoted 3 lines

+++ b/testcases/kernel/irq/.gitignore

testcases/kernel/irq/.gitignore new file +1 -0
@@ -0,0 +1 @@
+irqbalance01

/irqbalance01

testcases/kernel/irq/Makefile line 7

Quoted 13 lines
testcases/kernel/irq/Makefile new file +7 -0
new file mode 100644
index 000000000..085e06fac
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+top_srcdir		?= ../../..
+
+include $(top_srcdir)/include/mk/testcases.mk
+
+INSTALL_TARGETS		:= *.sh

This should be removed (probably copy paste error

Quoted 20 lines
testcases/kernel/irq/Makefile new file +2 -0
+
+include $(top_srcdir)/include/mk/generic_leaf_target.mk
testcases/kernel/irq/irqbalance01.c new file +12 -0
new file mode 100644
index 000000000..0a476839c
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 SUSE LLC <[email protected]> */
+/*\
+ * [Description]
+ *
+ * Check that something (e.g. irqbalance daemon) is performing IRQ
+ * load balancing.
+ *
+ * On most systems userland needs to set /proc/irq/$IRQ/smp_affinity
+ * to prevent many IRQs being delivered to the same CPU.
+ *
+ * Note some drivers and IRQ controllers will distribute IRQs

testcases/kernel/irq/irqbalance01.c line 22

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+ * evenly. Some systems will have housekeeping CPUs configured. Some
+ * IRQs can not be masked etc. So this test is not appropriate for all
+ * scenarios.
+ *
+ * Furthermore, exactly how IRQs should be distributed is a
+ * performance and/or security issue. This is only a generic smoke
+ * test. It will hopefully detect misconfigured systems and total
+ * balancing failures which are often silent errors.
+ *
+ * Heuristic: Evidence of Change

Add blank new line here to get better docparse formatting:

Quoted 4 lines

testcases/kernel/irq/irqbalance01.c line 24

Quoted 2 lines
testcases/kernel/irq/irqbalance01.c new file +2 -0
+ * 1. Find IRQs with a non-zero count
+ * 2. Check if they are now disallowed

LGTM. It'd be interesting to hear opinion of IRQ subsystem maintainer Thomas Gleixner.

I guess I can copy them in on V2.

Signature · 2 lines
Thank you,
Richard.
#3 lore raw

PATCH v2 1/1 kernel/irq: Add irqbalance01

Add first test specifically targeting interrupts and IRQ management.

This includes some comments inline because I think the parsing code is unavoidably confusing.

Note on the CPU mask parsing; there is already some code for parsing and manipulating bitmaps in the LTP. However it is absurdly complicated and we don't need actual bitmaps. In fact an array of bytes is more flexible.

---

To be clear this is a Linux Test Project test.

CC'ing IRQ subsystem and irqbalance maintainers in case they are interested.

V2:
* Use tst_printf
* Fix typos
* rm .sh from Makefile

 runtest/irq                         |   1 +
 testcases/kernel/irq/.gitignore     |   1 +
 testcases/kernel/irq/Makefile       |   7 +
 testcases/kernel/irq/irqbalance01.c | 290 ++++++++++++++++++++++++++++
 4 files changed, 299 insertions(+)
 create mode 100644 runtest/irq
 create mode 100644 testcases/kernel/irq/.gitignore
 create mode 100644 testcases/kernel/irq/Makefile
 create mode 100644 testcases/kernel/irq/irqbalance01.c
runtest/irq new file +1 -0
new file mode 100644
index 000000000..56d0d23c8
@@ -0,0 +1 @@
+irqbalance01 irqbalance01
testcases/kernel/irq/.gitignore new file +1 -0
new file mode 100644
index 000000000..8ed69a99c
@@ -0,0 +1 @@
+irqbalance01
testcases/kernel/irq/Makefile new file +7 -0
new file mode 100644
index 000000000..aa51da7cb
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+top_srcdir		?= ../../..
+
+include $(top_srcdir)/include/mk/testcases.mk
+
+include $(top_srcdir)/include/mk/generic_leaf_target.mk
testcases/kernel/irq/irqbalance01.c new file +290 -0

290 lines not on this page · raw message

Signature · 1 line
2.31.1
#4 lore raw

PATCH v2 1/1 kernel/irq: Add irqbalance01

Hi Richie,

Quoted 4 lines

V2:
* Use tst_printf
* Fix typos
* rm .sh from Makefile

Thanks for a quick update.

Quoted 1 line
testcases/kernel/irq/irqbalance01.c +0 -0

...

testcases/kernel/irq/irqbalance01.c line 252

Quoted 2 lines
testcases/kernel/irq/irqbalance01.c +2 -0
+	tst_res(changed ? TPASS : TFAIL,
+		"Heuristic: Detected %zu irq-cpu pairs have been dissallowed",

^ disallowed
(will be fixed during merge)

Kind regards,
Petr

#5 lore raw

PATCH v2 1/1 kernel/irq: Add irqbalance01

In reply to Richard Palethorpe

Hi!

Quoted 24 lines
testcases/kernel/irq/irqbalance01.c new file +24 -0
+	/* Read the CPU affinity masks for each IRQ. See bitmap_string() in the kernel (%*pb) */
+	for (row = 0; row < nr_irqs; row++) {
+		sprintf(path, "/proc/irq/%u/smp_affinity", irq_ids[row]);
+		ret = SAFE_FILE_READAT(0, path, buf, size);
+		if (ret < 1)
+			tst_brk(TBROK, "Empty /proc/irq/%u/smp_affinity?", irq_ids[row]);
+		c = buf;
+		col = 0;
+
+		while (*c != '\0') {
+			switch (*c) {
+			case '\n':
+			case ' ':
+			case ',':
+				c++;
+				continue;
+			case '0' ... '9':
+				acc = *c - '0';
+				break;
+			case 'a' ... 'f':
+				acc = 10 + *c - 'a';
+				break;
+			default:
+				tst_res(TINFO, "%u/smp_affnity: %s", irq_ids[row], buf);

testcases/kernel/irq/irqbalance01.c line 198

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+				tst_brk(TBROK, "Wasn't expecting 0x%02x", *c);
+			}
+
+			for (mask_pos = 0; mask_pos < 4; mask_pos++) {
+				if (mask_pos + col >= nr_cpus)
+					break;
+
+				irq_affinity[row * nr_cpus + (nr_cpus - 1 - col - mask_pos)] =
+					(acc & (8 >> mask_pos)) ? ALLOW : DENY;
+			}

The mask here is actually wrong, since we go backward in the irq_affinity array the bitmask has to grow not shrink so this should be (acc & (1<<mask_pos))

testcases/kernel/irq/irqbalance01.c line 212

Quoted 13 lines
testcases/kernel/irq/irqbalance01.c new file +13 -0
+			col += mask_pos;
+			c++;
+		}
+	}
+
+	free(buf);
+}
+
+static void print_irq_info(void)
+{
+	size_t row, col;
+	unsigned int count;
+	enum affinity aff;

It would be nice if we printed the CPUX header here as well so that it's clear which column describes which CPU.

Quoted 7 lines
testcases/kernel/irq/irqbalance01.c new file +7 -0
+	for (row = 0; row < nr_irqs; row++) {
+		tst_printf("%5u:", irq_ids[row]);
+
+		for (col = 0; col < nr_cpus; col++) {
+			count = irq_stats[row * nr_cpus + col];
+			aff = irq_affinity[row * nr_cpus + col] == ALLOW ? '+' : '-';
+

testcases/kernel/irq/irqbalance01.c line 230

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+			tst_printf("%10u%c", count, aff);
+		}
+
+		tst_printf("\n");
+	}
+
+	tst_printf("Total:");
+
+	for (col = 0; col < nr_cpus; col++)
+		tst_printf("%11u", irq_stats[row * nr_cpus + col]);

^
And I would change this to "%10u " so that the value aligns with the rest of the numbers in the columns.

Review

Quoted 2 lines
(quoted diff) +2 -0
+	tst_printf("\n");
+}

Otherwise the rest looks good.

Signature · 2 lines
Cyril Hrubis
[email protected]
#6 lore raw

PATCH v2 1/1 kernel/irq: Add irqbalance01

In reply to Richard Palethorpe

Hi!

Quoted 9 lines
testcases/kernel/irq/irqbalance01.c new file +9 -0
+static void collect_irq_info(void)
+{
+	char *buf = NULL, *c, *first_row;
+	char path[PATH_MAX];
+	size_t size = 1024;
+	size_t ret, row, col;
+	long acc;
+	unsigned int cpu_total, mask_pos;
+	int fd = SAFE_OPEN("/proc/interrupts", O_RDONLY);

testcases/kernel/irq/irqbalance01.c line 76

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+
+	nr_cpus = 0;
+	nr_irqs = 0;
+
+	do {
+		size *= 2;
+		buf = SAFE_REALLOC(buf, size);
+		SAFE_LSEEK(fd, 0, SEEK_SET);
+		ret = SAFE_READ(0, fd, buf, size - 1);
+	} while (ret >= size - 1);

And actually this does not seem to work as expected for me, it reads about half of the table, stops shortly before 4096. I guess that we cannot really read it in one read() like this and that we are supposed to read it one $PAGE_SIZE at a time instead. So we need a loop that increases by $PAGE_SIZE and we would pass buf+len to the read, until the read returns bytes read.

Quoted 116 lines
testcases/kernel/irq/irqbalance01.c new file +116 -0
+	SAFE_CLOSE(fd);
+
+	if (ret < 1)
+		tst_brk(TBROK, "Empty /proc/interrupts?");
+
+	buf[ret] = '\0';
+
+	/* Count CPUs, header columns are like /CPU[0-9]+/ */
+	for (c = buf; *c != '\0' && *c != '\n'; c++) {
+		if (!strncmp(c, "CPU", 3))
+			nr_cpus++;
+	}
+
+	c++;
+	first_row = c;
+	/* Count IRQs, real IRQs start with /[0-9]+:/ */
+	while (*c != '\0') {
+		switch (*c) {
+		case ' ':
+		case '\t':
+		case '\n':
+		case '0' ... '9':
+			c++;
+			break;
+		case ':':
+			nr_irqs++;
+			/* fall-through */
+		default:
+			while (*c != '\n' && *c != '\0')
+				c++;
+		}
+	}
+
+	tst_res(TINFO, "Found %u CPUS, %u IRQs", nr_cpus, nr_irqs);
+
+	irq_ids = SAFE_REALLOC(irq_ids, nr_irqs * sizeof(*irq_ids));
+	irq_stats = SAFE_REALLOC(irq_stats, nr_cpus * (nr_irqs + 1) * sizeof(*irq_stats));
+	irq_affinity = SAFE_REALLOC(irq_affinity, nr_cpus * nr_irqs * sizeof(*irq_affinity));
+
+	c = first_row;
+	acc = -1;
+	row = col = 0;
+	/* Parse columns containing IRQ counts and IRQ IDs into acc. Ignore everything else. */
+	while (*c != '\0') {
+		switch (*c) {
+		case ' ':
+		case '\t':
+			if (acc >= 0) {
+				irq_stats[row * nr_cpus + col] = acc;
+				acc = -1;
+				col++;
+			}
+			break;
+		case '\n':
+			col = 0;
+			row++;
+			break;
+		case '0' ... '9':
+			if (acc == -1)
+				acc = 0;
+
+			acc *= 10;
+			acc += *c - '0';
+			break;
+		case ':':
+			irq_ids[row] = acc;
+			acc = -1;
+			break;
+		default:
+			acc = -1;
+			while (*c != '\n' && *c != '\0')
+				c++;
+			continue;
+		}
+
+		c++;
+	}
+
+	for (col = 0; col < nr_cpus; col++) {
+		cpu_total = 0;
+
+		for (row = 0; row < nr_irqs; row++)
+			cpu_total += irq_stats[row * nr_cpus + col];
+
+		irq_stats[row * nr_cpus + col] = cpu_total;
+	}
+
+	/* Read the CPU affinity masks for each IRQ. See bitmap_string() in the kernel (%*pb) */
+	for (row = 0; row < nr_irqs; row++) {
+		sprintf(path, "/proc/irq/%u/smp_affinity", irq_ids[row]);
+		ret = SAFE_FILE_READAT(0, path, buf, size);
+		if (ret < 1)
+			tst_brk(TBROK, "Empty /proc/irq/%u/smp_affinity?", irq_ids[row]);
+		c = buf;
+		col = 0;
+
+		while (*c != '\0') {
+			switch (*c) {
+			case '\n':
+			case ' ':
+			case ',':
+				c++;
+				continue;
+			case '0' ... '9':
+				acc = *c - '0';
+				break;
+			case 'a' ... 'f':
+				acc = 10 + *c - 'a';
+				break;
+			default:
+				tst_res(TINFO, "%u/smp_affnity: %s", irq_ids[row], buf);
+				tst_brk(TBROK, "Wasn't expecting 0x%02x", *c);
+			}
+
+			for (mask_pos = 0; mask_pos < 4; mask_pos++) {
+				if (mask_pos + col >= nr_cpus)

testcases/kernel/irq/irqbalance01.c line 203

Quoted 10 lines
testcases/kernel/irq/irqbalance01.c new file +10 -0
+					break;
+
+				irq_affinity[row * nr_cpus + (nr_cpus - 1 - col - mask_pos)] =
+					(acc & (8 >> mask_pos)) ? ALLOW : DENY;
+			}
+
+			col += mask_pos;
+			c++;
+		}
+	}

Actually having a closer look this whole parsing looks wrong and the reason why it does not work for me is that my machine has nr_cpus that is not power of two i.e. 12 in this case and the size of the mask seems to be rounded to 32 bits when presented in the proc files. I would have expected that the kernel would choose the closest power of two i.e. 16, but that does not seem to be the case here.

I guess that it would be actually easier to read the whole affinity into a string and then parse it from the end instead.

Quoted 92 lines
testcases/kernel/irq/irqbalance01.c new file +86 -0
+	free(buf);
+}
+
+static void print_irq_info(void)
+{
+	size_t row, col;
+	unsigned int count;
+	enum affinity aff;
+
+	for (row = 0; row < nr_irqs; row++) {
+		tst_printf("%5u:", irq_ids[row]);
+
+		for (col = 0; col < nr_cpus; col++) {
+			count = irq_stats[row * nr_cpus + col];
+			aff = irq_affinity[row * nr_cpus + col] == ALLOW ? '+' : '-';
+
+			tst_printf("%10u%c", count, aff);
+		}
+
+		tst_printf("\n");
+	}
+
+	tst_printf("Total:");
+
+	for (col = 0; col < nr_cpus; col++)
+		tst_printf("%11u", irq_stats[row * nr_cpus + col]);
+
+	tst_printf("\n");
+}
+
+static void evidence_of_change(void)
+{
+	size_t row, col, changed = 0;
+
+	for (row = 0; row < nr_irqs; row++) {
+		for (col = 0; col < nr_cpus; col++) {
+			if (!irq_stats[row * nr_cpus + col])
+				continue;
+
+			if (irq_affinity[row * nr_cpus + col] == ALLOW)
+				continue;
+
+			changed++;
+		}
+	}
+
+	tst_res(changed ? TPASS : TFAIL,
+		"Heuristic: Detected %zu irq-cpu pairs have been dissallowed",
+		changed);
+}
+
+static void setup(void)
+{
+	collect_irq_info();
+	print_irq_info();
+
+	if (nr_cpus < 1)
+		tst_brk(TBROK, "No CPUs found in /proc/interrupts?");
+
+	if (nr_irqs < 1)
+		tst_brk(TBROK, "No IRQs found in /proc/interrupts?");
+}
+
+static void run(void)
+{
+	collect_irq_info();
+
+	evidence_of_change();
+}
+
+static void cleanup(void)
+{
+	if (irq_ids)
+		free(irq_ids);
+	if (irq_stats)
+		free(irq_stats);
+	if (irq_affinity)
+		free(irq_affinity);
+}
+
+static struct tst_test test = {
+	.test_all = run,
+	.setup = setup,
+	.cleanup = cleanup,
+	.min_cpus = 2,
+};
Signature · 5 lines
2.31.1


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp
Signature · 2 lines
Cyril Hrubis
[email protected]