tst_test: Fail the test subprocess cannot be killed

This commit adds a small helper library to find a process(es) given a process group ID and dump their stacks.

Example output:

$ ./shmctl05
tst_test.c:1015: INFO: Timeout per run is 0h 00m 10s
Test timeouted, sending SIGKILL!
tst_test.c:1059: TFAIL: Test process child stuck in the kernel!
tst_find_pid.c:90: INFO: Pid 1272 stuck in kernel!
Kernel stacktrace follows:
[<ffffffffa3c12564>] __switch_to_asm+0x34/0x70
[<ffffffffa3c12570>] __switch_to_asm+0x40/0x70
[<ffffffffa3625761>] __switch_to+0x2c1/0x6e0
[<ffffffffa393e194>] call_rwsem_down_read_failed+0x14/0x30
[<ffffffffa3704802>] acct_collect+0x42/0x1a0
[<ffffffffa367d36a>] do_exit+0x74a/0xaf0
[<ffffffffa3c13d27>] rewind_stack_do_exit+0x17/0x20
[<ffffffffffffffff>] 0xffffffffffffffff
tst_test.c:1061: FAIL: Congratulation, likely test hit a kernel bug.

TODO: The main test process uses signal handler and alarm to call _exit if the
child process that executes the actuall test timeouts. We need to redesign
this if we want to dump the stack in that case as well.

Signed-off-by Cyril Hrubis <[email protected]>

CC Jan Stancek <[email protected]>

---
 include/tst_dump_stacks.h |  25 +++++++++++
 lib/tst_dump_stacks.c     | 108
 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/tst_test.c            |   3 +-
 3 files changed, 135 insertions(+), 1 deletion(-)
 create mode 100644 include/tst_dump_stacks.h
 create mode 100644 lib/tst_dump_stacks.c

include/tst_dump_stacks.h new file +25 -0

new file mode 100644
index 000000000..643cc58a8
@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2018 Cyril Hrubis <[email protected]>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TST_DUMP_STACKS__
+#define TST_DUMP_STACKS__
+
+void tst_dump_stacks_by_pgid(pid_t pgid);
+
+void tst_dump_stack_by_pid(pid_t pid);
+
+#endif /* TST_DUMP_STACKS__ */

lib/tst_dump_stacks.c new file +108 -0

new file mode 100644
index 000000000..aa97c6820
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2018 Cyril Hrubis <[email protected]>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <ctype.h>
+#include <stdio.h>
+
+#define TST_NO_DEFAULT_MAIN 1
+#include "tst_test.h"
+
+static void *process_search_init(void)
+{
+	DIR *dir = SAFE_OPENDIR("/proc/");
+
+	return dir;
+}
+
+static int is_number(const char *str)
+{
+	do {
+		if (!isdigit(*str))
+			return 0;
+	} while (*(++str));
+
+	return 1;
+}
+
+static int process_search_pgid_next(void *pid_search, pid_t pgid)
+{
+	struct dirent *ent;
+	DIR *dir = pid_search;
+	char path[1024];
+	int ppgid, pid;
+	FILE *f;
+
+	while ((ent = readdir(dir))) {
+		if (ent->d_type != DT_DIR)
+			continue;
+		if (!is_number(ent->d_name))
+			continue;
+
+		snprintf(path, sizeof(path), "/proc/%s/stat", ent->d_name);
+
+		f = fopen(path, "r");
+		if (!f)
+			continue;
+
+		if (fscanf(f, "%i %*s %*c %*i %i", &pid, &ppgid) != 2) {
+			tst_res(TWARN, "fscanf(%s) failed!", ent->d_name);
+			fclose(f);
+			continue;
+		}
+
+		fclose(f);
+
+		if (ppgid == pgid)
+			break;
+	}
+
+	if (ent)
+		return pid;
+
+	closedir(dir);
+	return -1;
+}
+
+void tst_dump_stack_by_pid(pid_t pid)
+{
+	int fd, len;
+	char buf[512];
+	char path[1024];
+
+	tst_res(TINFO, "Pid %i stuck in kernel!", pid);
+
+	fprintf(stderr, "Kernel stacktrace follows:\n");
+	fflush(stderr);
+
+	snprintf(path, sizeof(path), "/proc/%i/stack", pid);
+
+	fd = SAFE_OPEN(path, O_RDONLY);
+
+	while ((len = SAFE_READ(0, fd, buf, sizeof(buf))) > 0)
+		SAFE_WRITE(1, 2, buf, len);
+
+	SAFE_CLOSE(fd);
+}
+
+void tst_dump_stacks_by_pgid(pid_t pgid)
+{
+	void *ps = process_search_init();
+	int pid;
+
+	while ((pid = process_search_pgid_next(ps, pgid)) != -1)
+		tst_dump_stack_by_pid(pid);
+}

lib/tst_test.c +0 -0

index 329168a24..d9476c02c 100644

If there are any leftover children the main test process will likely be killed while sleeping in wait(). That is because all child processes are either waited explicitely by the test code or implicitly by the test library.

We also send SIGKILL to the whole process group, so if one of the children continues to live for long enough that very likely means that it ended up stuck in the kernel.

So if there are any processes left with in the process group we created once the process group leader i.e. main test process has been waited for we loop for a short while to give the init daemon chance to reap the process after it has been reparented and if that does not happen for a few seconds we declare the process to be stuck in the kernel.

Signed-off-by Cyril Hrubis <[email protected]>

CC Eric Biggers <[email protected]>

---
 lib/tst_test.c | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

lib/tst_test.c +1 -1

index 80808854e..329168a24 100644
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016 Cyril Hrubis <[email protected]>
+ * Copyright (c) 2015-2018 Cyril Hrubis <[email protected]>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

lib/tst_test.c +11 -1

+
+               usleep(sleep);
+               sleep*=2;
+
+               if (retries++ <= 14)
+                       continue;
+
+               tst_res(TFAIL, "Test process child stuck in the kernel!");
+               tst_brk(TFAIL, "Congratulation, likely test hit a kernel bug.");
+       }
+
        if (WIFEXITED(status) && WEXITSTATUS(status))
                return WEXITSTATUS(status);
 
--

2.13.6

--
Mailing list info: https://lists.linux.it/listinfo/ltp

Hi Cyril,

Quoted 36 lines · Cyril Hrubis

If there are any leftover children the main test process will likely be killed while sleeping in wait(). That is because all child processes are either waited explicitely by the test code or implicitly by the test library.

We also send SIGKILL to the whole process group, so if one of the children continues to live for long enough that very likely means that it ended up stuck in the kernel.

So if there are any processes left with in the process group we created once the process group leader i.e. main test process has been waited for we loop for a short while to give the init daemon chance to reap the process after it has been reparented and if that does not happen for a few seconds we declare the process to be stuck in the kernel.

Signed-off-by Cyril Hrubis <[email protected]>

CC Eric Biggers <[email protected]>

---
 lib/tst_test.c | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

lib/tst_test.c +1 -1

index 80808854e..329168a24 100644
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016 Cyril Hrubis <[email protected]>
+ * Copyright (c) 2015-2018 Cyril Hrubis <[email protected]>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by

@@ -1047,6 +1047,21 @@ static int fork_testrun(void)
        alarm(0);
        SAFE_SIGNAL(SIGINT, SIG_DFL);
 

lib/tst_test.c static int fork_testrun(void) · line 1053

Quoted 4 lines

lib/tst_test.c +4 -0

+       unsigned int sleep = 100;
+       unsigned int retries = 0;
+
+       while (kill(-test_pid, 0) == 0) {

I'm a little worried about here, image that, if a process_A(test_pid) exist to make function kill(-test_pid, 0) return 0 at first time, then we go into this while loop, but during the sleeping time process_A exit and system reuse the test_pid to another process_B, we will still keep looping and very probably make mistake to report TFAIL(with stack of process_B dump to ltp user in PATCH 2/2).

Quoted 19 lines

lib/tst_test.c +11 -1

+
+               usleep(sleep);
+               sleep*=2;
+
+               if (retries++ <= 14)
+                       continue;
+
+               tst_res(TFAIL, "Test process child stuck in the kernel!");
+               tst_brk(TFAIL, "Congratulation, likely test hit a kernel bug.");
+       }
+
        if (WIFEXITED(status) && WEXITSTATUS(status))
                return WEXITSTATUS(status);
 
--

2.13.6

Quoted 2 lines

--
Mailing list info: https://lists.linux.it/listinfo/ltp

--
Regards,
Li Wang

Quoted 6 lines

I'm a little worried about here, image that, if a process_A(test_pid) exist to make function kill(-test_pid, 0) return 0 at first time, then we go into this while loop, but during the sleeping time process_A exit and system reuse the test_pid to another process_B, we will still keep looping and very probably make mistake to report TFAIL(with stack of process_B dump to ltp user in PATCH 2/2).

Maybe we could verify the content of '/proc/test_pid/cmdline' in this loop to make sure test_pid is still using by the process we wanted?

lib/tst_test.c static int fork_testrun(void) · line 1053

Quoted 4 lines

lib/tst_test.c +4 -0

+       unsigned int sleep = 100;
+       unsigned int retries = 0;
+
+       while (kill(-test_pid, 0) == 0) {

I'm a little worried about here, image that, if a process_A(test_pid) exist to make function kill(-test_pid, 0) return 0 at first time, then we go into this while loop, but during the sleeping time process_A exit and system reuse the test_pid to another process_B, we will still keep looping and very probably make mistake to report TFAIL(with stack of process_B dump to ltp user in PATCH 2/2).