Skip to contentSkip to conversation

mm: only interrupt taking all mm locks on fatal signal

PATCH In review Mikulas Patocka 8 participants 18 messages Jan 4 – Jan 7

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH v3 2/3 mm: only interrupt taking all mm locks on fatal signal

If a process sets up a timer that periodically sends a signal in short intervals and if it executes some kernel code that calls mm_take_all_locks, we get random -EINTR failures.

The function mm_take_all_locks fails with -EINTR if there is pending signal. The -EINTR is propagated up the call stack to userspace and userspace fails if it gets this error.

In order to fix these failures, this commit changes signal_pending(current) to fatal_signal_pending(current) in mm_take_all_locks, so that it is interrupted only if the signal is actually killing the process.

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

include <stdio.h>
include <stdlib.h>
include <unistd.h>
include <string.h>
include <signal.h>
include <sys/time.h>

define CL_TARGET_OPENCL_VERSION 300
include <CL/opencl.h>

static void fn(void)

{
	while (1) {
		int32_t err;
		cl_device_id device;
		err = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 1, &device, NULL);
		if (err != CL_SUCCESS) {
			fprintf(stderr, "clGetDeviceIDs failed: %d\n", err);
			exit(1);
		}
		write(2, "-", 1);
	}
}

static void alrm(int sig)

{
	write(2, ".", 1);
}

int main(void)

{
	struct itimerval it;
	struct sigaction sa;
	memset(&sa, 0, sizeof sa);
	sa.sa_handler = alrm;
	sa.sa_flags = SA_RESTART;
	sigaction(SIGALRM, &sa, NULL);
	it.it_interval.tv_sec = 0;
	it.it_interval.tv_usec = 50;
	it.it_value.tv_sec = 0;
	it.it_value.tv_usec = 50;
	setitimer(ITIMER_REAL, &it, NULL);
	fn();
	return 1;
}

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

---
 mm/vma.c |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
mm/vma.c +4 -4
Index: mm/mm/vma.c
===================================================================
@@ -2166,14 +2166,14 @@ int mm_take_all_locks(struct mm_struct *
 	 * is reached.
 	 */
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		vma_start_write(vma);
 	}
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				is_vm_hugetlb_page(vma))
@@ -2182,7 +2182,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				!is_vm_hugetlb_page(vma))
@@ -2191,7 +2191,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->anon_vma)
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
Acked-by on v3 2/3
#1 lore raw
Quoted 71 lines · Mikulas Patocka

If a process sets up a timer that periodically sends a signal in short intervals and if it executes some kernel code that calls mm_take_all_locks, we get random -EINTR failures.

The function mm_take_all_locks fails with -EINTR if there is pending signal. The -EINTR is propagated up the call stack to userspace and userspace fails if it gets this error.

In order to fix these failures, this commit changes signal_pending(current) to fatal_signal_pending(current) in mm_take_all_locks, so that it is interrupted only if the signal is actually killing the process.

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

include <stdio.h>
include <stdlib.h>
include <unistd.h>
include <string.h>
include <signal.h>
include <sys/time.h>

define CL_TARGET_OPENCL_VERSION 300
include <CL/opencl.h>

static void fn(void)

{
	while (1) {
		int32_t err;
		cl_device_id device;
		err = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 1, &device, NULL);
		if (err != CL_SUCCESS) {
			fprintf(stderr, "clGetDeviceIDs failed: %d\n", err);
			exit(1);
		}
		write(2, "-", 1);
	}
}

static void alrm(int sig)

{
	write(2, ".", 1);
}

int main(void)

{
	struct itimerval it;
	struct sigaction sa;
	memset(&sa, 0, sizeof sa);
	sa.sa_handler = alrm;
	sa.sa_flags = SA_RESTART;
	sigaction(SIGALRM, &sa, NULL);
	it.it_interval.tv_sec = 0;
	it.it_interval.tv_usec = 50;
	it.it_value.tv_sec = 0;
	it.it_value.tv_usec = 50;
	setitimer(ITIMER_REAL, &it, NULL);
	fn();
	return 1;
}
Quoted 8 lines

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

This makes sense to me as a backportable bugfix. But I wonder if going forward we should rather make all that locking killable instead of the hopeful checks between individual lock attempts.

Quoted 10 lines
---
 mm/vma.c |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

Index: mm/mm/vma.c
===================================================================

mm/vma.c int mm_take_all_locks(struct mm_struct * · line 2171

Quoted 7 lines
mm/vma.c +1 -1
@@ -2166,14 +2166,14 @@ int mm_take_all_locks(struct mm_struct *
 	 * is reached.
 	 */
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		vma_start_write(vma);

E.g. here I think we already added a killable variant recently?

Quoted 28 lines
mm/vma.c +3 -3
 	}
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				is_vm_hugetlb_page(vma))
@@ -2182,7 +2182,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				!is_vm_hugetlb_page(vma))
@@ -2191,7 +2191,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->anon_vma)
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
Reviewed-by
#2 lore raw
Quoted 75 lines · Vlastimil Babka
Quoted 74 lines · Mikulas Patocka

If a process sets up a timer that periodically sends a signal in short intervals and if it executes some kernel code that calls mm_take_all_locks, we get random -EINTR failures.

The function mm_take_all_locks fails with -EINTR if there is pending signal. The -EINTR is propagated up the call stack to userspace and userspace fails if it gets this error.

In order to fix these failures, this commit changes signal_pending(current) to fatal_signal_pending(current) in mm_take_all_locks, so that it is interrupted only if the signal is actually killing the process.

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

include <stdio.h>
include <stdlib.h>
include <unistd.h>
include <string.h>
include <signal.h>
include <sys/time.h>

define CL_TARGET_OPENCL_VERSION 300
include <CL/opencl.h>

static void fn(void)

{
	while (1) {
		int32_t err;
		cl_device_id device;
		err = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 1, &device, NULL);
		if (err != CL_SUCCESS) {
			fprintf(stderr, "clGetDeviceIDs failed: %d\n", err);
			exit(1);
		}
		write(2, "-", 1);
	}
}

static void alrm(int sig)

{
	write(2, ".", 1);
}

int main(void)

{
	struct itimerval it;
	struct sigaction sa;
	memset(&sa, 0, sizeof sa);
	sa.sa_handler = alrm;
	sa.sa_flags = SA_RESTART;
	sigaction(SIGALRM, &sa, NULL);
	it.it_interval.tv_sec = 0;
	it.it_interval.tv_usec = 50;
	it.it_value.tv_sec = 0;
	it.it_value.tv_usec = 50;
	setitimer(ITIMER_REAL, &it, NULL);
	fn();
	return 1;
}

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

Quoted 7 lines
Quoted 5 lines

Also feel free to add:

On assumption that nobody is explicitly relying on bizarre-o 'interrupt this if _any_ signal arises'. But since it's making _actual workloads_ break I don't see how this can be wrong.

Quoted 4 lines

This makes sense to me as a backportable bugfix. But I wonder if going forward we should rather make all that locking killable instead of the hopeful checks between individual lock attempts.

Agreed. But anything like that should be a follow-up, let's get this backported first.

Quoted 6 lines
Quoted 6 lines

---
 mm/vma.c |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
Quoted 14 lines

mm/vma.c int mm_take_all_locks(struct mm_struct * · line 2171

Quoted 12 lines

Index: mm/mm/vma.c
===================================================================

mm/vma.c +1 -1
@@ -2166,14 +2166,14 @@ int mm_take_all_locks(struct mm_struct *
 	 * is reached.
 	 */
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		vma_start_write(vma);

E.g. here I think we already added a killable variant recently?

Definitely unbackportable ;)

Quoted 31 lines
Quoted 29 lines
mm/vma.c +3 -3
 	}
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				is_vm_hugetlb_page(vma))
@@ -2182,7 +2182,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				!is_vm_hugetlb_page(vma))
@@ -2191,7 +2191,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->anon_vma)
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
#3 lore raw
In reply to Mikulas Patocka
Quoted 4 lines · Mikulas Patocka

If a process sets up a timer that periodically sends a signal in short intervals and if it executes some kernel code that calls mm_take_all_locks, we get random -EINTR failures.

Quoted 8 lines

The function mm_take_all_locks fails with -EINTR if there is pending signal. The -EINTR is propagated up the call stack to userspace and userspace fails if it gets this error.

In order to fix these failures, this commit changes signal_pending(current) to fatal_signal_pending(current) in mm_take_all_locks, so that it is interrupted only if the signal is actually killing the process.

I may be missing context because I didn't get 1/3 of this patch set, nor can I find it in my ML searching. Nor did I get the cover letter, or find it. Is this series threaded?

What you are doing is changing a really horrible loop across all VMAs, that happens 4 times, to a less interruptible method.

I'm not sure I'm okay with this. Everyone else does seem fine with it, because userspace basically never checks error codes for a retry (or really anything, and sometimes not even for an error at all).

But this could potentially have larger consequences for those applications that register signal handlers, couldn't it? That is, they expect to get a return based on some existing code but now it won't return and the application is forced to wait for all locks to be taken regardless of how long it takes - or to force kill the application?

We regularly get people requesting the default number of vmas be increased. This means that processes that approach max_map_count will wait until 4 loops through the vmas before they can be interrupted, or they have to kill the process.

Quoted 5 lines

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

If you only get the error message sometimes, does that mean there is another signal check that isn't covered by this change - or another call path?

Quoted 7 lines

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

This seems to imply that all signal handling should only happen on fatal signals?

...

Quoted 3 lines

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

They aren't random failures, they are a response to a signal sent to the process that may be taking a very long time to do something.

I really don't see how continuously sending signals at a short interval interrupting system calls can be considered random failures, especially when the return is -EINTR which literally means "Interrupted system call". How else would you interrupt a system call, if not a signal?

I feel like we are making the code less versatile to work around the fact that userspace didn't realise that system calls could be interrupted without a fatal signal.

And from that view, I consider this a functional change and not a bug fix.

Thanks,
Liam

#4 lore raw
Quoted 3 lines · Liam R. Howlett

I may be missing context because I didn't get 1/3 of this patch set, nor can I find it in my ML searching. Nor did I get the cover letter, or find it. Is this series threaded?

You can ignore the patch 1/3, it just changes memory management test suite.

Quoted 6 lines

What you are doing is changing a really horrible loop across all VMAs, that happens 4 times, to a less interruptible method.

I'm not sure I'm okay with this. Everyone else does seem fine with it, because userspace basically never checks error codes for a retry (or really anything, and sometimes not even for an error at all).

Everyone else does seem fine because they don't use periodic signals :-)

OpenCL is not the first thing that got broken by periodic signals. In the past, I found bugs in Linux/alpha, Linux/hppa, Cygwin, Intel Software Developer's Emulator regarding periodic signals. fork() was also buggy and it was fixed by c3ad2c3b02e953ead2b8d52a0c9e70312930c3d0.

Quoted 5 lines

But this could potentially have larger consequences for those applications that register signal handlers, couldn't it? That is, they expect to get a return based on some existing code but now it won't return and the application is forced to wait for all locks to be taken regardless of how long it takes - or to force kill the application?

Do you have any userspace application that expects open("/dev/kfd") to be interrupted with -EINTR and breaks when it isn't?

Quoted 6 lines

We regularly get people requesting the default number of vmas be increased. This means that processes that approach max_map_count will wait until 4 loops through the vmas before they can be interrupted, or they have to kill the process.

Quoted 1 line
Quoted 8 lines
Quoted 4 lines

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

If you only get the error message sometimes, does that mean there is another signal check that isn't covered by this change - or another call path?

This call path is also triggered by -EINTR from mm_take_all_locks:
"init_user_pages -> amdgpu_hmm_register -> mmu_interval_notifier_insert -> mmu_notifier_register -> __mmu_notifier_register -> mm_take_all_locks -> return -EINTR". I am not expert in the GPU code, so I don't know how much serious it is.

Quoted 7 lines
Quoted 6 lines

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

Quoted 7 lines

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

This seems to imply that all signal handling should only happen on fatal signals?

No - the kernel should do what applications expect. open is (according to the man page) supposed to be interrupted when opening slow devices (for example fifo). I'm wondering whether /dev/kfd should be considered a slow device or not.

Quoted 5 lines

...

Quoted 3 lines

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

Quoted 7 lines

They aren't random failures, they are a response to a signal sent to the process that may be taking a very long time to do something.

I really don't see how continuously sending signals at a short interval interrupting system calls can be considered random failures, especially when the return is -EINTR which literally means "Interrupted system call". How else would you interrupt a system call, if not a signal?

The AMDGPU OpenCL implementation attempts to open /dev/kfd and if it gets -EINTR, it behaves as if OpenCL were unavailable - it won't report itself in clGetPlatformIDs and it will make clGetDeviceIDs fail.

So, we are dealing with random failures - any single signal received at wrong time can make OpenCL fail.

Even if I disabled the periodic timer, the failure could be triggered by other signals, for example SIGWINCH when the user resizes the terminal, or SIGCHLD when a subprocess exits.

Quoted 9 lines

I feel like we are making the code less versatile to work around the fact that userspace didn't realise that system calls could be interrupted without a fatal signal.

And from that view, I consider this a functional change and not a bug fix.

Thanks,
Liam

In practice, I use 10ms timer and I get occasional OpenCL failures. In the test example, I used 50us timer, so that it is reproduced reliably - but decreasing the timer frequency doesn't fix the failure, it just makes it happen less often.

Mikulas

#5 lore raw
Quoted 11 lines · Mikulas Patocka
Quoted 3 lines · Liam R. Howlett

I may be missing context because I didn't get 1/3 of this patch set, nor can I find it in my ML searching. Nor did I get the cover letter, or find it. Is this series threaded?

You can ignore the patch 1/3, it just changes memory management test suite.

Quoted 8 lines
Quoted 6 lines

What you are doing is changing a really horrible loop across all VMAs, that happens 4 times, to a less interruptible method.

I'm not sure I'm okay with this. Everyone else does seem fine with it, because userspace basically never checks error codes for a retry (or really anything, and sometimes not even for an error at all).

Everyone else does seem fine because they don't use periodic signals :-)

I meant everyone else on this thread.

Quoted 6 lines

OpenCL is not the first thing that got broken by periodic signals. In the past, I found bugs in Linux/alpha, Linux/hppa, Cygwin, Intel Software Developer's Emulator regarding periodic signals. fork() was also buggy and it was fixed by c3ad2c3b02e953ead2b8d52a0c9e70312930c3d0.

Quoted 8 lines
Quoted 5 lines

But this could potentially have larger consequences for those applications that register signal handlers, couldn't it? That is, they expect to get a return based on some existing code but now it won't return and the application is forced to wait for all locks to be taken regardless of how long it takes - or to force kill the application?

Do you have any userspace application that expects open("/dev/kfd") to be interrupted with -EINTR and breaks when it isn't?

Your change doesn't just affect /dev/kfd though, it affects any call path that leads to looping across all vmas four times in a row.

And since this could take a long time, I'm not sure it's a good idea to stop responding to all non-fatal signal.

Quoted 16 lines
Quoted 7 lines

We regularly get people requesting the default number of vmas be increased. This means that processes that approach max_map_count will wait until 4 loops through the vmas before they can be interrupted, or they have to kill the process.

Quoted 1 line
Quoted 8 lines
Quoted 4 lines

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

If you only get the error message sometimes, does that mean there is another signal check that isn't covered by this change - or another call path?

Quoted 5 lines

This call path is also triggered by -EINTR from mm_take_all_locks:
"init_user_pages -> amdgpu_hmm_register -> mmu_interval_notifier_insert -> mmu_notifier_register -> __mmu_notifier_register -> mm_take_all_locks -> return -EINTR". I am not expert in the GPU code, so I don't know how much serious it is.

Okay, so the other call paths also end up getting the -EINTR from this function? Can you please add that detail to the commit message?

This means that -EINTR can no longer be returned from open(), right? Otherwise you are just reducing a race condition between open() and a signal entering from your timer.

Quoted 13 lines
Quoted 13 lines
Quoted 7 lines

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

Quoted 7 lines
Quoted 2 lines

This seems to imply that all signal handling should only happen on fatal signals?

No - the kernel should do what applications expect. open is (according to the man page) supposed to be interrupted when opening slow devices (for example fifo). I'm wondering whether /dev/kfd should be considered a slow device or not.

I disagree. The kernel should do what was agreed to in an API, otherwise our whole deck of cards falls over. We may not even have the resources to do what the applications expect. One could also code an application that expects to violate the security of the system.

This is documented behaviour from the open() call, so it certainly sounds like an issue if userspace doesn't expect this return.

Any other -EINTR system call will also cause you problems since you continuously send signals to your process, so we'll have to change them all for this to work?

Quoted 9 lines
Quoted 9 lines

...

Quoted 3 lines

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

They aren't random failures, they are a response to a signal sent to the process that may be taking a very long time to do something.

Quoted 8 lines
Quoted 4 lines

I really don't see how continuously sending signals at a short interval interrupting system calls can be considered random failures, especially when the return is -EINTR which literally means "Interrupted system call". How else would you interrupt a system call, if not a signal?

The AMDGPU OpenCL implementation attempts to open /dev/kfd and if it gets -EINTR, it behaves as if OpenCL were unavailable - it won't report itself in clGetPlatformIDs and it will make clGetDeviceIDs fail.

This is the userspace ignoring what the error code means and just aborting on any error. This is a change in behaviour on the kernel side to work around what they are doing.

It also sounds like it can be avoided by userspace not sending signals during the open process, or even to retry at a higher level if a recoverable error occurs.

Again, I'm not saying that this change isn't the right thing to do, but it's not the kernels fault that userspace is aborting on -EINTR. And it's not the kernels fault that the userspace application is sending signals during a system call that will respond to those signals.

This is changing the kernels behaviour to avoid having to deal with two cascading failures in userspace: 1. The opencl not retrying on interrupted open calls and 2. sending signals to a process in the middle of a system call that will respond to signals.

I am saying that calling this a bug fix is wrong.

Even though Vlastimil has pointed out that this locking did not happen prior to 7906d00cd1f6, the contract to userspace for the open() system call could still return -EINTR and that change predates amdgpu's kfd in 2015. So I don't consider the 'kernel broke userspace' argument in this case.

Quoted 3 lines

So, we are dealing with random failures - any single signal received at wrong time can make OpenCL fail.

I feel like we are working with two different definitions of random. I would consider a failure caused by a periodic signal to be an expected and predictable event.

"random failures in any code that calls mm_take_all_locks" makes it sound like we're going to hit some uninitialized variable that sometimes isn't zero and exit, but really we're receiving a signal that may need to do something within a reasonable amount of time.

Quoted 4 lines

Even if I disabled the periodic timer, the failure could be triggered by other signals, for example SIGWINCH when the user resizes the terminal, or SIGCHLD when a subprocess exits.

Those are also not random, they are expected signals caused by events.

I'm trying to say this git commit message is wrong and misleading.

Quoted 8 lines
Quoted 8 lines

I feel like we are making the code less versatile to work around the fact that userspace didn't realise that system calls could be interrupted without a fatal signal.

And from that view, I consider this a functional change and not a bug fix.

Quoted 7 lines
Quoted 2 lines

Thanks,
Liam

In practice, I use 10ms timer and I get occasional OpenCL failures. In the test example, I used 50us timer, so that it is reproduced reliably - but decreasing the timer frequency doesn't fix the failure, it just makes it happen less often.

This does not fit my definition of random, bug, or failure.

Thanks,
Liam

#6 lore raw
Quoted 6 lines · Liam R. Howlett
Quoted 5 lines · Mikulas Patocka
Quoted 4 lines

If you only get the error message sometimes, does that mean there is another signal check that isn't covered by this change - or another call path?

Quoted 8 lines
Quoted 5 lines

This call path is also triggered by -EINTR from mm_take_all_locks:
"init_user_pages -> amdgpu_hmm_register -> mmu_interval_notifier_insert -> mmu_notifier_register -> __mmu_notifier_register -> mm_take_all_locks -> return -EINTR". I am not expert in the GPU code, so I don't know how much serious it is.

Okay, so the other call paths also end up getting the -EINTR from this function? Can you please add that detail to the commit message?

Yes. I'd like to ask the GPU people to look at it and say how much damage this -EINTR could do. I don't know - I just saw the messages "Failed to register MMU notifier: -4" in the syslog.

Quoted 3 lines

This means that -EINTR can no longer be returned from open(), right? Otherwise you are just reducing a race condition between open() and a signal entering from your timer.

EINTR can be returned from open() in cases when it was historically behaving this way - such as opening a fifo when there is no matching process having it open.

But I think that opening /dev/kfd doesn't fall into this category.

NFS has an "intr" flag that makes the filesystem syscalls interruptible by signals. It is off by default, because many programs don't expect EINTR when opening, reading or writing plain files on a filesystem.

Quoted 3 lines

Any other -EINTR system call will also cause you problems since you continuously send signals to your process, so we'll have to change them all for this to work?

I use SA_RESTART for the signals. And I retry all the syscalls on EINTR just in case SA_RESTART didn't work. So, I don't experience random failures in my code due to the periodic signal.

But there is code that I have no control over - such as the OpenCL shared library.

Quoted 6 lines

This is the userspace ignoring what the error code means and just aborting on any error. This is a change in behaviour on the kernel side to work around what they are doing.

It also sounds like it can be avoided by userspace not sending signals during the open process, or even to

So far, I worked around this issue by blocking all signals around clGetPlatformIDs and clGetDeviceIDs - but this is a hack.

Quoted 1 line

retry at a higher level if a recoverable error occurs.

If clGetDeviceIDs fails and I call clGetDeviceIDs again, it doesn't even attempt to open /dev/kfd again and fails right away. So, I can't work around it by retrying it.

Quoted 5 lines
Quoted 3 lines

Even if I disabled the periodic timer, the failure could be triggered by other signals, for example SIGWINCH when the user resizes the terminal, or SIGCHLD when a subprocess exits.

Those are also not random, they are expected signals caused by events.

From the process's point of view, they are random - the process doesn't know when the user will drag the corner of the terminal window and resize it. If the process spawns a subprocess, it cannot predict when will the subprocess exit and SIGCHLD will be delivered.

If we don't change it, we end up with unreliable software stack that can fail during rare events, such as dragging the corner of the window.

Quoted 1 line

I'm trying to say this git commit message is wrong and misleading.

OK, so I'll try to rewrite the commit message and submit version 4 of the patch.

Mikulas

reviewer
#7 lore raw
Quoted 27 lines · Mikulas Patocka
Quoted 6 lines · Liam R. Howlett
Quoted 5 lines · Mikulas Patocka
Quoted 4 lines

If you only get the error message sometimes, does that mean there is another signal check that isn't covered by this change - or another call path?

Quoted 8 lines
Quoted 5 lines

This call path is also triggered by -EINTR from mm_take_all_locks:
"init_user_pages -> amdgpu_hmm_register -> mmu_interval_notifier_insert -> mmu_notifier_register -> __mmu_notifier_register -> mm_take_all_locks -> return -EINTR". I am not expert in the GPU code, so I don't know how much serious it is.

Okay, so the other call paths also end up getting the -EINTR from this function? Can you please add that detail to the commit message?

Yes. I'd like to ask the GPU people to look at it and say how much damage this -EINTR could do. I don't know - I just saw the messages "Failed to register MMU notifier: -4" in the syslog.

Quoted 3 lines

This means that -EINTR can no longer be returned from open(), right? Otherwise you are just reducing a race condition between open() and a signal entering from your timer.

Quoted 6 lines

EINTR can be returned from open() in cases when it was historically behaving this way - such as opening a fifo when there is no matching process having it open.

But I think that opening /dev/kfd doesn't fall into this category.

Well, it's a device - opening can and often does have side-effects. It's not too far-fetched to -EINTR here.

Quoted 8 lines

NFS has an "intr" flag that makes the filesystem syscalls interruptible by signals. It is off by default, because many programs don't expect EINTR when opening, reading or writing plain files on a filesystem.

Quoted 3 lines

Any other -EINTR system call will also cause you problems since you continuously send signals to your process, so we'll have to change them all for this to work?

Quoted 6 lines

I use SA_RESTART for the signals. And I retry all the syscalls on EINTR just in case SA_RESTART didn't work. So, I don't experience random failures in my code due to the periodic signal.

But there is code that I have no control over - such as the OpenCL shared library.

Right. So I am wondering if just returning -ERESTARTSYS (whether in mm_take_all_locks(), or in the AMD driver) would satisfy both parties.

Folks installing and using signals need to pay attention and set SA_RESTART, but that's already best practice when dealing with third-party code. open(2) should be transparently restartable.

WDYT?

Signature · 1 line
Pedro
#8 lore raw
Quoted 11 lines · Pedro Falcato
Quoted 10 lines · Mikulas Patocka
Quoted 2 lines · Liam R. Howlett

Okay, so the other call paths also end up getting the -EINTR from this function? Can you please add that detail to the commit message?

Yes. I'd like to ask the GPU people to look at it and say how much damage this -EINTR could do. I don't know - I just saw the messages "Failed to register MMU notifier: -4" in the syslog.

I read the ROCm source code and it seems to restart all ioctls on EINTR or EAGAIN correctly.

So, this syslog message is harmless.

The only problem is that open() is not restarted (even if the signal has the SA_RESTART flag).

Quoted 11 lines
Quoted 10 lines
Quoted 3 lines

Any other -EINTR system call will also cause you problems since you continuously send signals to your process, so we'll have to change them all for this to work?

I use SA_RESTART for the signals. And I retry all the syscalls on EINTR just in case SA_RESTART didn't work. So, I don't experience random failures in my code due to the periodic signal.

But there is code that I have no control over - such as the OpenCL shared library.

Quoted 8 lines

Right. So I am wondering if just returning -ERESTARTSYS (whether in mm_take_all_locks(), or in the AMD driver) would satisfy both parties.

Folks installing and using signals need to pay attention and set SA_RESTART, but that's already best practice when dealing with third-party code. open(2) should be transparently restartable.

WDYT?

I tested it with -ERESTARTSYS and it works. I get the messages "init_user_pages: Failed to register MMU notifier: -512" in the syslog - but that is benign.

Mikulas

#9 lore raw
In reply to Mikulas Patocka
Quoted 3 lines · Mikulas Patocka

OK, so I'll try to rewrite the commit message and submit version 4 of the patch.

I'd suggest (yes, it's the opposite of what Lorenzo asked for earlier, sorry) to make the necessary tools/testing/vma/vma_internal.h modification part of this patch again and not separate patch. It will remove the need to indicate it's stable dep of the fix, as it will be all self-contained.

Yes, backporting to a kernel that doesn't yet have tools/testing/vma/vma_internal.h will make the patch not apply, but dropping a non-applicable hunk is trivial for stable maintainers. There will be more involved conflicts anyway due to mm_take_all_locks() moving between files.

Thanks,
Vlastimil

Quoted 2 lines

Mikulas

#10 lore raw
In reply to Mikulas Patocka
Quoted 6 lines · Mikulas Patocka
Quoted 3 lines · Liam R. Howlett

Any other -EINTR system call will also cause you problems since you continuously send signals to your process, so we'll have to change them all for this to work?

Quoted 6 lines

I use SA_RESTART for the signals. And I retry all the syscalls on EINTR just in case SA_RESTART didn't work. So, I don't experience random failures in my code due to the periodic signal.

But there is code that I have no control over - such as the OpenCL shared library.

AFAIK all current OpenCL support code for AMD GPUs is open source. I see no reason why it couldn't be fixed.

Signature · 2 lines
Earthling Michel Dänzer       \        GNOME / Xwayland / Mesa developer
https://redhat.com             \               Libre software enthusiast
#11 lore raw
In reply to Liam R. Howlett
Quoted 6 lines · Liam R. Howlett

* Mikulas Patocka <[email protected]> [260104 16:17]:

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

I'd go further than that. If user space fails to retry the system call in response to -EINTR, that's a user-space bug, period. It can happen anytime for any number of other reasons. (That most system calls happen to get away without it most of the time doesn't make it not a bug)

(This is assuming the system call can be safely retried, otherwise returning -EINTR to user space for non-fatal signals would be a kernel bug)

Quoted 11 lines
Quoted 2 lines

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

They aren't random failures, they are a response to a signal sent to the process that may be taking a very long time to do something.

I really don't see how continuously sending signals at a short interval interrupting system calls can be considered random failures, especially when the return is -EINTR which literally means "Interrupted system call". How else would you interrupt a system call, if not a signal?

Quoted 6 lines

I feel like we are making the code less versatile to work around the fact that userspace didn't realise that system calls could be interrupted without a fatal signal.

And from that view, I consider this a functional change and not a bug fix.

I agree.

Signature · 2 lines
Earthling Michel Dänzer       \        GNOME / Xwayland / Mesa developer
https://redhat.com             \               Libre software enthusiast
#12 lore raw
Quoted 8 lines · Michel Dänzer
Quoted 6 lines · Liam R. Howlett

* Mikulas Patocka <[email protected]> [260104 16:17]:

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

Quoted 4 lines

I'd go further than that. If user space fails to retry the system call in response to -EINTR, that's a user-space bug, period. It can happen anytime for any number of other reasons. (That most system calls happen to get away without it most of the time doesn't make it not a bug)

So, I tried this - just for fun - and the machine doesn't even boot. I get a lot of errors about inability to open particular files on the console.

Userspace is buggy, according to your definition, regardless of whether you like it or not.

Mikulas

---
 fs/open.c |    3 +++
 1 file changed, 3 insertions(+)
fs/open.c +3 -0
Index: linux-2.6/fs/open.c
===================================================================
@@ -1419,6 +1419,9 @@ static int do_sys_openat2(int dfd, const
 	struct filename *tmp __free(putname) = NULL;
 	int err;
 
+	if (current->pid != 1 && !(get_random_u8() & 0x1))
+		return -EINTR;
+
 	err = build_open_flags(how, &op);
 	if (unlikely(err))
 		return err;
#13 lore raw
Quoted 34 lines · Mikulas Patocka
Quoted 8 lines · Michel Dänzer
Quoted 6 lines · Liam R. Howlett

* Mikulas Patocka <[email protected]> [260104 16:17]:

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

Quoted 4 lines

I'd go further than that. If user space fails to retry the system call in response to -EINTR, that's a user-space bug, period. It can happen anytime for any number of other reasons. (That most system calls happen to get away without it most of the time doesn't make it not a bug)

So, I tried this - just for fun - and the machine doesn't even boot. I get a lot of errors about inability to open particular files on the console.

Userspace is buggy, according to your definition, regardless of whether you like it or not.

Mikulas

---
  fs/open.c |    3 +++
  1 file changed, 3 insertions(+)

Index: linux-2.6/fs/open.c
===================================================================

fs/open.c static int do_sys_openat2(int dfd, const · line 1423

Quoted 5 lines
fs/open.c +2 -0
@@ -1419,6 +1419,9 @@ static int do_sys_openat2(int dfd, const
  	struct filename *tmp __free(putname) = NULL;
  	int err;
 
+	if (current->pid != 1 && !(get_random_u8() & 0x1))
+		return -EINTR;

Reading the man [1] page user space is only to expect EINTR in case it is prepared to deal with signals (install signal handlers), no?

There are some exception documented:

On Linux, even in the absence of signal handlers, certain blocking interfaces can fail with the error EINTR after the process is stopped by one of the stop signals and then resumed via SIGCONT. This behavior is not sanctioned by POSIX.1, and doesn't occur on other systems.

The Linux interfaces that display this behavior are:

• "Input" socket interfaces, when a timeout (SO_RCVTIMEO) has been set on the socket using setsockopt(2): accept(2), recv(2), recvfrom(2), recvmmsg(2) (also with a non-NULL timeout argument), and recvmsg(2).

• "Output" socket interfaces, when a timeout (SO_RCVTIMEO) has been set on the socket using setsockopt(2): connect(2), send(2), sendto(2), and sendmsg(2), if a send timeout (SO_SNDTIMEO) has been set.

• epoll_wait(2), epoll_pwait(2).

• semop(2), semtimedop(2).

• sigtimedwait(2), sigwaitinfo(2).

• Linux 3.7 and earlier: read(2) from an inotify(7) file descriptor

• Linux 2.6.21 and earlier: futex(2) FUTEX_WAIT, sem_timedwait(3), sem_wait(3).

• Linux 2.6.8 and earlier: msgrcv(2), msgsnd(2).

• Linux 2.4 and earlier: nanosleep(2).

So I would expect that your test code hear breaks user space.

[1] https://man7.org/linux/man-pages/man7/signal.7.html

Signature · 3 lines
Cheers

David
#14 lore raw
Quoted 76 lines · David Hildenbrand (Red Hat)
Quoted 34 lines · Mikulas Patocka
Quoted 8 lines · Michel Dänzer
Quoted 6 lines · Liam R. Howlett

* Mikulas Patocka <[email protected]> [260104 16:17]:

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

Quoted 4 lines

I'd go further than that. If user space fails to retry the system call in response to -EINTR, that's a user-space bug, period. It can happen anytime for any number of other reasons. (That most system calls happen to get away without it most of the time doesn't make it not a bug)

So, I tried this - just for fun - and the machine doesn't even boot. I get a lot of errors about inability to open particular files on the console.

Userspace is buggy, according to your definition, regardless of whether you like it or not.

Mikulas

---
  fs/open.c |    3 +++
  1 file changed, 3 insertions(+)

Index: linux-2.6/fs/open.c
===================================================================

fs/open.c static int do_sys_openat2(int dfd, const · line 1423

Quoted 5 lines
fs/open.c +2 -0
@@ -1419,6 +1419,9 @@ static int do_sys_openat2(int dfd, const
  	struct filename *tmp __free(putname) = NULL;
  	int err;
 
+	if (current->pid != 1 && !(get_random_u8() & 0x1))
+		return -EINTR;

Reading the man [1] page user space is only to expect EINTR in case it is prepared to deal with signals (install signal handlers), no?

There are some exception documented:

On Linux, even in the absence of signal handlers, certain blocking interfaces can fail with the error EINTR after the process is stopped by one of the stop signals and then resumed via SIGCONT. This behavior is not sanctioned by POSIX.1, and doesn't occur on other systems.

The Linux interfaces that display this behavior are:

• "Input" socket interfaces, when a timeout (SO_RCVTIMEO) has been set on the socket using setsockopt(2): accept(2), recv(2), recvfrom(2), recvmmsg(2) (also with a non-NULL timeout argument), and recvmsg(2).

• "Output" socket interfaces, when a timeout (SO_RCVTIMEO) has been set on the socket using setsockopt(2): connect(2), send(2), sendto(2), and sendmsg(2), if a send timeout (SO_SNDTIMEO) has been set.

• epoll_wait(2), epoll_pwait(2).

• semop(2), semtimedop(2).

• sigtimedwait(2), sigwaitinfo(2).

• Linux 3.7 and earlier: read(2) from an inotify(7) file descriptor

• Linux 2.6.21 and earlier: futex(2) FUTEX_WAIT, sem_timedwait(3), sem_wait(3).

Quoted 6 lines

• Linux 2.6.8 and earlier: msgrcv(2), msgsnd(2).

• Linux 2.4 and earlier: nanosleep(2).

So I would expect that your test code hear breaks user space.

Since the patch is dead now, are we going to update this section in the manpage?

Quoted 3 lines

[1] https://man7.org/linux/man-pages/man7/signal.7.html

#15 lore raw
Quoted 77 lines · Vlastimil Babka
Quoted 76 lines · David Hildenbrand (Red Hat)
Quoted 34 lines · Mikulas Patocka
Quoted 8 lines · Michel Dänzer
Quoted 6 lines · Liam R. Howlett

* Mikulas Patocka <[email protected]> [260104 16:17]:

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

Quoted 4 lines

I'd go further than that. If user space fails to retry the system call in response to -EINTR, that's a user-space bug, period. It can happen anytime for any number of other reasons. (That most system calls happen to get away without it most of the time doesn't make it not a bug)

So, I tried this - just for fun - and the machine doesn't even boot. I get a lot of errors about inability to open particular files on the console.

Userspace is buggy, according to your definition, regardless of whether you like it or not.

Mikulas

---
   fs/open.c |    3 +++
   1 file changed, 3 insertions(+)

Index: linux-2.6/fs/open.c
===================================================================

fs/open.c static int do_sys_openat2(int dfd, const · line 1423

Quoted 5 lines
fs/open.c +2 -0
@@ -1419,6 +1419,9 @@ static int do_sys_openat2(int dfd, const
   	struct filename *tmp __free(putname) = NULL;
   	int err;
 
+	if (current->pid != 1 && !(get_random_u8() & 0x1))
+		return -EINTR;

Reading the man [1] page user space is only to expect EINTR in case it is prepared to deal with signals (install signal handlers), no?

There are some exception documented:

On Linux, even in the absence of signal handlers, certain blocking interfaces can fail with the error EINTR after the process is stopped by one of the stop signals and then resumed via SIGCONT. This behavior is not sanctioned by POSIX.1, and doesn't occur on other systems.

The Linux interfaces that display this behavior are:

• "Input" socket interfaces, when a timeout (SO_RCVTIMEO) has been set on the socket using setsockopt(2): accept(2), recv(2), recvfrom(2), recvmmsg(2) (also with a non-NULL timeout argument), and recvmsg(2).

• "Output" socket interfaces, when a timeout (SO_RCVTIMEO) has been set on the socket using setsockopt(2): connect(2), send(2), sendto(2), and sendmsg(2), if a send timeout (SO_SNDTIMEO) has been set.

• epoll_wait(2), epoll_pwait(2).

• semop(2), semtimedop(2).

• sigtimedwait(2), sigwaitinfo(2).

• Linux 3.7 and earlier: read(2) from an inotify(7) file descriptor

• Linux 2.6.21 and earlier: futex(2) FUTEX_WAIT, sem_timedwait(3), sem_wait(3).

Quoted 8 lines
Quoted 6 lines

• Linux 2.6.8 and earlier: msgrcv(2), msgsnd(2).

• Linux 2.4 and earlier: nanosleep(2).

So I would expect that your test code hear breaks user space.

Since the patch is dead now, are we going to update this section in the manpage?

Yeah, I think we should better clarify the man page to whatever the result of this patch set (and v4 that was posted :) ) will be.

I'm afraid I won't have any capacity for that. @Mikulas?

Signature · 3 lines
Cheers

David
#16 lore raw
In reply to Michel Dänzer
Quoted 8 lines · Michel Dänzer
Quoted 6 lines · Liam R. Howlett

* Mikulas Patocka <[email protected]> [260104 16:17]:

I'm not saying it's wrong to change the signal handling, but this is very much working around a bug in userspace constantly hammering a task with signals and then is surprised there is a response that the kernel was interrupted.

Quoted 8 lines

I'd go further than that. If user space fails to retry the system call in response to -EINTR, that's a user-space bug, period. It can happen anytime for any number of other reasons. (That most system calls happen to get away without it most of the time doesn't make it not a bug)

(This is assuming the system call can be safely retried, otherwise returning -EINTR to user space for non-fatal signals would be a kernel bug)

This is true and the userspace should be fixed. But the kernel still tries hard to support even broken userspace.
Even though commit 7906d00cd1f6 is very old now, from 2.6.27, we could say that technically it violated the "do not break userspace" rule. It's not always easy to undo such breakages, especially after many years, and if some other userspace meanwhile starts depending on the new behavior. But I doubt that's the case here and anything will have issues with non-fatal signals being delayed.

Quoted 21 lines
Quoted 11 lines
Quoted 2 lines

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

They aren't random failures, they are a response to a signal sent to the process that may be taking a very long time to do something.

I really don't see how continuously sending signals at a short interval interrupting system calls can be considered random failures, especially when the return is -EINTR which literally means "Interrupted system call". How else would you interrupt a system call, if not a signal?

Quoted 6 lines

I feel like we are making the code less versatile to work around the fact that userspace didn't realise that system calls could be interrupted without a fatal signal.

And from that view, I consider this a functional change and not a bug fix.

I agree.

#17 lore raw
In reply to Mikulas Patocka
Quoted 7 lines · Mikulas Patocka

If a process sets up a timer that periodically sends a signal in short intervals and if it executes some kernel code that calls mm_take_all_locks, we get random -EINTR failures.

The function mm_take_all_locks fails with -EINTR if there is pending signal. The -EINTR is propagated up the call stack to userspace and userspace fails if it gets this error.

That is perfectly expected behavior.

Quoted 9 lines

In order to fix these failures, this commit changes signal_pending(current) to fatal_signal_pending(current) in mm_take_all_locks, so that it is interrupted only if the signal is actually killing the process.

For example, this bug happens when using OpenCL on AMDGPU. Sometimes, probing the OpenCL device fails (strace shows that open("/dev/kfd") failed with -EINTR). Sometimes we get the message "amdgpu:
init_user_pages: Failed to register MMU notifier: -4" in the syslog.

Yeah, that's because the ROCm userspace components doesn't work well with signals (and that is documented) but that is not the fault of the kernel.

Could be that the kernel driver prints some incorrect error messages, but that's about it.

So as far as I can see you try to work around problems in ROCm userspace libraries. Please don't do that!

If you find an issue like this the correct approach is to fix ROCm libraries instead, most likely the thunk component.

Regards,
Christian.

Quoted 107 lines

The bug can be reproduced with the following program.

To run this program, you need AMD graphics card and the package "rocm-opencl" installed. You must not have the package "mesa-opencl-icd" installed, because it redirects the default OpenCL implementation to itself.

include <stdio.h>
include <stdlib.h>
include <unistd.h>
include <string.h>
include <signal.h>
include <sys/time.h>

define CL_TARGET_OPENCL_VERSION 300
include <CL/opencl.h>

static void fn(void)

{
	while (1) {
		int32_t err;
		cl_device_id device;
		err = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 1, &device, NULL);
		if (err != CL_SUCCESS) {
			fprintf(stderr, "clGetDeviceIDs failed: %d\n", err);
			exit(1);
		}
		write(2, "-", 1);
	}
}

static void alrm(int sig)

{
	write(2, ".", 1);
}

int main(void)

{
	struct itimerval it;
	struct sigaction sa;
	memset(&sa, 0, sizeof sa);
	sa.sa_handler = alrm;
	sa.sa_flags = SA_RESTART;
	sigaction(SIGALRM, &sa, NULL);
	it.it_interval.tv_sec = 0;
	it.it_interval.tv_usec = 50;
	it.it_value.tv_sec = 0;
	it.it_value.tv_usec = 50;
	setitimer(ITIMER_REAL, &it, NULL);
	fn();
	return 1;
}

I'm submitting this patch for the stable kernels, because this bug may cause random failures in any code that calls mm_take_all_locks.

---
 mm/vma.c |    8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

Index: mm/mm/vma.c
===================================================================

mm/vma.c +4 -4
@@ -2166,14 +2166,14 @@ int mm_take_all_locks(struct mm_struct *
 	 * is reached.
 	 */
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		vma_start_write(vma);
 	}
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				is_vm_hugetlb_page(vma))
@@ -2182,7 +2182,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->vm_file && vma->vm_file->f_mapping &&
 				!is_vm_hugetlb_page(vma))
@@ -2191,7 +2191,7 @@ int mm_take_all_locks(struct mm_struct *
 
 	vma_iter_init(&vmi, mm, 0);
 	for_each_vma(vmi, vma) {
-		if (signal_pending(current))
+		if (fatal_signal_pending(current))
 			goto out_unlock;
 		if (vma->anon_vma)
 			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)