Skip to contentSkip to conversation

mm: sysctl: add panic_on_mm_error sysctl

PATCH In review Grzegorz Halat 6 participants 8 messages Jan 27, 2020

Subsystems (6)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 mm: sysctl: add panic_on_mm_error sysctl

Memory management subsystem performs various checks at runtime, if an inconsistency is detected then such event is being logged and kernel continues to run. While debugging such problems it is helpful to collect memory dump as early as possible. Currently, there is no easy way to panic kernel when such error is detected.

It was proposed[1] to panic the kernel if panic_on_oops is set but this approach was not accepted. One of alternative proposals was introduction of a new sysctl.

The patch adds panic_on_mm_error sysctl. If the sysctl is set then the kernel will be crashed when an inconsistency is detected by memory management. This currently means panic when bad page or bad PTE is detected(this may be extended to other places in MM).

Another use case of this sysctl may be in security-wise environments, it may be more desired to crash machine than continue to run with potentially damaged data structures.

[1] https://marc.info/?l=linux-mm&m=142649500728327&w=2

---
 Documentation/admin-guide/sysctl/kernel.rst | 12 ++++++++++++
 include/linux/kernel.h                      |  1 +
 kernel/sysctl.c                             |  9 +++++++++
 mm/memory.c                                 |  7 +++++++
 mm/page_alloc.c                             |  4 +++-
 5 files changed, 32 insertions(+), 1 deletion(-)
Documentation/admin-guide/sysctl/kernel.rst +12 -0
index def074807cee..2fecd6b2547e 100644
@@ -61,6 +61,7 @@ show up in /proc/sys/kernel:
 - overflowgid
 - overflowuid
 - panic
+- panic_on_mm_error
 - panic_on_oops
 - panic_on_stackoverflow
 - panic_on_unrecovered_nmi
@@ -611,6 +612,17 @@ an IO error.
    and you can use this option to take a crash dump.
 
 
+panic_on_mm_error:
+==============
+
+Controls the kernel's behaviour when inconsistency is detected
+by memory management code, for example bad page state or bad PTE.
+
+0: try to continue operation.
+
+1: panic immediately.
+
+
 panic_on_oops:
 ==============
 
include/linux/kernel.h +1 -0
index 0d9db2a14f44..5f9d408512ff 100644
@@ -518,6 +518,7 @@ extern int oops_in_progress;		/* If set, an oops, panic(), BUG() or die() is in
 extern int panic_timeout;
 extern unsigned long panic_print;
 extern int panic_on_oops;
+extern int panic_on_mm_error;
 extern int panic_on_unrecovered_nmi;
 extern int panic_on_io_nmi;
 extern int panic_on_warn;
kernel/sysctl.c +9 -0
index 70665934d53e..6477e1cce28b 100644
@@ -1238,6 +1238,15 @@ static struct ctl_table kern_table[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_ONE,
 	},
+	{
+		.procname	= "panic_on_mm_error",
+		.data		= &panic_on_mm_error,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
 	{
 		.procname	= "timer_migration",
mm/memory.c +7 -0
index 45442d9a4f52..cce74ff39447 100644
@@ -71,6 +71,7 @@
 #include <linux/dax.h>
 #include <linux/oom.h>
 #include <linux/numa.h>
+#include <linux/module.h>
 
 #include <trace/events/kmem.h>
 
@@ -88,6 +89,8 @@
 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
 #endif
 
+int panic_on_mm_error __read_mostly;
+
 #ifndef CONFIG_NEED_MULTIPLE_NODES
 /* use the per-pgdat data instead for discontigmem - mbligh */
 unsigned long max_mapnr;
@@ -543,6 +546,10 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
 		 vma->vm_ops ? vma->vm_ops->fault : NULL,
 		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
 		 mapping ? mapping->a_ops->readpage : NULL);
+
+	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page map detected");
 	dump_stack();
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 }
mm/page_alloc.c +3 -1
index d047bf7d8fd4..2ea6a65ba011 100644
@@ -643,9 +643,11 @@ static void bad_page(struct page *page, const char *reason,
 	if (bad_flags)
 		pr_alert("bad because of flags: %#lx(%pGp)\n",
 						bad_flags, &bad_flags);
-	dump_page_owner(page);
 
+	dump_page_owner(page);
 	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page state detected");
 	dump_stack();
 out:
 	/* Leave bad fields for debug, except PageBuddy could make trouble */
Signature · 1 line
2.21.1
Reviewed-by on v1 1/1
#1 lore raw
Quoted 144 lines · Grzegorz Halat

Memory management subsystem performs various checks at runtime, if an inconsistency is detected then such event is being logged and kernel continues to run. While debugging such problems it is helpful to collect memory dump as early as possible. Currently, there is no easy way to panic kernel when such error is detected.

It was proposed[1] to panic the kernel if panic_on_oops is set but this approach was not accepted. One of alternative proposals was introduction of a new sysctl.

The patch adds panic_on_mm_error sysctl. If the sysctl is set then the kernel will be crashed when an inconsistency is detected by memory management. This currently means panic when bad page or bad PTE is detected(this may be extended to other places in MM).

Another use case of this sysctl may be in security-wise environments, it may be more desired to crash machine than continue to run with potentially damaged data structures.

[1] https://marc.info/?l=linux-mm&m=142649500728327&w=2

---
 Documentation/admin-guide/sysctl/kernel.rst | 12 ++++++++++++
 include/linux/kernel.h                      |  1 +
 kernel/sysctl.c                             |  9 +++++++++
 mm/memory.c                                 |  7 +++++++
 mm/page_alloc.c                             |  4 +++-
 5 files changed, 32 insertions(+), 1 deletion(-)
Documentation/admin-guide/sysctl/kernel.rst +12 -0
index def074807cee..2fecd6b2547e 100644
@@ -61,6 +61,7 @@ show up in /proc/sys/kernel:
 - overflowgid
 - overflowuid
 - panic
+- panic_on_mm_error
 - panic_on_oops
 - panic_on_stackoverflow
 - panic_on_unrecovered_nmi
@@ -611,6 +612,17 @@ an IO error.
    and you can use this option to take a crash dump.
 
 
+panic_on_mm_error:
+==============
+
+Controls the kernel's behaviour when inconsistency is detected
+by memory management code, for example bad page state or bad PTE.
+
+0: try to continue operation.
+
+1: panic immediately.
+
+
 panic_on_oops:
 ==============
 
include/linux/kernel.h +1 -0
index 0d9db2a14f44..5f9d408512ff 100644
@@ -518,6 +518,7 @@ extern int oops_in_progress;		/* If set, an oops, panic(), BUG() or die() is in
 extern int panic_timeout;
 extern unsigned long panic_print;
 extern int panic_on_oops;
+extern int panic_on_mm_error;
 extern int panic_on_unrecovered_nmi;
 extern int panic_on_io_nmi;
 extern int panic_on_warn;
kernel/sysctl.c +9 -0
index 70665934d53e..6477e1cce28b 100644
@@ -1238,6 +1238,15 @@ static struct ctl_table kern_table[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_ONE,
 	},
+	{
+		.procname	= "panic_on_mm_error",
+		.data		= &panic_on_mm_error,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
 	{
 		.procname	= "timer_migration",
mm/memory.c +7 -0
index 45442d9a4f52..cce74ff39447 100644
@@ -71,6 +71,7 @@
 #include <linux/dax.h>
 #include <linux/oom.h>
 #include <linux/numa.h>
+#include <linux/module.h>
 
 #include <trace/events/kmem.h>
 
@@ -88,6 +89,8 @@
 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
 #endif
 
+int panic_on_mm_error __read_mostly;
+
 #ifndef CONFIG_NEED_MULTIPLE_NODES
 /* use the per-pgdat data instead for discontigmem - mbligh */
 unsigned long max_mapnr;
@@ -543,6 +546,10 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
 		 vma->vm_ops ? vma->vm_ops->fault : NULL,
 		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
 		 mapping ? mapping->a_ops->readpage : NULL);
+
+	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page map detected");
 	dump_stack();
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 }
mm/page_alloc.c +3 -1
index d047bf7d8fd4..2ea6a65ba011 100644
@@ -643,9 +643,11 @@ static void bad_page(struct page *page, const char *reason,
 	if (bad_flags)
 		pr_alert("bad because of flags: %#lx(%pGp)\n",
 						bad_flags, &bad_flags);
-	dump_page_owner(page);
 
+	dump_page_owner(page);
 	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page state detected");
 	dump_stack();
 out:
 	/* Leave bad fields for debug, except PageBuddy could make trouble */
Signature · 1 line
2.21.1
Signature · 3 lines
  Best regards,
    Oleksandr Natalenko (post-factum)
    Senior Software Maintenance Engineer
Reviewed-by on v1 1/1
#2 lore raw
In reply to Grzegorz Halat
Quoted 131 lines · Grzegorz Halat

Memory management subsystem performs various checks at runtime, if an inconsistency is detected then such event is being logged and kernel continues to run. While debugging such problems it is helpful to collect memory dump as early as possible. Currently, there is no easy way to panic kernel when such error is detected.

It was proposed[1] to panic the kernel if panic_on_oops is set but this approach was not accepted. One of alternative proposals was introduction of a new sysctl.

The patch adds panic_on_mm_error sysctl. If the sysctl is set then the kernel will be crashed when an inconsistency is detected by memory management. This currently means panic when bad page or bad PTE is detected(this may be extended to other places in MM).

Another use case of this sysctl may be in security-wise environments, it may be more desired to crash machine than continue to run with potentially damaged data structures.

[1] https://marc.info/?l=linux-mm&m=142649500728327&w=2

---
 Documentation/admin-guide/sysctl/kernel.rst | 12 ++++++++++++
 include/linux/kernel.h                      |  1 +
 kernel/sysctl.c                             |  9 +++++++++
 mm/memory.c                                 |  7 +++++++
 mm/page_alloc.c                             |  4 +++-
 5 files changed, 32 insertions(+), 1 deletion(-)
Documentation/admin-guide/sysctl/kernel.rst +12 -0
index def074807cee..2fecd6b2547e 100644
@@ -61,6 +61,7 @@ show up in /proc/sys/kernel:
 - overflowgid
 - overflowuid
 - panic
+- panic_on_mm_error
 - panic_on_oops
 - panic_on_stackoverflow
 - panic_on_unrecovered_nmi
@@ -611,6 +612,17 @@ an IO error.
    and you can use this option to take a crash dump.
 
 
+panic_on_mm_error:
+==============
+
+Controls the kernel's behaviour when inconsistency is detected
+by memory management code, for example bad page state or bad PTE.
+
+0: try to continue operation.
+
+1: panic immediately.
+
+
 panic_on_oops:
 ==============
 
include/linux/kernel.h +1 -0
index 0d9db2a14f44..5f9d408512ff 100644
@@ -518,6 +518,7 @@ extern int oops_in_progress;		/* If set, an oops, panic(), BUG() or die() is in
 extern int panic_timeout;
 extern unsigned long panic_print;
 extern int panic_on_oops;
+extern int panic_on_mm_error;
 extern int panic_on_unrecovered_nmi;
 extern int panic_on_io_nmi;
 extern int panic_on_warn;
kernel/sysctl.c +9 -0
index 70665934d53e..6477e1cce28b 100644
@@ -1238,6 +1238,15 @@ static struct ctl_table kern_table[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_ONE,
 	},
+	{
+		.procname	= "panic_on_mm_error",
+		.data		= &panic_on_mm_error,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
 	{
 		.procname	= "timer_migration",
mm/memory.c +7 -0
index 45442d9a4f52..cce74ff39447 100644
@@ -71,6 +71,7 @@
 #include <linux/dax.h>
 #include <linux/oom.h>
 #include <linux/numa.h>
+#include <linux/module.h>
 
 #include <trace/events/kmem.h>
 
@@ -88,6 +89,8 @@
 #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
 #endif
 
+int panic_on_mm_error __read_mostly;
+
 #ifndef CONFIG_NEED_MULTIPLE_NODES
 /* use the per-pgdat data instead for discontigmem - mbligh */
 unsigned long max_mapnr;
@@ -543,6 +546,10 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
 		 vma->vm_ops ? vma->vm_ops->fault : NULL,
 		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
 		 mapping ? mapping->a_ops->readpage : NULL);
+
+	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page map detected");
 	dump_stack();
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 }
mm/page_alloc.c +0 -0
index d047bf7d8fd4..2ea6a65ba011 100644
@@ -643,9 +643,11 @@ static void bad_page(struct page *page, const char *reason,
 	if (bad_flags)
 		pr_alert("bad because of flags: %#lx(%pGp)\n",

mm/page_alloc.c static void bad_page(struct page *page, const char *reason, · line 653

Quoted 10 lines
mm/page_alloc.c +3 -1
 						bad_flags, &bad_flags);
-	dump_page_owner(page);
 
+	dump_page_owner(page);
 	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page state detected");
 	dump_stack();
 out:
 	/* Leave bad fields for debug, except PageBuddy could make trouble */
Signature · 1 line
Aaron Tomlin
#3 lore raw
In reply to Grzegorz Halat

Documentation/admin-guide/sysctl/kernel.rst show up in /proc/sys/kernel: · line 64

Quoted 9 lines · Grzegorz Halat
Documentation/admin-guide/sysctl/kernel.rst +1 -0
index def074807cee..2fecd6b2547e 100644
@@ -61,6 +61,7 @@ show up in /proc/sys/kernel:
 - overflowgid
 - overflowuid
 - panic
+- panic_on_mm_error

Maybe panic_on_inconsistent_mm is better, for an MM error sounds too generic (e.g. is page allocation failure an error, is OOM killer an error, is NULL pointer dereference an error, is use-after-free access an error) ?

Also, should this be in /proc/sys/vm/ than /proc/sys/kernel/ ?

Quoted 3 lines
Documentation/admin-guide/sysctl/kernel.rst +0 -0
 - panic_on_oops
 - panic_on_stackoverflow
 - panic_on_unrecovered_nmi
#4 lore raw
Quoted 14 lines · Tetsuo Handa

Documentation/admin-guide/sysctl/kernel.rst show up in /proc/sys/kernel: · line 64

Quoted 9 lines · Grzegorz Halat
Documentation/admin-guide/sysctl/kernel.rst +1 -0
index def074807cee..2fecd6b2547e 100644
@@ -61,6 +61,7 @@ show up in /proc/sys/kernel:
 - overflowgid
 - overflowuid
 - panic
+- panic_on_mm_error

Maybe panic_on_inconsistent_mm is better, for an MM error sounds too generic (e.g. is page allocation failure an error, is OOM killer an error, is NULL pointer dereference an error, is use-after-free access an error) ?

Fair enough; 'panic_on_inconsistent_mm' is more concise.

Signature · 1 line
Aaron Tomlin
#5 lore raw
In reply to Tetsuo Handa
Quoted 5 lines · Tetsuo Handa

Maybe panic_on_inconsistent_mm is better, for an MM error sounds too generic (e.g. is page allocation failure an error, is OOM killer an error, is NULL pointer dereference an error, is use-after-free access an error) ?

yes, panic_on_inconsistent_mm is better

Quoted 1 line

Also, should this be in /proc/sys/vm/ than /proc/sys/kernel/ ?

Agreed

I will wait a day or two for more feedback and send V2 with sysctl named as 'vm.panic_on_inconsistent_mm'.

Thanks,
--
Grzegorz Halat

#6 lore raw
In reply to Grzegorz Halat
Quoted 14 lines · Grzegorz Halat

Memory management subsystem performs various checks at runtime, if an inconsistency is detected then such event is being logged and kernel continues to run. While debugging such problems it is helpful to collect memory dump as early as possible. Currently, there is no easy way to panic kernel when such error is detected.

It was proposed[1] to panic the kernel if panic_on_oops is set but this approach was not accepted. One of alternative proposals was introduction of a new sysctl.

Quoted 8 lines

The patch adds panic_on_mm_error sysctl. If the sysctl is set then the kernel will be crashed when an inconsistency is detected by memory management. This currently means panic when bad page or bad PTE is detected(this may be extended to other places in MM).

Another use case of this sysctl may be in security-wise environments, it may be more desired to crash machine than continue to run with potentially damaged data structures.

Well, on the other hand, this will allow a normal user to more easily crash the system due to a recoverable bug which could result in local DoS.

#7 lore raw
In reply to Grzegorz Halat
Quoted 114 lines · Grzegorz Halat

Memory management subsystem performs various checks at runtime, if an inconsistency is detected then such event is being logged and kernel continues to run. While debugging such problems it is helpful to collect memory dump as early as possible. Currently, there is no easy way to panic kernel when such error is detected.

It was proposed[1] to panic the kernel if panic_on_oops is set but this approach was not accepted. One of alternative proposals was introduction of a new sysctl.

The patch adds panic_on_mm_error sysctl. If the sysctl is set then the kernel will be crashed when an inconsistency is detected by memory management. This currently means panic when bad page or bad PTE is detected(this may be extended to other places in MM).

Another use case of this sysctl may be in security-wise environments, it may be more desired to crash machine than continue to run with potentially damaged data structures.

[1] https://marc.info/?l=linux-mm&m=142649500728327&w=2

---
  Documentation/admin-guide/sysctl/kernel.rst | 12 ++++++++++++
  include/linux/kernel.h                      |  1 +
  kernel/sysctl.c                             |  9 +++++++++
  mm/memory.c                                 |  7 +++++++
  mm/page_alloc.c                             |  4 +++-
  5 files changed, 32 insertions(+), 1 deletion(-)
Documentation/admin-guide/sysctl/kernel.rst +12 -0
index def074807cee..2fecd6b2547e 100644
@@ -61,6 +61,7 @@ show up in /proc/sys/kernel:
  - overflowgid
  - overflowuid
  - panic
+- panic_on_mm_error
  - panic_on_oops
  - panic_on_stackoverflow
  - panic_on_unrecovered_nmi
@@ -611,6 +612,17 @@ an IO error.
     and you can use this option to take a crash dump.
 
 
+panic_on_mm_error:
+==============
+
+Controls the kernel's behaviour when inconsistency is detected
+by memory management code, for example bad page state or bad PTE.
+
+0: try to continue operation.
+
+1: panic immediately.
+
+
  panic_on_oops:
  ==============
 
include/linux/kernel.h +1 -0
index 0d9db2a14f44..5f9d408512ff 100644
@@ -518,6 +518,7 @@ extern int oops_in_progress;		/* If set, an oops, panic(), BUG() or die() is in
  extern int panic_timeout;
  extern unsigned long panic_print;
  extern int panic_on_oops;
+extern int panic_on_mm_error;
  extern int panic_on_unrecovered_nmi;
  extern int panic_on_io_nmi;
  extern int panic_on_warn;
kernel/sysctl.c +9 -0
index 70665934d53e..6477e1cce28b 100644
@@ -1238,6 +1238,15 @@ static struct ctl_table kern_table[] = {
  		.extra1		= SYSCTL_ZERO,
  		.extra2		= SYSCTL_ONE,
  	},
+	{
+		.procname	= "panic_on_mm_error",
+		.data		= &panic_on_mm_error,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= SYSCTL_ONE,
+	},
  #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
  	{
  		.procname	= "timer_migration",
mm/memory.c +3 -0
index 45442d9a4f52..cce74ff39447 100644
@@ -71,6 +71,7 @@
  #include <linux/dax.h>
  #include <linux/oom.h>
  #include <linux/numa.h>
+#include <linux/module.h>
 
  #include <trace/events/kmem.h>
 
@@ -88,6 +89,8 @@
  #warning Unfortunate NUMA and NUMA Balancing config, growing page-frame for last_cpupid.
  #endif
 
+int panic_on_mm_error __read_mostly;
+
  #ifndef CONFIG_NEED_MULTIPLE_NODES
  /* use the per-pgdat data instead for discontigmem - mbligh */
  unsigned long max_mapnr;

mm/memory.c static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr, · line 550

Quoted 5 lines
mm/memory.c +2 -0
@@ -543,6 +546,10 @@ static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
  		 vma->vm_ops ? vma->vm_ops->fault : NULL,
  		 vma->vm_file ? vma->vm_file->f_op->mmap : NULL,
  		 mapping ? mapping->a_ops->readpage : NULL);
+
+	print_modules();

I'm not sure that printing all modules for each bad pte is a good idea. It already prints 'readpage' which is enough to guess filesystem or driver.

Maybe print modules under if (panic...) below?

Quoted 23 lines
mm/memory.c +2 -0
+	if (panic_on_mm_error)
+		panic("Bad page map detected");
  	dump_stack();
  	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  }
mm/page_alloc.c +3 -1
index d047bf7d8fd4..2ea6a65ba011 100644
@@ -643,9 +643,11 @@ static void bad_page(struct page *page, const char *reason,
  	if (bad_flags)
  		pr_alert("bad because of flags: %#lx(%pGp)\n",
  						bad_flags, &bad_flags);
-	dump_page_owner(page);
 
+	dump_page_owner(page);
  	print_modules();
+	if (panic_on_mm_error)
+		panic("Bad page state detected");
  	dump_stack();
  out:
  	/* Leave bad fields for debug, except PageBuddy could make trouble */