User space out of memory approach
Subsystems (3)
Hi all,
We have done a comparison between the kernel version and user space version and apparently the behavior is similar. You can also get this patch and module to test it and compare with kernel OOM Killer. Here goes a patch and a module that moves the kernel space OOM Killer algorithm to user space. Let us know about your ideas.
*******************
PATCH
*******************
fs/proc/array.c +10 -0
@@ -470,3 +470,13 @@return sprintf (buffer,"%d %d %d %d %d %d %d\n" , size, resident, shared, text, lib, data,0 ); } + +int proc_pid_oom (struct task_struct *task,char * buffer) +{ +int res; + res =sprintf (buffer,"%d %lu %lu\n" , + task->pid, + task->utime, + task->stime); +return res; +}
fs/proc/base.c +5 -0
@@ -60,6 +60,7 @@PROC_TGID_MAPS ,PROC_TGID_MOUNTS ,PROC_TGID_WCHAN , +PROC_TGID_OOM , #ifdef CONFIG_SCHEDSTATS PROC_TGID_SCHEDSTAT, #endif
@@ -86,6 +87,7 @@PROC_TID_MAPS ,PROC_TID_MOUNTS ,PROC_TID_WCHAN , +PROC_TID_OOM , #ifdef CONFIG_SCHEDSTATS PROC_TID_SCHEDSTAT, #endif
@@ -132,6 +134,7 @@ #ifdef CONFIG_SCHEDSTATS E(PROC_TGID_SCHEDSTAT, "schedstat", S_IFREG|S_IRUGO), #endif + E(PROC_TGID_OOM, "oom", S_IFREG|S_IRUGO), {0,0,NULL,0} }; static struct pid_entry tid_base_stuff[] = {
@@ -157,6 +160,7 @@#ifdef CONFIG_SCHEDSTATS E (PROC_TID_SCHEDSTAT ,"schedstat" ,S_IFREG |S_IRUGO ), #endif +E (PROC_TID_OOM ,"oom" ,S_IFREG |S_IRUGO ), {0,0,NULL,0} };
@@ -193,6 +197,7 @@int proc_tgid_stat (struct task_struct *,char *);int proc_pid_status (struct task_struct *,char *);int proc_pid_statm (struct task_struct *,char *); +int proc_pid_oom (struct task_struct *,char *); static int proc_fd_link(struct inode *inode, struct dentry **dentry,
struct vfsmount **mnt)
{
@@ -1377,6 +1382,11 @@
ei->op.proc_read = proc_pid_schedstat; break;
#endif
+ case PROC_TID_OOM:
+ case PROC_TGID_OOM:
+ inode->i_fop = &proc_info_file_operations;
+ ei->op.proc_read = proc_pid_oom;
+ break;
default:
printk("procfs: impossible type (%d)",p->type); iput(inode);
diff -urN linux-2.6.10/include/linux/oom_kill.h linux-2.6.10-oom/include/linux/oom_kill.h
--- linux-2.6.10/include/linux/oom_kill.h 1969-12-31 20:00:00.000000000 -0400
+++ linux-2.6.10-oom/include/linux/oom_kill.h 2005-01-10
15:42:26.000000000 -0400
@@ -0,0 +1,6 @@
+struct candidate_process {
+ pid_t pid;
+ struct list_head pid_list;
+};
+
+struct list_head *loop_counter;
mm/#oom_kill.c# +0 -366
366 lines not on this page · raw message
mm/oom_kill.c +69 -3
108 lines not on this page · raw message
******************
Module oom.c
******************
/* 2005 * Bruna Moreira <[email protected]> * Edjard Mota <[email protected]> * Ilias Biris <[email protected]> * Mauricio Lin <[email protected]> * * Embedded Linux Lab - 10LE Institulo Nokia de Tecnologia - INdT * * Create a /proc/oom that allows user space Out of Memory Killer * to write the list of pids through it to the kernel. When out of * memory happens the kernel selects the process to be killed from * /proc/oom.
*
* Approach suggested by Tony Lindgren <[email protected]> */
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/proc_fs.h>
#include <linux/sched.h>
#include <linux/oom_kill.h>
#include <asm/uaccess.h>
#define MODULE_NAME "oom"
extern struct candidate_process *candidate;
static unsigned int nr_pids;
static struct proc_dir_entry *oom_file;
extern struct list_head pidqueue_head;
static DECLARE_MUTEX(user_oom_lock);
static unsigned long size_of_bytes;
static inline void add_to_pidqueue(struct candidate_process * p)
{
list_add_tail(&p->pid_list, &pidqueue_head);
nr_pids++;
//printk(KERN_DEBUG "add nr_pids = %d (%d) \n", nr_pids, p->pid);
}static inline void del_from_pidqueue(struct candidate_process * p)
{
nr_pids--;
//printk(KERN_DEBUG "delete nr_pids = %d \n", nr_pids);
list_del(&p->pid_list);
}static inline void del_all_from_pidqueue(void)
{
if (!list_empty(&pidqueue_head)) {
struct list_head *tmp;
list_for_each_safe(loop_counter, tmp, &pidqueue_head) {
candidate = list_entry(loop_counter, struct candidate_process, pid_list);
del_from_pidqueue(candidate);
kfree(candidate);
}
}
}static int proc_read_oom(char *page, char **start,
off_t off, int count, int *eof, void *data)
{
int len = 0;
char *output;
char *item;
if (!list_empty(&pidqueue_head)) {
item = kmalloc(6, GFP_KERNEL);
output = kmalloc(size_of_bytes, GFP_KERNEL);
strcpy(output, "");
list_for_each(loop_counter, &pidqueue_head) {
candidate = list_entry(loop_counter, struct candidate_process, pid_list);
sprintf(item, "%d ", candidate->pid);
strcat(output, item);
}
strcat(output, "\n");
len = sprintf(page, output);
kfree(output);
kfree(item);
}
else {
len = sprintf(page, "\n");
}
return len;
}static int proc_write_oom(struct file *file,
const char *buffer,
unsigned long count,
void *data)
{
unsigned long len;
char *input, *item;
len = count;
input = kmalloc(count+1, GFP_KERNEL);
if (!input)
return -ENOMEM;
if(copy_from_user(input, buffer, count)) {
kfree(input);
return -EFAULT;
}
input[count] = ' ';
len = strlen(input);
int prev_index = 0;
int i;
if (down_interruptible(&user_oom_lock)) {
kfree(input);
return -EINTR;
}
del_all_from_pidqueue();
for (i=0; i<len; i++) {
if (input[i] == ' ') {
input[i] = '\0';
item = kmalloc(i-prev_index, GFP_KERNEL);
strcpy(item, &input[prev_index]);
prev_index = i+1;
candidate = (struct candidate_process *)kmalloc(sizeof(struct
candidate_process), GFP_KERNEL);
candidate->pid = simple_strtoul(item, NULL, 10);
add_to_pidqueue(candidate);
kfree(item);
}
}
size_of_bytes = len;
up(&user_oom_lock);
kfree(input);
return count;
}static int __init init_oom(void)
{
int flag = 0;
oom_file = create_proc_entry("oom", S_IRUGO | S_IWUSR, NULL);
oom_file->read_proc = proc_read_oom;
oom_file->write_proc = proc_write_oom;
oom_file->owner = THIS_MODULE;
nr_pids = 0;
size_of_bytes = 0;
printk(KERN_DEBUG "%s included\n", MODULE_NAME);
return flag;
}static void __exit cleanup_oom(void)
{
del_all_from_pidqueue();
remove_proc_entry("oom", NULL);
printk(KERN_DEBUG "%s removed\n", MODULE_NAME);
}module_init(init_oom);module_exit(cleanup_oom);MODULE_LICENSE("GPL");
***********************Makefile for module
***********************
obj-m := oom.o
MAKE := /usr/bin/make
KDIR := $(HOME)/linux-2.6.10-oom
PWD := $(PWD)
default:
$(MAKE) -C $(KDIR) SUBDIRS=$(PWD) modules
************************
README
************************
1. Getting the kernel
Create a directory called "download" in your home and go to this directory
# mkdir download # cd download
Download the kernel 2.6.9 from ftp://ftp.kernel.org/pub/linux/kernel/v2.6/linux-2.6.10.tar.bz2 to the directory you have created previuosly.
Extract it in your home as below
# tar xjvf download/linux-2.6.10.tar.bz2
Rename the kernel directory to linux-2.6.10-oom
# mv linux-2.6.10 linux-2.6.10-oom
Go to the kernel source tree and patch the kernel using "2.6.9.oom.patch"
# cd linux-2.6.10-oom # patch -p1 < ../2.6.10.oom.patch
The command above assumes that you have put the patch file in your home directory.
Configure you kernel using "make menuconfig" or "make xconfig"
2. Compiling the kernel module
After compiling your kernel you have to compile the kernel module "oom.c"
Put the Makefile and oom.c in the same directory.
Type make in order to create the "oom.ko" as below
# make
After typing make, check if it was created the "oom.ko" in your current directory.
3. Run the oom module
You have to insert the module now as root:
# insmod oom.ko
Check if the /proc/oom was created in your system.
# cat /proc/oom # # echo 11 22 33 > /proc/oom # cat /proc/oom # 11 22 33
4. User OOM (Optional)
After that you can compile the user_oom.c as below (you can compile it as normal user):
# gcc user_oom.c -o user_oom -lm
As root you have to run it:
# ./user_oom
************************
TODO
************************
Reorganize the kernel and user space code (remove some superfluous stuff or move them to a more viable file) in order to reduce the stack space or make the code more readable and clear.
Change some static allocation to dynamic allocation as array to linked list.
Create a /dev/oom to get the memory info instead of reading it from /proc/meminfo continuously.
Suggestions are welcome. The user space OOM application (optional) will be sent later.
BR,
Mauricio Lin.
Quoted 7 lines · Mauricio Lin
Hi all,
We have done a comparison between the kernel version and user space version and apparently the behavior is similar. You can also get this patch and module to test it and compare with kernel OOM Killer. Here goes a patch and a module that moves the kernel space OOM Killer algorithm to user space. Let us know about your ideas.
No comments on the code itself - It is interesting to have certain pids "not selectable" by the OOM killer. Patches which have similar funcionality have been floating around.
The userspace OOM killer is dangerous though. You have to guarantee that allocations will NOT happen until the OOM killer is executed and the killed process is dead and has its pages freed - allocations under OOM can cause deadlocks.
"OOM-killer-in-userspace" is unreliable, not sure if its worth the effort making it reliable (mlock it, flagged as PF_MEMALLOC, etc).
Hi,
I guess it the idea was not fully and well explained. It is not the OOM Killer itself that was moved to user space but rather its ranking algorithm. Ranking is not an specific functionality of kernel space. Kernel only need to know which process whould be killed.
In that sense the approach is different and might be worth testing, mainly for cases where we want to allow better policies of ranking. For example, an embedded device with few resources and important different running applications:
whic one is the best? To my understanding the current ranking policy does not necessarily chooses the best one to be killed.
br
Edjard
Quoted 24 lines · Marcelo Tosatti
Quoted 7 lines · Mauricio Lin
Hi all,
We have done a comparison between the kernel version and user space version and apparently the behavior is similar. You can also get this patch and module to test it and compare with kernel OOM Killer. Here goes a patch and a module that moves the kernel space OOM Killer algorithm to user space. Let us know about your ideas.
No comments on the code itself - It is interesting to have certain pids "not selectable" by the OOM killer. Patches which have similar funcionality have been floating around.
The userspace OOM killer is dangerous though. You have to guarantee that allocations will NOT happen until the OOM killer is executed and the killed process is dead and has its pages freed - allocations under OOM can cause deadlocks.
"OOM-killer-in-userspace" is unreliable, not sure if its worth the effort making it reliable (mlock it, flagged as
PF_MEMALLOC, etc).
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 7 lines · Edjard Souza Mota
Hi,
I guess it the idea was not fully and well explained. It is not the OOM Killer itself that was moved to user space but rather its ranking algorithm. Ranking is not an specific functionality of kernel space. Kernel only need to know which process whould be killed.
Quoted 5 lines
In that sense the approach is different and might be worth testing, mainly for cases where we want to allow better policies of ranking. For example, an embedded device with few resources and important different running applications:
whic one is the best? To my understanding the current ranking policy does not necessarily chooses the best one to be killed.
Sorry, I misunderstood. Should have read the code before shouting.
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
Few comments about the code:
retry:
- p = select_bad_process(); + printk(KERN_DEBUG "A good walker leaves no tracks.\n"); + p = select_process();
You want to fallback to select_bad_process() if no candidate has been selected at select_process().
You also want to move "oom" to /proc/sys/vm/.
Hi,
Quoted 2 lines
Sorry, I misunderstood. Should have read the code before shouting.
Better shouting then shooting :)!
br
Edjard
Quoted 21 lines
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
Few comments about the code:
retry:
mm/oom_kill.c +2
-1- p =select_bad_process (); +printk (KERN_DEBUG "A good walker leaves no tracks.\n" ); + p =select_process ();You want to fallback to
select_bad_process()if no candidate has been selected atselect_process().You also want to move "oom" to
/proc/sys/vm/.
Signature · 1 line
"In a world without fences ... who needs Gates?"
actually 'than' is much better then :-)
Quoted 37 lines · Edjard Souza Mota
Hi,
Quoted 2 lines
Sorry, I misunderstood. Should have read the code before shouting.
Better shouting then shooting :)!
br
Edjard
Quoted 15 lines
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
Few comments about the code:
retry:
mm/oom_kill.c +2
-1- p =select_bad_process (); +printk (KERN_DEBUG "A good walker leaves no tracks.\n" ); + p =select_process ();Quoted 6 lines
You want to fallback to
select_bad_process()if no candidate has been selected atselect_process().You also want to move "oom" to
/proc/sys/vm/.--
"In a world without fences ... who needs Gates?"
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 30 lines · Marcelo Tosatti
Quoted 7 lines · Edjard Souza Mota
Hi,
I guess it the idea was not fully and well explained. It is not the OOM Killer itself that was moved to user space but rather its ranking algorithm. Ranking is not an specific functionality of kernel space. Kernel only need to know which process whould be killed.
Quoted 5 lines
In that sense the approach is different and might be worth testing, mainly for cases where we want to allow better policies of ranking. For example, an embedded device with few resources and important different running applications:
whic one is the best? To my understanding the current ranking policy does not necessarily chooses the best one to be killed.Sorry, I misunderstood. Should have read the code before shouting.
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
Few comments about the code:
retry:
mm/oom_kill.c +2
-1- p =select_bad_process (); +printk (KERN_DEBUG "A good walker leaves no tracks.\n" ); + p =select_process ();
Quoted 2 lines
You want to fallback to
select_bad_process()if no candidate has been selected atselect_process().
The idea is turn off the select_bad_process() and the new select_process() will get the list of pids to be killed from /proc/oom. But the ranking algorithms is the same, I mean is the Rik van Riel algorithm. Do you think it is worthwhile to maintain the select_bad_process (kernel space algorithm) if we have the select_process() function?
Quoted 2 lines
You also want to move "oom" to
/proc/sys/vm/.
This can be possible. Do you think that it is a good place to move the oom?
BR,
Mauricio Lin.
Hi all,
I noticed something wrong with the first patch. Here goes the fixed patch:
***********
PATCH
***********
fs/proc/array.c +10 -0
@@ -470,3 +470,13 @@return sprintf (buffer,"%d %d %d %d %d %d %d\n" , size, resident, shared, text, lib, data,0 ); } + +int proc_pid_oom (struct task_struct *task,char * buffer) +{ +int res; + res =sprintf (buffer,"%d %lu %lu\n" , + task->pid, + task->utime, + task->stime); +return res; +}
fs/proc/base.c +5 -0
@@ -60,6 +60,7 @@PROC_TGID_MAPS ,PROC_TGID_MOUNTS ,PROC_TGID_WCHAN , +PROC_TGID_OOM , #ifdef CONFIG_SCHEDSTATS PROC_TGID_SCHEDSTAT, #endif
@@ -86,6 +87,7 @@PROC_TID_MAPS ,PROC_TID_MOUNTS ,PROC_TID_WCHAN , +PROC_TID_OOM , #ifdef CONFIG_SCHEDSTATS PROC_TID_SCHEDSTAT, #endif
@@ -132,6 +134,7 @@ #ifdef CONFIG_SCHEDSTATS E(PROC_TGID_SCHEDSTAT, "schedstat", S_IFREG|S_IRUGO), #endif + E(PROC_TGID_OOM, "oom", S_IFREG|S_IRUGO), {0,0,NULL,0} }; static struct pid_entry tid_base_stuff[] = {
@@ -157,6 +160,7 @@#ifdef CONFIG_SCHEDSTATS E (PROC_TID_SCHEDSTAT ,"schedstat" ,S_IFREG |S_IRUGO ), #endif +E (PROC_TID_OOM ,"oom" ,S_IFREG |S_IRUGO ), {0,0,NULL,0} };
@@ -193,6 +197,7 @@int proc_tgid_stat (struct task_struct *,char *);int proc_pid_status (struct task_struct *,char *);int proc_pid_statm (struct task_struct *,char *); +int proc_pid_oom (struct task_struct *,char *); static int proc_fd_link(struct inode *inode, struct dentry **dentry,
struct vfsmount **mnt)
{
@@ -1377,6 +1382,11 @@
ei->op.proc_read = proc_pid_schedstat; break;
#endif
+ case PROC_TID_OOM:
+ case PROC_TGID_OOM:
+ inode->i_fop = &proc_info_file_operations;
+ ei->op.proc_read = proc_pid_oom;
+ break;
default:
printk("procfs: impossible type (%d)",p->type); iput(inode);
diff -urN linux-2.6.10/include/linux/oom_kill.h linux-2.6.10-oom/include/linux/oom_kill.h
--- linux-2.6.10/include/linux/oom_kill.h 1969-12-31 20:00:00.000000000 -0400
+++ linux-2.6.10-oom/include/linux/oom_kill.h 2005-01-10
15:42:26.000000000 -0400
@@ -0,0 +1,6 @@
+struct candidate_process {
+ pid_t pid;
+ struct list_head pid_list;
+};
+
+struct list_head *loop_counter;
mm/oom_kill.c +69 -3
108 lines not on this page · raw message
BR,
Mauricio Lin.
Quoted 32 lines · Mauricio Lin
Quoted 30 lines · Marcelo Tosatti
Quoted 7 lines · Edjard Souza Mota
Hi,
I guess it the idea was not fully and well explained. It is not the OOM Killer itself that was moved to user space but rather its ranking algorithm. Ranking is not an specific functionality of kernel space. Kernel only need to know which process whould be killed.
Quoted 5 lines
In that sense the approach is different and might be worth testing, mainly for cases where we want to allow better policies of ranking. For example, an embedded device with few resources and important different running applications:
whic one is the best? To my understanding the current ranking policy does not necessarily chooses the best one to be killed.Sorry, I misunderstood. Should have read the code before shouting.
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
Few comments about the code:
retry:
mm/oom_kill.c +2
-1- p =select_bad_process (); +printk (KERN_DEBUG "A good walker leaves no tracks.\n" ); + p =select_process ();
Quoted 8 lines
Quoted 2 lines
You want to fallback to
select_bad_process()if no candidate has been selected atselect_process().The idea is turn off the
select_bad_process()and the newselect_process()will get the list of pids to be killed from/proc/oom. But the ranking algorithms is the same, I mean is the Rik van Riel algorithm. Do you think it is worthwhile to maintain theselect_bad_process(kernel space algorithm) if we have theselect_process()function?
Yes, if select_process() fails (in case no process is on the candidate list), i fallbacking to select_bad_process() is important I think.
Quoted 4 lines
Quoted 2 lines
You also want to move "oom" to
/proc/sys/vm/.This can be possible. Do you think that it is a good place to move the oom?
Yep.
Quoted 3 lines · Marcelo Tosatti
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
It's not a requirement for users. The current implementation in the kernel it's just broken, ugly code.
Quoted 2 lines
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
I have no objections against the userspace provided candidate list option, but as long as the main sources of trouble
- invocation
- reentrancy
- timed, counted, blah ugly protection
- selection problem
are not fixed properly, we don't need to discuss the inclusion of a userspace provided candidate list.
Postpone this until the main problem is fixed. There is a proper confirmed fix for this available. It was posted more than once.
Merging a fix which helps only 0,001 % of the users to hide the mess instead of fixing the real problem is a real interesting engineering aproach.
I don't deny, that after the source of trouble is fixed it is worth to think about the merging of this addon to allow interested users to define the culprits instead of relying on an always imperfect selection algorithm.
tglx
Some points on Thomas comments,
Quoted 11 lines
I have no objections against the userspace provided candidate list option, but as long as the main sources of trouble
- invocation
- reentrancy
- timed, counted, blah ugly protection
- selection problemare not fixed properly, we don't need to discuss the inclusion of a userspace provided candidate list.
Any solution that doesn't offer a proper approach to the above issues should not be discussed anyway. By allowing the ranking goes up to the user space is not meant only for user testing ranking, but to keep the OOM Killer kernel code simpler and clean. As a matter of fact, even protected.
Consider the invocation for example. It comes in two phases with this proposal:
1) ranking for the most likely culprits only starts when memory consumption gets close to the red zone (for example 98% or something like that).
2) killing just gets the first candidate from the list and kills it. No need to calculate
at kernel level.
The selection problem is very dependent on the ranking algorithm. For PCs it may not be a trouble, but for emdedded devices? yes it is. The ranking at the kernel level uses only int type of integer. If you get the log file for the ranking
in any embedded device you will notice that many processes end up with the same ranking point. Thus, there will never be the best choice in this way.
By moving just the ranking to the user space fix this problem 'cause you may use float to order PIDs with different indexes. The good side effect is that we allow better ways of choosing the culprit by means of diffrent calculations to meet different patterns of memory consumtion.
Quoted 11 lines
Postpone this until the main problem is fixed. There is a proper confirmed fix for this available. It was posted more than once.
Merging a fix which helps only 0,001 % of the users to hide the mess instead of fixing the real problem is a real interesting engineering aproach.
I don't deny, that after the source of trouble is fixed it is worth to think about the merging of this addon to allow interested users to define the culprits instead of relying on an always imperfect selection algorithm.
br
Edjard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 11 lines · Edjard Souza Mota
Quoted 10 lines
I have no objections against the userspace provided candidate list option, but as long as the main sources of trouble
- invocation
- reentrancy
- timed, counted, blah ugly protection
- selection problemare not fixed properly, we don't need to discuss the inclusion of a userspace provided candidate list.
Quoted 7 lines
Any solution that doesn't offer a proper approach to the above issues should not be discussed anyway. By allowing the ranking goes up to the user space is not meant only for user testing ranking, but to keep the OOM Killer kernel code simpler and clean. As a matter of fact, even protected.
Consider the invocation for example. It comes in two phases with this proposal:
I consider the invocation of out_of_memory in the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... in out_of_memory, which aren't neccecary at all
This must be solved first in a proper way, before we talk about ranking.
You are definitely curing the symptom instead of the cause.
Quoted 5 lines
1) ranking for the most likely culprits only starts when memory consumption gets close to the red zone (for example 98% or something like that).
2) killing just gets the first candidate from the list and kills it. No need to calculate
at kernel level.
What is the default behaviour when no userspace settings are available - Nothing ? Are you really expecting that we change every root fs in order to be able to upgrade the kernel for solving this _kernel_ problem ?
Who is setting up those userspace constraints ? Joe User, who is barely able to find the power on button on his box ? The sysadmin, who will have to adjust the list for each box depending on the apps it runs or the user who is logged into the box ?
Memory management _is_ a kernel task and so the shortage of memory has to be handled by the kernel on its own in the first place. Adding user space support for certain tasks is a good thing, but not a solution to the problem itself.
Quoted 6 lines
The selection problem is very dependent on the ranking algorithm. For PCs it may not be a trouble, but for emdedded devices? yes it is. The ranking at the kernel level uses only int type of integer. If you get the log file for the ranking
in any embedded device you will notice that many processes end up with the same ranking point. Thus, there will never be the best choice in this way.
I know the constrains of embedded boxes well enough to know that there is a bit of a difference to a desktop machine.
Quoted 4 lines
By moving just the ranking to the user space fix this problem 'cause you may use float to order PIDs with different indexes. The good side effect is that we allow better ways of choosing the culprit by means of diffrent calculations to meet different patterns of memory consumtion.
I'm running Andrea's and my combined fixes on a couple of embedded and desktop boxes and it has proven to be a proper in kernel solution for the in kernel problem.
I don't argue againts the ability to provide a culprit list to the kernel, but as I said before it only can be a optional addon to a proper in kernel solution.
tglx
Quoted 7 lines · Thomas Gleixner
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at all
Thomas, you're obviously right, it's not even worth discussing this. The 6 patches I posted (and my version is the only one that includes all the outstanding fixes) have to be applied. Than we can think about the rest.
Rik's two patches (writeback-highmem and writeback_nr_scanned) should be applied too since they're obviously right too (and they're completely orthogonal with our 6). Rik's 2/2 looked more like an hack and it shouldn't be applied.
Quoted 2 lines · Andrea Arcangeli
The 6 patches I posted
I have the original versions of these saved away but they generate a ton of rejects now. When you sync them up to Linus's current tree could you pleae resend them all?
Quoted 3 lines · Andrew Morton
I have the original versions of these saved away but they generate a ton of rejects now. When you sync them up to Linus's current tree could you pleae resend them all?
Can I trust the kernel CVS to be uptodate? I normally use it for such things but I'd prefer to be sure that I can trust it before risking wasting time (it has been unstable recently and I so I was working with patches in the meantime).
Quoted 7 lines · Andrea Arcangeli
Quoted 3 lines · Andrew Morton
I have the original versions of these saved away but they generate a ton of rejects now. When you sync them up to Linus's current tree could you pleae resend them all?
Can I trust the kernel CVS to be uptodate?
I don't know - I've never used it. Others might know.
http://www.kernel.org/pub/linux/kernel/v2.5/testing/cset/ is always up to date. The problematic patches were merged three days ago though.
Quoted 4 lines
I normally use it for such
things but I'd prefer to be sure that I can trust it before risking wasting time (it has been unstable recently and I so I was working with patches in the meantime).
No huge rush - any time in the next week or two would suit. I'd expect -rc1 within the next week anwyay.
Thanks.
Quoted 14 lines · Andrea Arcangeli
Quoted 7 lines · Thomas Gleixner
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at allThomas, you're obviously right, it's not even worth discussing this. The 6 patches I posted (and my version is the only one that includes all the outstanding fixes) have to be applied. Than we can think about the rest.
Quoted 4 lines
Rik's two patches (writeback-highmem and
writeback_nr_scanned) should be applied too since they're obviously right too (and they're completely orthogonal with our 6). Rik's 2/2 looked more like an hack and it shouldn't be applied.
This patchsets should be in -mm by now? :)
Hi,
Quoted 7 lines
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at all
You're 100% right! It was not the purpose to solve it in the first place, but rather
to remove the ranking algorithm out of kernel.
Hmmm, it seems you removed the selection problem from your original list, that means we made our point. Thankyou!
Quoted 10 lines
This must be solved first in a proper way, before we talk about ranking.
You are definitely curing the symptom instead of the cause.
Quoted 5 lines
1) ranking for the most likely culprits only starts when memory consumption gets close to the red zone (for example 98% or something like that).
2) killing just gets the first candidate from the list and kills it. No need to calculate
at kernel level.
Quoted 3 lines
What is the default behaviour when no userspace settings are available - Nothing ? Are you really expecting that we change every root fs in order to be able to upgrade the kernel for solving this _kernel_ problem ?
No, I certainly don't. But, have seen the application we also posted? It is a test for while, that actually starts a deamon when you boot the kernel and does rate this application, i.e. an application with root rating priority so it will never be killed and never lack space for itself.
So, the answer to your 2nd very good point.
But as I wrote above it is an idea to be tested not to impose any change in every
root fs.
Quoted 4 lines
Who is setting up those userspace constraints ? Joe User, who is barely able to find the power on button on his box ? The sysadmin, who will have to adjust the list for each box depending on the apps it runs or the user who is logged into the box ?
Well from the discussion below and your reasonable argument it is clear that we are thinking of it for embeded deveices, in the first place.
Quoted 4 lines
Memory management _is_ a kernel task and so the shortage of memory has to be handled by the kernel on its own in the first place. Adding user space support for certain tasks is a good thing, but not a solution to the problem itself.
Yes, you're 100% right again. Sorry but, Does this mean that for everything it manages the kernel should compute rates for PID? It seems that only tests can show which way is more appropriate. Perhaps it may end up showing that both ways reach the same result, but, perhaps, with different algorithm ingenuity. We shall work on this test data and it soon. Thank you.
Quoted 10 lines
Quoted 7 lines
The selection problem is very dependent on the ranking algorithm. For PCs it may not be a trouble, but for emdedded devices? yes it is. The ranking at the kernel level uses only int type of integer. If you get the log file for the ranking
in any embedded device you will notice that many processes end up with the same ranking point. Thus, there will never be the best choice in this way.I know the constrains of embedded boxes well enough to know that there is a bit of a difference to a desktop machine.
I didn't mean you don't know. Don't it personal.
Quoted 6 lines
Quoted 5 lines
By moving just the ranking to the user space fix this problem 'cause you may use float to order PIDs with different indexes. The good side effect is that we allow better ways of choosing the culprit by means of diffrent calculations to meet different patterns of memory consumtion.
Quoted 7 lines
I'm running Andrea's and my combined fixes on a couple of embedded and desktop boxes and it has proven to be a proper in kernel solution for the in kernel problem.
I don't argue againts the ability to provide a culprit list to the kernel, but as I said before it only can be a optional addon to a proper in kernel solution.
Back to your list of problems, we tackled just one. The selection problem. For sure we will investigate on your solution and see a faster way to enhance ours with with your approach. Who knows the other way around might work as well?
Thank you again
br
Edjard
Quoted 6 lines
tglx
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 4 lines · Edjard Souza Mota
Quoted 4 lines
You are definitely curing the symptom instead of the cause.
Quoted 2 lines
1) ranking for the most likely culprits only starts when memory consumption gets close to the red zone (for example 98% or something like that).
We do the ranking only in the oom situation, so what's your point ?
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
2) killing just gets the first candidate from the list and kills it. No need to calculate
at kernel level.
So I need a userspace change in order to solve a kernel problem ?
Quoted 9 lines
Quoted 3 lines
What is the default behaviour when no userspace settings are available - Nothing ? Are you really expecting that we change every root fs in order to be able to upgrade the kernel for solving this _kernel_ problem ?
No, I certainly don't. But, have seen the application we also posted? It is a test for while, that actually starts a deamon when you boot the kernel and does rate this application, i.e. an application with root rating priority so it will never be killed and never lack space for itself.
So, the answer to your 2nd very good point.
You did not answer my question at all. I do not want to update my rootfs to solve a problem which exists in the kernel and must be solved in the kernel.
tglx
Quoted 5 lines · Thomas Gleixner
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
2) killing just gets the first candidate from the list and kills it. No need to calculate
at kernel level.So I need a userspace change in order to solve a kernel problem ?
Allowing userspace to tune is a great idea. However we cannot invoke userland at oom-time to make the decision, or it would be deadlock prone (userland may be swapped out or it might require minor allocations of memory, if we were to allow userspace to do the decision it would be required to be a mlockall userland and not allowed to do syscalls, and even then it could mess up with the stack or signal handlers). So the safe thing to do is to assign different ratings to different userspace tasks. Of course this is inherited from the childs. That is a reasonable approach IMHO. Kurt wrote that patch, I only ported it to a more recent codebase.
This way you can rate your important services and the not important ones.
Anyway as you've mentioned in a earlier email, there were more fundamental problems than the selection algorithm, the userspace rating was the lowest one in the prio list.
Hi,
Quoted 6 lines
Allowing userspace to tune is a great idea. However we cannot invoke userland at oom-time to make the decision, or it would be deadlock prone (userland may be swapped out or it might require minor allocations of memory, if we were to allow userspace to do the decision it would be required to be a mlockall userland and not allowed to do syscalls, and even then it could mess up with the stack or signal handlers).
Hmm, no it is not the case. The deamon application would start from the boot. It only keeps the list of candidates whenever you're getting close to red zone.
There is no deadlock.
Deamon just started at user space, and does only calculation. It doesn't take decision at all. That OOM killer at kernel level who get the list and chooses
who to shoot dead.
Quoted 4 lines
So the safe thing to do is to assign different ratings to different userspace tasks. Of course this is inherited from the childs. That is a reasonable approach IMHO. Kurt wrote that patch, I only ported it to a more recent codebase.
Could be. Interesting idea. We shall keep thinking about it. Have you done some experiment like that?
Quoted 8 lines
This way you can rate your important services and the not important ones.
Anyway as you've mentioned in a earlier email, there were more fundamental problems than the selection algorithm, the userspace rating was the lowest one in the prio list.
Yes, agreed. Our point was just to re-organize current OOM killer to release the kernel from doing rating, which is not its task any way.
br
Edjard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 2 lines · Edjard Souza Mota
Yes, agreed. Our point was just to re-organize current OOM killer to release the kernel from doing rating, which is not its task any way.
It is a kernel task and will always be a kernel task. The kernel manages memory resources and therefor is the place which is responsible to solve the oom situation.
The userland daemon or what ever can only be a add on to give a hint for the final decision.
tglx
Quoted 4 lines · Edjard Souza Mota
Deamon just started at user space, and does only calculation. It doesn't take decision at all. That OOM killer at kernel level who get the list and chooses
who to shoot dead.
Then this is exactly what the oomkilladj patch from Kurt is doing. You tune it with this:
andrea@dualathlon:~> cat /proc/self/oom_adj
0
andrea@dualathlon:~> cat /proc/self/oom_score
627
andrea@dualathlon:~>
(the second one is the score)
With this script I can tell exactly which is going to be the next killed task if the box were to run oom:
ls /proc/*/oom_score| grep -v self | sed 's/\(.*\)\/\(.*\)/echo -n "\1 "; cat \1\/\2/'|sh | sort -nr +1| head -n 1
In this case it would be pid 4175:
/proc/4175 32923
andrea@dualathlon:~> ps 4175
PID TTY STAT TIME COMMAND 4175 ? Ss 0:03 kdeinit: Running...
andrea@dualathlon:~>
Quoted 2 lines
Could be. Interesting idea. We shall keep thinking about it. Have you done some experiment like that?
We ship it in production, it worked so far. Though I don't know if it's flexible as much as you need. Sure it's not going to make the oom killer worse to have some way of tuning it ;).
If you've a better API we can discuss it, the above was quite non-intrusive, it's simple and it does the trick so I don' dislike it.
Anyway as said in the other email, before discussing this stuff we should get the rest fixed. There were more serious problems than the task selection algorithm.
Quoted 2 lines
Yes, agreed. Our point was just to re-organize current OOM killer to release the kernel from doing rating, which is not its task any way.
I believe the kernel can have an huristic that gets right 99% of cases. But for sure the kernel *can't* always get it right, since only the admin knows the semantics and the importance of the stuff being computed, and the oomkilladj is there exactly to let the kernel learn about it too. The kernel has no clue that it's going to kill the database to leave a buggy videogame running, for the kernel all processes are important the same unless it's being tuned by userspace somehow.
The only thing the kernel can do is to take the best decision that will prevent more oom killing in the future. The kernel should exclude from the untuned selection all tasks that even if they're killed, the box would run out of memory again. So the one task that is allocating the memory at the fastest rate, is the best one to kill normally. The current selection algorithm however is not taking into account the allocation rate at all and in turn I believe the current oom killer is quite far from the ideal oom killer. But this is a different topic, it has nothing to do with the current patches, nor with the userland tuning.
Hi,
Quoted 27 lines
Quoted 4 lines
Deamon just started at user space, and does only calculation. It doesn't take decision at all. That OOM killer at kernel level who get the list and chooses
who to shoot dead.Then this is exactly what the oomkilladj patch from Kurt is doing. You tune it with this:
andrea@dualathlon:~> cat
/proc/self/oom_adj
0
andrea@dualathlon:~> cat/proc/self/oom_score
627
andrea@dualathlon:~>(the second one is the score)
With this script I can tell exactly which is going to be the next killed task if the box were to run oom:
ls /proc/*/oom_score| grep -v self | sed 's/\(.*\)\/\(.*\)/echo -n "\1 "; cat \1\/\2/'|sh | sort -nr +1| head -n 1
In this case it would be pid 4175:
/proc/417532923
andrea@dualathlon:~> ps 4175PID TTY STAT TIME COMMAND 4175 ? Ss 0:03 kdeinit: Running...
Quoted 8 lines
andrea@dualathlon:~>
Quoted 2 lines
Could be. Interesting idea. We shall keep thinking about it. Have you done some experiment like that?
We ship it in production, it worked so far. Though I don't know if it's flexible as much as you need. Sure it's not going to make the oom killer worse to have some way of tuning it ;).
Yeap, that's what we believe and this very tiny contribution is, as far as I can see,
orthogonal to you work and we are willing to complement it with more experiments.
Quoted 2 lines
If you've a better API we can discuss it, the above was quite non-intrusive, it's simple and it does the trick so I don' dislike it.
Ok. We already started checking that and we will give you a feed back soon.
Quoted 3 lines
Anyway as said in the other email, before discussing this stuff we should get the rest fixed. There were more serious problems than the task selection algorithm.
Yes, indeed. I agree you and other guys on this respect. It is just that sometimes
small stones inside our shoes cause greater damage in the long run than the big ones we've just faced. All othe problems are like the big ones, and we took the path to remove the small ones first.
Quoted 4 lines
Quoted 3 lines
Yes, agreed. Our point was just to re-organize current OOM killer to release the kernel from doing rating, which is not its task any way.
Quoted 8 lines
I believe the kernel can have an huristic that gets right 99% of cases. But for sure the kernel *can't* always get it right, since only the admin knows the semantics and the importance of the stuff being computed, and the oomkilladj is there exactly to let the kernel learn about it too. The kernel has no clue that it's going to kill the database to leave a buggy videogame running, for the kernel all processes are important the same unless it's being tuned by userspace somehow.
Ok, I realise this may look too philosophical to other folks in this list but just for the sake of reality Linux is going to mobility nowadays. No one denies that. What for? There are certainly better answers then mines, but we hope to allow the freedom of choosing for the application you want to have just on the palm of hand, for affordable prices and fair competition. This is not new, sorry about that.
Taking this into account, who holds the semantics of the importance of applications running but the user of a mobile device? Is it the kernel? Who is the admin?
Quoted 4 lines
The only thing the kernel can do is to take the best decision that will prevent more oom killing in the future. The kernel should exclude from the untuned selection all tasks that even if they're killed, the box would run out of memory again.
For sure there are certain levels of security that operators and manufacturers wnat to have and they warn us what we can and we can't do. Just like with easy-to-use consumer electronic products. Again, nothing new.
Quoted 6 lines
So the one task that is allocating the memory at the fastest rate, is the best one to kill normally. The current selection algorithm however is not taking into account the allocation rate at all and in turn I believe the current oom killer is quite far from the ideal oom killer. But this is a different topic, it has nothing to do with the current patches, nor with the userland tuning.
Yes, your'e absolutely right. The userland tuning is just the part that will allow, let's say, a user profile-based policy of ranking, and this may have impact on such patterns of memory consuption, or rate allocation if you like.
br
Edjard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Hi folks,
i believe the OOM has been a matter of frequent discussion for some time now. The initial implementation was considered good enough for some, ugly and inefficient in selecting the right task to kill for others and downright unecessary for yet more of linux users. I belong to somewhere in between the 1st and the second group.
I personally believe that heuristics never work 100% of the time, even
when carefully thought of, as Andrea conceded. Nevertheless the
kernel needs some guidance upon which to base a decision. This guidance can be a default, or a user-based solution using whatever possible algorithm to do the ranking. One thing is for sure, the kernel should be the one to do the kill ...
I believe that we need to test some more and see what should be the default ranking approach. Certainly Andrea's implementation looks promising, but as a user I would like to opt if I feel capable to implement my own ranking algorithm, in a way that is controlled by myself.
my 2 cents
--
Ilias Biris
Quoted 72 lines · Andrea Arcangeli
Quoted 4 lines · Edjard Souza Mota
Deamon just started at user space, and does only calculation. It doesn't take decision at all. That OOM killer at kernel level who get the list and chooses
who to shoot dead.Then this is exactly what the oomkilladj patch from Kurt is doing. You tune it with this:
andrea@dualathlon:~> cat
/proc/self/oom_adj
0
andrea@dualathlon:~> cat/proc/self/oom_score
627
andrea@dualathlon:~>(the second one is the score)
With this script I can tell exactly which is going to be the next killed task if the box were to run oom:
ls /proc/*/oom_score| grep -v self | sed 's/\(.*\)\/\(.*\)/echo -n "\1 "; cat \1\/\2/'|sh | sort -nr +1| head -n 1
In this case it would be pid 4175:
/proc/417532923
andrea@dualathlon:~> ps 4175PID TTY STAT TIME COMMAND 4175 ? Ss 0:03 kdeinit: Running...andrea@dualathlon:~>
Quoted 2 lines
Could be. Interesting idea. We shall keep thinking about it. Have you done some experiment like that?
We ship it in production, it worked so far. Though I don't know if it's flexible as much as you need. Sure it's not going to make the oom killer worse to have some way of tuning it ;).
If you've a better API we can discuss it, the above was quite non-intrusive, it's simple and it does the trick so I don' dislike it.
Anyway as said in the other email, before discussing this stuff we should get the rest fixed. There were more serious problems than the task selection algorithm.
Quoted 2 lines
Yes, agreed. Our point was just to re-organize current OOM killer to release the kernel from doing rating, which is not its task any way.
I believe the kernel can have an huristic that gets right 99% of cases. But for sure the kernel *can't* always get it right, since only the admin knows the semantics and the importance of the stuff being computed, and the oomkilladj is there exactly to let the kernel learn about it too. The kernel has no clue that it's going to kill the database to leave a buggy videogame running, for the kernel all processes are important the same unless it's being tuned by userspace somehow.
The only thing the kernel can do is to take the best decision that will prevent more oom killing in the future. The kernel should exclude from the untuned selection all tasks that even if they're killed, the box would run out of memory again. So the one task that is allocating the memory at the fastest rate, is the best one to kill normally. The current selection algorithm however is not taking into account the allocation rate at all and in turn I believe the current oom killer is quite far from the ideal oom killer. But this is a different topic, it has nothing to do with the current patches, nor with the userland tuning.
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Hi,
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
1) ranking for the most likely culprits only starts when memory consumption gets close to the red zone (for example 98% or something like that).
We do the ranking only in the oom situation, so what's your point ?
The point is that kernel doesn't need to keep watching the memory space every time a process needs memory. Only when memory is close this red zone.
Quoted 6 lines
Quoted 4 lines
Quoted 4 lines
Quoted 4 lines
2) killing just gets the first candidate from the list and kills it. No need to calculate
at kernel level.So I need a userspace change in order to solve a kernel problem ?
You could see in another way. Release the kernel from calculating rating so that better approaches of choosing the culprit can be proposed and tested withoud mess too much the code.
Quoted 11 lines
Quoted 10 lines
Quoted 4 lines
What is the default behaviour when no userspace settings are available - Nothing ? Are you really expecting that we change every root fs in order to be able to upgrade the kernel for solving this _kernel_ problem ?
No, I certainly don't. But, have seen the application we also posted? It is a test for while, that actually starts a deamon when you boot the kernel and does rate this application, i.e. an application with root rating priority so it will never be killed and never lack space for itself.
So, the answer to your 2nd very good point.
Quoted 3 lines
You did not answer my question at all. I do not want to update my rootfs to solve a problem which exists in the kernel and must be solved in the kernel.
If you stick on this thought, then there is more to say, but rather send test results soon so that you all can evaluate.
br
Edjard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 9 lines · Edjard Souza Mota
Hi,
Quoted 4 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
1) ranking for the most likely culprits only starts when memory consumption gets close to the red zone (for example 98% or something like that).
We do the ranking only in the oom situation, so what's your point ?
The point is that kernel doesn't need to keep watching the memory space every time a process needs memory. Only when memory is close this red zone.
Oh, I see. The mechanism which is doing memory management must not be aware of the resources which it is managing ? Am I missing a point ?
tglx
Quoted 9 lines · Thomas Gleixner
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at allThis must be solved first in a proper way, before we talk about ranking.
echo "2" >/proc/sys/vm/overcommit_memory
End of problem (except for extreme cases) and with current 2.6.10-bk (and -ac because I pulled the patch back into -ac) also for most extreme cases as Andries pre-reserves the stack address spaces.
Hi
where I come from we say (jokingly of course) 'got a headache? chop your own head ... end of problem'.
Though your system is not guaranteed to become more stable. When you forbid overcommitting memory, all you do is make failure occur for ALL processes at a different time. A process is happily doing something useful when all of a sudden its fork may die due to 'out of memory' ... Moreover shutting down overcommit will do that for all processes, not just the one culprit that could be chopped off by oom...
Maybe it is just me but I think with overcommiting a system works more reliably :-)
Quoted 23 lines · Alan Cox
Quoted 9 lines · Thomas Gleixner
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at allThis must be solved first in a proper way, before we talk about ranking.
echo "2" >/proc/sys/vm/overcommit_memory
End of problem (except for extreme cases) and with current 2.6.10-bk (and -ac because I pulled the patch back into -ac) also for most extreme cases as Andries pre-reserves the stack address spaces.
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Signature · 1 line
Ilias Biris
well looking into Alan's email again I think I answered thinking on the wrong side :-) that the suggestion was to switch off OOM altogether and be done with all the discussion... tsk tsk tsk too defensive and hasty I guess :-)
Thinking it in another way alan's email could have the dimension of switching off overcommitment (and thus OOM) whilst in the user-space ranking stage to avoid reentrancy and invocation of oom again and again before killing something. It also solves the issue of using timed/counted resources which is plain ugly and evil. It would though be necessary to switch OOM back on when the OOMK has finally done the kill.
Did I get it right this time Alan?
Quoted 44 lines · Ilias Biris
Hi
where I come from we say (jokingly of course) 'got a headache? chop your own head ... end of problem'.
Though your system is not guaranteed to become more stable. When you forbid overcommitting memory, all you do is make failure occur for ALL processes at a different time. A process is happily doing something useful when all of a sudden its fork may die due to 'out of memory' ... Moreover shutting down overcommit will do that for all processes, not just the one culprit that could be chopped off by oom...
Maybe it is just me but I think with overcommiting a system works more reliably :-)
Quoted 17 lines · Alan Cox
Quoted 9 lines · Thomas Gleixner
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at allThis must be solved first in a proper way, before we talk about ranking.
echo "2" >/proc/sys/vm/overcommit_memory
End of problem (except for extreme cases) and with current 2.6.10-bk (and -ac because I pulled the patch back into -ac) also for most extreme cases as Andries pre-reserves the stack address spaces.
Quoted 6 lines
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/--
Ilias Biris
Signature · 1 line
Ilias Biris
Quoted 13 lines · Ilias Biris
well looking into Alan's email again I think I answered thinking on the wrong side :-) that the suggestion was to switch off OOM altogether and be done with all the discussion... tsk tsk tsk too defensive and hasty I guess :-)
Thinking it in another way alan's email could have the dimension of switching off overcommitment (and thus OOM) whilst in the user-space ranking stage to avoid reentrancy and invocation of oom again and again before killing something. It also solves the issue of using timed/counted resources which is plain ugly and evil. It would though be necessary to switch OOM back on when the OOMK has finally done the kill.
Quoted 1 line
Did I get it right this time Alan?
I don't get it at all.
Fixes for wrong invocation, reentrancy avoidance, removal of the ugly and evil timer,counter hacks are in the wild since more than 6 weeks. They solve the problem without any userspace interaction.
The userspace provided preferrable victim list is an improvement of the generic heuristic and therefor imperfect selection mechanism and nothing else.
tglx
Hi,
Quoted 18 lines · Thomas Gleixner
Quoted 13 lines · Ilias Biris
well looking into Alan's email again I think I answered thinking on the wrong side :-) that the suggestion was to switch off OOM altogether and be done with all the discussion... tsk tsk tsk too defensive and hasty I guess :-)
Thinking it in another way alan's email could have the dimension of switching off overcommitment (and thus OOM) whilst in the user-space ranking stage to avoid reentrancy and invocation of oom again and again before killing something. It also solves the issue of using timed/counted resources which is plain ugly and evil. It would though be necessary to switch OOM back on when the OOMK has finally done the kill.
Quoted 1 line
Did I get it right this time Alan?
I don't get it at all.
Quoted 7 lines
Fixes for wrong invocation, reentrancy avoidance, removal of the ugly and evil timer,counter hacks are in the wild since more than 6 weeks. They solve the problem without any userspace interaction.
The userspace provided preferrable victim list is an improvement of the generic heuristic and therefor imperfect selection mechanism and nothing else.
Once again the user space ranking of PIDs for OOM killer purposes didn't propose a new selection mechanism. This misinterpretation is misleading the discussion of whether it may have som use in embedded devices or not. Even though, I believe that for PCs environment it has a great potential when you think that admins don't stay all the time in front of a computer. Once and while they also have a rest :). In these cases, instead of simply start ranking a deamon could dispatch a msg to her/him saying the system is approaching a red zone.
br,
Edjard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 7 lines · Edjard Souza Mota
propose a new selection mechanism. This misinterpretation is misleading the discussion of whether it may have som use in embedded devices or not. Even though, I believe that for PCs environment it has a great potential when you think that admins don't stay all the time in front of a computer. Once and while they also have a rest :). In these cases, instead of simply start ranking a deamon could dispatch a msg to her/him saying the system is approaching a red zone.
Pretty intriguing. Maybe the daemon waits for the answer mail then, where the admin confirms to select the daemon as the preferrable victim of oom.
tglx
Quoted 9 lines · Thomas Gleixner
Quoted 7 lines · Edjard Souza Mota
propose a new selection mechanism. This misinterpretation is misleading the discussion of whether it may have som use in embedded devices or not. Even though, I believe that for PCs environment it has a great potential when you think that admins don't stay all the time in front of a computer. Once and while they also have a rest :). In these cases, instead of simply start ranking a deamon could dispatch a msg to her/him saying the system is approaching a red zone.
Quoted 5 lines
Pretty intriguing. Maybe the daemon waits for the answer mail then, where the admin confirms to select the daemon as the preferrable victim of oom.
tglx
Hi,
Not worth commenting this "intriguing" idea.
Let's wait test results and Alan's answers to Ilias interpretation.
br
Edjard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 4 lines · Ilias Biris
well looking into Alan's email again I think I answered thinking on the wrong side :-) that the suggestion was to switch off OOM altogether and be done with all the discussion... tsk tsk tsk too defensive and hasty I guess :-)
Thats what mode 2 is all about. There are some problems with over-early triggering of OOM that Andrea fixed that are still relevant (or stick "never OOM if mode == 2" into your kernel)
Quoted 1 line
Did I get it right this time Alan?
Basically yes - the real problem with the OOM situation is there is no correct answer. People have spent years screwing around with the OOM killer selection logic and while you can make it pick large tasks or old tasks or growing tasks easily nobody has a good heuristic about what to die because it depends on the users wishes. OOM requires AF_TELEPATHY sockets and we don't have them.
For most users simply not allowing the mess to occur solves the problem
- not all but most.
Hi,
Thanks Alan...
Quoted 19 lines
Quoted 4 lines
well looking into Alan's email again I think I answered thinking on the wrong side :-) that the suggestion was to switch off OOM altogether and be done with all the discussion... tsk tsk tsk too defensive and hasty I guess :-)
Thats what mode 2 is all about. There are some problems with over-early triggering of OOM that Andrea fixed that are still relevant (or stick "never OOM if mode == 2" into your kernel)
Quoted 1 line
Did I get it right this time Alan?
Basically yes - the real problem with the OOM situation is there is no correct answer. People have spent years screwing around with the OOM killer selection logic and while you can make it pick large tasks or old tasks or growing tasks easily nobody has a good heuristic about what to die because it depends on the users wishes. OOM requires
AF_TELEPATHYsockets and we don't have them.
Quoted 3 lines
For most users simply not allowing the mess to occur solves the problem
- not all but most.
What do you think about the point we are trying to make, i.e., moving the ranking of PIDs to be killed to user space? Or, making user have some influence on it? We were misunderstood because the patch we sent was to make "a slight" organization in the way OOM killer compute rates to PIDs, not to change its selection logic. But now, we can discuss (I mean implement) alternative selection
logics without messing the code at kernel space. The parameters and criteria on how
to combine them can be open to more people test it according to platform and, if not user, at least according to application memory consumpition pattern.
Well, while AF_TELEPATH socket is not on its way :) ... we may at least experiment
different raking policies.
br
Edard
Signature · 1 line
"In a world without fences ... who needs Gates?"
Quoted 4 lines · Edjard Souza Mota
What do you think about the point we are trying to make, i.e., moving the ranking of PIDs to be killed to user space? Or, making user have some influence on it? We were misunderstood because the patch we sent was to make "a slight" organization in the way OOM killer compute rates to PIDs, not to change its
Im sceptical there is an answer but moving it to user space (or at least implementing /proc tunables in user space to experiment) certainly seems to be the right way to find out.
Quoted 3 lines
Well, while
AF_TELEPATHsocket is not on its way :) ... we may at least experiment
different raking policies.
agreed
Quoted 9 lines · Alan Cox
Quoted 4 lines · Edjard Souza Mota
What do you think about the point we are trying to make, i.e., moving the ranking of PIDs to be killed to user space? Or, making user have some influence on it? We were misunderstood because the patch we sent was to make "a slight" organization in the way OOM killer compute rates to PIDs, not to change its
Im sceptical there is an answer but moving it to user space (or at least implementing /proc tunables in user space to experiment) certainly seems to be the right way to find out.
No objections against an userspace tuning mechanism, but I still doubt that replacing the always imperfect in kernel selection completely is feasable.
tglx
Quoted 9 lines · Ilias Biris
Hi
where I come from we say (jokingly of course) 'got a headache? chop your own head ... end of problem'.
Though your system is not guaranteed to become more stable. When you forbid overcommitting memory, all you do is make failure occur for ALL processes at a different time. A process is happily doing something useful when all of a sudden its fork may die due to 'out of memory'
Application which does not check fork, malloc, etc for success is buggy.
--
vda
Quoted 9 lines · Alan Cox
Quoted 8 lines · Thomas Gleixner
I consider the invocation of
out_of_memoryin the first place. This is the real root of the problems. The ranking is a different playground. Your solution does not solve
- invocation madness
- reentrancy protection
- the ugly mess of timers, counters... inout_of_memory, which aren't neccecary at all
Quoted 7 lines
Quoted 1 line
This must be solved first in a proper way, before we talk about ranking.
echo "2" >/proc/sys/vm/overcommit_memory
End of problem (except for extreme cases) and with current 2.6.10-bk (and -ac because I pulled the patch back into -ac) also for most extreme cases as Andries pre-reserves the stack address spaces.
Maybe for you it's end of problem, but I still can reproduce the weird behaviour with 2.6.10-bk with the tests I posted before. I don't buy end of problem proclamations, as long as I can proove the contrary.
The only working patch so far is Andrea's fix of the invocation, which also solves the reentrancy problem, gets rid of the timer,counter hack and my small contribution to the selection algorithm.
tglx
Quoted 1 line · Thomas Gleixner
confirmed fix for this available. It was posted more than once.
I posted 6 patches (1/4,2/4,3/4,4/4,5/4,6/4), they should be all applied to mainline, they're self contained. They add the userspace ratings too.
Those patches fixes a longstanding PF_MEMDIE race too and they optimize used_math as well.
I'm running with all 6 patches applied with an uptime of 6 days on SMP and no problems at all. They're all 6 patches applied to the kotd too (plus the other bits posted on l-k as well for the write throttling, just one bit is still missing but I'll add it soon):
ftp://ftp.suse.com/pub/projects/kernel/kotd/i386/HEAD
Hi Andrea,
I applied your patch and I am checking your code. It is really a very interesting work. I have a question about the function __set_current_state(TASK_INTERRUPTIBLE) you put in out_of_memory function. Do not you think it would be better put set_current_state instead of __set_current_state function? AFAIK the set_current_state function is more feasible for SMP systems, right?
BR,
Mauricio Lin.
Quoted 18 lines · Andrea Arcangeli
Quoted 1 line · Thomas Gleixner
confirmed fix for this available. It was posted more than once.
I posted 6 patches (1/4,2/4,3/4,4/4,5/4,6/4), they should be all applied to mainline, they're self contained. They add the userspace ratings too.
Those patches fixes a longstanding
PF_MEMDIErace too and they optimizeused_mathas well.I'm running with all 6 patches applied with an uptime of 6 days on SMP and no problems at all. They're all 6 patches applied to the kotd too (plus the other bits posted on l-k as well for the write throttling, just one bit is still missing but I'll add it soon):
ftp://ftp.suse.com/pub/projects/kernel/kotd/i386/HEAD
Hi Andrew,
I have another question. You included an oom_adj entry in /proc for each process. This was the approach you used in order to allow someone or something to interfere the ranking algorithm from userland, right? So if i have an another ranking algorithm in user space, I can use it to complement the kernel decision as necessary. Was it your idea?
BR,
Mauricio Lin.
Quoted 34 lines · Mauricio Lin
Hi Andrea,
I applied your patch and I am checking your code. It is really a very interesting work. I have a question about the function
__set_current_state(TASK_INTERRUPTIBLE)you put inout_of_memoryfunction. Do not you think it would be better putset_current_stateinstead of__set_current_statefunction? AFAIK theset_current_statefunction is more feasible for SMP systems, right?BR,
Mauricio Lin.
Quoted 18 lines · Andrea Arcangeli
Quoted 1 line · Thomas Gleixner
confirmed fix for this available. It was posted more than once.
I posted 6 patches (1/4,2/4,3/4,4/4,5/4,6/4), they should be all applied to mainline, they're self contained. They add the userspace ratings too.
Those patches fixes a longstanding
PF_MEMDIErace too and they optimizeused_mathas well.I'm running with all 6 patches applied with an uptime of 6 days on SMP and no problems at all. They're all 6 patches applied to the kotd too (plus the other bits posted on l-k as well for the write throttling, just one bit is still missing but I'll add it soon):
ftp://ftp.suse.com/pub/projects/kernel/kotd/i386/HEAD
Quoted 7 lines · Mauricio Lin
Hi Andrew,
I have another question. You included an
oom_adjentry in /proc for each process. This was the approach you used in order to allow someone or something to interfere the ranking algorithm from userland, right? So if i have an another ranking algorithm in user space, I can use it to complement the kernel decision as necessary. Was it your idea?
Yes, you should use your userspace algorithm to tune the oom killer via the oom_adj and you can check the effect of your changes with oom_score. I posted a one liner ugly script to do that a few days ago on l-k.
The oom_adj has this effect on the badness() code:
/*
* Adjust the score by oomkilladj.
*/
if (p->oomkilladj) {
if (p->oomkilladj > 0)
points <<= p->oomkilladj;
else
points >>= -(p->oomkilladj);
}The biggest the points become, the more likely the task will be choosen by the oom killer.
Hi Andrea,
Your OOM Killer patch was tested and a strange behaviour was found. Basically as normal user we started some applications as openoffice, mozilla and emacs.
And as a root (in another tty) we started a simple program that uses malloc in a forever loop as below:
int main (void)
{
int * mem;
for (;;)
mem = (int *) malloc(sizeof(int));
return 0;
}
Using the original OOM Killer, malloc is the first killed application and the sytem is restored in a useful state. After applying your patch and accomplish the same experiment, the OOM Killer it does not kill malloc program and it enters in a kind of forever loop as below:
1) out_of_memory is invoked;
2) select_bad_process is invoked;
3) the following condition is fullfied;
if ((unlikely(test_tsk_thread_flag(p, TIF_MEMDIE)) || (p->flags & PF_EXITING)) &&
!(p->flags & PF_DEAD)) return ERR_PTR(-1UL);
4) step 1, 2 ,3 above is executed again;
This loop (step 1 until step 4) lasts during a long time (and nothing is killed) until I give up and reboot the system after waiting for some minutes.
Any comments? What do you think about our test case? Could you accomplish the same test case using malloc program as root and other graphical applications as normal user?
Let me know about your ideas.
BR,
Mauricio Lin.
Quoted 28 lines · Andrea Arcangeli
Quoted 7 lines · Mauricio Lin
Hi Andrew,
I have another question. You included an
oom_adjentry in /proc for each process. This was the approach you used in order to allow someone or something to interfere the ranking algorithm from userland, right? So if i have an another ranking algorithm in user space, I can use it to complement the kernel decision as necessary. Was it your idea?Yes, you should use your userspace algorithm to tune the oom killer via the
oom_adjand you can check the effect of your changes withoom_score. I posted a one liner ugly script to do that a few days ago on l-k.The
oom_adjhas this effect on thebadness()code:/* * Adjust the score by oomkilladj. */ if (p->oomkilladj) { if (p->oomkilladj > 0) points <<= p->oomkilladj; else points >>= -(p->oomkilladj); }The biggest the points become, the more likely the task will be choosen by the oom killer.
Quoted 17 lines · Mauricio Lin
Hi Andrea,
Your OOM Killer patch was tested and a strange behaviour was found. Basically as normal user we started some applications as openoffice, mozilla and emacs.
And as a root (in another tty) we started a simple program that uses malloc in a forever loop as below:int main (void)
{
int * mem;
for (;;)mem = (int *) malloc(sizeof(int));return 0;
}
Quoted 7 lines
Using the original OOM Killer, malloc is the first killed application and the sytem is restored in a useful state. After applying your patch and accomplish the same experiment, the OOM Killer it does not kill malloc program and it enters in a kind of forever loop as below:
1)
out_of_memoryis invoked;
2)select_bad_processis invoked;
Which process is selected ?
Quoted 5 lines
3) the following condition is fullfied;
if ((unlikely(test_tsk_thread_flag(p, TIF_MEMDIE))|| (p->flags&PF_EXITING)) &&!(p->flags & PF_DEAD)) return ERR_PTR(-1UL);
???
Can you please show the kernel messages ?
tglx
Hi Thomaz,
Quoted 23 lines · Thomas Gleixner
Quoted 22 lines · Mauricio Lin
Hi Andrea,
Your OOM Killer patch was tested and a strange behaviour was found. Basically as normal user we started some applications as openoffice, mozilla and emacs.
And as a root (in another tty) we started a simple program that uses malloc in a forever loop as below:int main (void)
{
int * mem;
for (;;)mem = (int *) malloc(sizeof(int));return 0;
}Using the original OOM Killer, malloc is the first killed application and the sytem is restored in a useful state. After applying your patch and accomplish the same experiment, the OOM Killer it does not kill malloc program and it enters in a kind of forever loop as below:
Quoted 4 lines
Quoted 2 lines
1)
out_of_memoryis invoked;
2)select_bad_processis invoked;Which process is selected ?
Sometimes the first application to be killed is XFree. AFAIK the malloc is never killed, because the OOM Killer does not stop to do its work. Usually we are not able to check the kernel log file after rebooting the system. Because nothing was written there (perhaps syslogd or klogd were killed during OOM). But I can see the printk messages on the screen during OOM Killer action. This does not happen with original OOM Killer.
I put some printk in order to trace the OOM Killer and IMHO what is going is:
out_of_memory function is invoked and after that the select_bad_process is also invoked.
So its starts to point each task. But during the do_each_thread / while each_thread loop the
condition:
if ((unlikely(test_tsk_thread_flag(p, TIF_MEMDIE)) || (p->flags & PF_EXITING)) &&
!(p->flags & PF_DEAD))
return ERR_PTR(-1UL);is true and it leaves from select_bad_process function because of the return statement.
So the running code return from the point that select_bad_process was called, i.e., in the out_of_memory function. The condition statement in out_of_memory function:
if (PTR_ERR(p) == -1UL)
goto out;
is also true so it goes to "out" label and leaves from the out_of_memory function. But because of the OOM state the out_of_memory function is invoked again and after that the select_bad_process is also invoked again. And during the do_each_thread / while each_thread loop the same condition as mentioned above is true again. So it leaves from select_bad_process function because of the return statement and goes to "out" label and
leaves from the out_of_memory function again. This behaviour is repeated continuously
during a long time until I stop waiting and reboot the system using my own finger.
Quoted 1 line
Can you please show the kernel messages ?
OK. We will try to reach a situation that the printk messages can be written entirely in the log file and show you the kernel messages. But as I said: usually the printks messages are not written in the log file using Andrea's patch. But using the original OOM Killer we can see the messages in the log file. The syslog.conf file is the same for both OOM Killer(Andrea and Original). Do you have any idea what is happening to log file?
If you do not mind, you can accomplish the same test case as I mentioned on my last email. I would like to know if this problem happens to others people as well.
We tested on the laptop and desktop machines with 128MB of RAM and swap space disabled.
BR,
Mauricio Lin.
Quoted 1 line · Mauricio Lin
Sometimes the first application to be killed is XFree. AFAIK the
This makes more sense now. You need somebody trapping sigterm in order to lockup and X sure traps it to recover the text console.
Can you replace this:
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) {
force_sig(SIGTERM, p);
} else {
force_sig(SIGKILL, p);
}with this?
force_sig(SIGKILL, p);
in mm/oom_kill.c.
This should fix it. Problem is that SIGTERM is unsafe even if the app is not malicious, there's not enough ram to pagein the userland sighander, so the system lockups.
We need a sort of timeout where we fallback into SIGKILL if SIGTERM didn't help.
Anyway this is not a new bug, I didn't touch a single bit in that code. I'd really like to see current fixes merged, then we can take care of root app getting killed reliably. In all my test I always run the malicious app as non-root, and anyway I never trap sigterm (X is tiny in my setup, so it never gets killed). Probably the GUI stuff you opened has increased significantly X size for X to be killed.
Hi Andrea,
Quoted 8 lines · Andrea Arcangeli
Quoted 1 line · Mauricio Lin
Sometimes the first application to be killed is XFree. AFAIK the
This makes more sense now. You need somebody trapping sigterm in order to lockup and X sure traps it to recover the text console.
Can you replace this:
Quoted 7 lines
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) { force_sig(SIGTERM, p); } else { force_sig(SIGKILL, p); }with this?
OK, let me test it. If I get some news, I will let you know.
Quoted 4 lines
force_sig(SIGKILL, p);in
mm/oom_kill.c.
BR,
Mauricio Lin.
Hi Andrea,
Quoted 14 lines · Andrea Arcangeli
Quoted 1 line · Mauricio Lin
Sometimes the first application to be killed is XFree. AFAIK the
This makes more sense now. You need somebody trapping sigterm in order to lockup and X sure traps it to recover the text console.
Can you replace this:
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) { force_sig(SIGTERM, p); } else { force_sig(SIGKILL, p); }
Quoted 5 lines
with this?
force_sig(SIGKILL, p);in
mm/oom_kill.c.
Nice. Your suggestion made the error goes away.
We are still testing in order to compare between your OOM Killer and Original OOM Killer.
BR,
Mauricio Lin.
Quoted 19 lines · Mauricio Lin
Hi Andrea,
Quoted 16 lines · Andrea Arcangeli
Quoted 1 line · Mauricio Lin
Sometimes the first application to be killed is XFree. AFAIK the
This makes more sense now. You need somebody trapping sigterm in order to lockup and X sure traps it to recover the text console.
Can you replace this:
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) { force_sig(SIGTERM, p); } else { force_sig(SIGKILL, p); }with this?
Quoted 8 lines
Quoted 3 lines
force_sig(SIGKILL, p);in
mm/oom_kill.c.Nice. Your suggestion made the error goes away.
We are still testing in order to compare between your OOM Killer and Original OOM Killer.
Ok, thanks for the confirmation. So my theory was right.
Basically we've to make this patch, now that you already edited the code, can you diff and send a patch that will be the 6/5 in the serie?
(then after fixing this last very longstanding [now deadlock prone too] bug, we can think how to make at a 7/5 that will wait a few seconds after sending a sigterm, to fallback into a sigkill, that shouldn't be difficult, but the above 6/5 will already make the code correct)
Note, if you add swap it'll workaround it too since then the memhog will be allowed to grow to a larger rss than X. With 128m of ram and no swap, X is one of the biggest with xshm involved from some client app allocating lots of pictures. I could never notice since I always tested it either with swap or on higher mem systems and my test box runs with an idle X too which isn't that big ;).
Quoted 17 lines · Andrea Arcangeli
Quoted 17 lines
Quoted 9 lines
Can you replace this:
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) { force_sig(SIGTERM, p); } else { force_sig(SIGKILL, p); }Quoted 5 lines
with this?
force_sig(SIGKILL, p);in
mm/oom_kill.c.Nice. Your suggestion made the error goes away.
Quoted 8 lines
Quoted 2 lines
We are still testing in order to compare between your OOM Killer and Original OOM Killer.
Ok, thanks for the confirmation. So my theory was right.
Basically we've to make this patch, now that you already edited the code, can you diff and send a patch that will be the 6/5 in the serie?
I've already queued a patch for this:
mm/oom_kill.c renamed from mm/oom_kill.c~mm-fix-several-oom-killer-bugs-fix +1 -6
@@ -198,12 +198,7 @@ static void __oom_kill_task(task_t *p) p->time_slice =HZ ; p->memdie =1 ; -/* This process has hardware access, be more careful. */ -if (cap_t (p->cap_effective) &CAP_TO_MASK (CAP_SYS_RAWIO )) { -force_sig (SIGTERM , p); - }else { -force_sig (SIGKILL , p); - } +force_sig (SIGKILL , p); }static struct mm_struct *oom_kill_task (task_t *p)
However. This means that we'll now kill off tasks which had hardware access. What are the implications of this?
Quoted 8 lines · Andrew Morton
I've already queued a patch for this:
mm/oom_kill.c renamed from mm/oom_kill.c~mm-fix-several-oom-killer-bugs-fix +0
-0@@ -198,12 +198,7 @@ static void __oom_kill_task(task_t *p) p->time_slice =HZ ; p->memdie =1 ;
mm/oom_kill.c static void __oom_kill_task(task_t *p) · line 204
Quoted 10 lines
mm/oom_kill.c renamed from mm/oom_kill.c~mm-fix-several-oom-killer-bugs-fix +1
-6-/* This process has hardware access, be more careful. */ -if (cap_t (p->cap_effective) &CAP_TO_MASK (CAP_SYS_RAWIO )) { -force_sig (SIGTERM , p); - }else { -force_sig (SIGKILL , p); - } +force_sig (SIGKILL , p); }static struct mm_struct *oom_kill_task (task_t *p)
Thanks.
Quoted 2 lines
However. This means that we'll now kill off tasks which had hardware access. What are the implications of this?
The implication of the above is basically that the X server won't be able to restore the text mode, but that avoids the deadlock ;).
And they had not necessairly hardware access. They "might" have hardware access. Note that an app may have hardware access even if it has no rawio capabilities. One can run iopl and then change uid just fine. So the above check is quite weak since it leaves the kernel susceptible to bugs and memleaks in any app started by root. Kernel shouldn't trust root apps, all apps are buggy, root apps too (I even once fixed a signal race in /sbin/init that showed up with the schedule child first sched optimization ;).
iopl and ioperm are the only two things we care about. We can a synchronous reliable eflags/ioperm value only from the "regs" in the task context. Problem is that since we can pick a task to kill that isn't necessairly the current task, we should start to approximate, and assume the process is sleeping. The regs must be saved during reschedule, so it should cache the old contents. So perhaps we can get a pratically reliable eflags dump from the tss_struct. But this will not be common code and it'll require a specialized arch API. Like has_hw_access(). Only then we can make a stronger assumption and be truly careful about sending SIGKILL.
The right way to do this is probably to wait a few seconds before sending the sigkill. I'm not currently sure if it worth adding the has_hw_access(). But certainly I would prefer to do nothing special with only the sys_rawio capability. I thought I could wait the other patches to be merged to avoid confusion before making more changes (since it'd be a pretty self contained feature), but I can do that now if you prefer.
Quoted 3 lines · Andrea Arcangeli
And they had not necessairly hardware access. They "might" have hardware access.
On x86 we could perhaps test for non-nullness of tsk->thread->io_bitmap_ptr?
Quoted 4 lines
I thought I could wait the other patches
to be merged to avoid confusion before making more changes (since it'd be a pretty self contained feature), but I can do that now if you prefer.
I'll send your current stuff off to Linus in the next few days - we can let that sit for a while, use that as a base for further work.
Quoted 1 line · Andrew Morton
On x86 we could perhaps test for non-nullness of
tsk->thread->io_bitmap_ptr?
yes for ioports. But I'm afraid I was too optimistic about eflags for iopl, that's not in the per-task tss, it's only stored at the very top of the kernel stack and inherit during fork/clone. So we probably need to check esp0 and read the top of the stack to see if a task has eflags set. esp0 is definitely stored in the thread struct when the task is rescheduled, and it cannot change for each given task, so we can access it even while the task is runnable and it shouldn't be corrupted by iret. But the problem is sysenter is optimized not to save eflags on the kernel stack, so the top of the stack - 12bytes would not contain eflags if sysenter is in use.
So basically we'd need to change iopl to propagate the info to the task struct synchronously somehow, because we can't read it reliably from the kernel stack.
Hi Andrea,
Quoted 26 lines · Andrea Arcangeli
Quoted 25 lines · Mauricio Lin
Hi Andrea,
Quoted 14 lines · Andrea Arcangeli
Quoted 1 line · Mauricio Lin
Sometimes the first application to be killed is XFree. AFAIK the
This makes more sense now. You need somebody trapping sigterm in order to lockup and X sure traps it to recover the text console.
Can you replace this:
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) { force_sig(SIGTERM, p); } else { force_sig(SIGKILL, p); }Quoted 5 lines
with this?
force_sig(SIGKILL, p);in
mm/oom_kill.c.Nice. Your suggestion made the error goes away.
Quoted 7 lines
Quoted 2 lines
We are still testing in order to compare between your OOM Killer and Original OOM Killer.
Ok, thanks for the confirmation. So my theory was right.
Basically we've to make this patch, now that you already edited the code, can you diff and send a patch that will be the 6/5 in the serie?
OK. I will send the patch.
Quoted 11 lines
(then after fixing this last very longstanding [now deadlock prone too] bug, we can think how to make at a 7/5 that will wait a few seconds after sending a sigterm, to fallback into a sigkill, that shouldn't be difficult, but the above 6/5 will already make the code correct)
Note, if you add swap it'll workaround it too since then the memhog will be allowed to grow to a larger rss than X. With 128m of ram and no swap, X is one of the biggest with xshm involved from some client app allocating lots of pictures. I could never notice since I always tested it either with swap or on higher mem systems and my test box runs with an idle X too which isn't that big ;).
Well, we like to reduce the memory resources, because we also think about OOM Killer in small devices with few resources.
BR,
Mauricio Lin.
Hi Andrea,
Quoted 32 lines · Mauricio Lin
Hi Andrea,
Quoted 29 lines · Andrea Arcangeli
Quoted 27 lines · Mauricio Lin
Hi Andrea,
Quoted 14 lines · Andrea Arcangeli
Quoted 1 line · Mauricio Lin
Sometimes the first application to be killed is XFree. AFAIK the
This makes more sense now. You need somebody trapping sigterm in order to lockup and X sure traps it to recover the text console.
Can you replace this:
if (cap_t(p->cap_effective) & CAP_TO_MASK(CAP_SYS_RAWIO)) { force_sig(SIGTERM, p); } else { force_sig(SIGKILL, p); }Quoted 5 lines
with this?
force_sig(SIGKILL, p);in
mm/oom_kill.c.Nice. Your suggestion made the error goes away.
We are still testing in order to compare between your OOM Killer and Original OOM Killer.
Quoted 6 lines
Quoted 4 lines
Ok, thanks for the confirmation. So my theory was right.
Basically we've to make this patch, now that you already edited the code, can you diff and send a patch that will be the 6/5 in the serie?
OK. I will send the patch.
As you know, Andrew generated the patch. Here goes some test results about your OOM Killer and the Original OOm Killer. We accomplished 10 experiments for each OOM Killer and below are average values.
"Invocations" is the number of times that out_of_memory function is called. "Selections" is the number of times that select_bad_process function is called and "Killed" is the number of killed process.
Original OOM Killer
Invocations average = 51620/10 = 5162
Selections average = 30/10 = 3
Killed average = 38/10 = 3.8
Andrea OOM Killer
Invocations average = 213/10 = 21.3
Selections average = 213/10 = 21.3
Killed average = 52/10 = 5.2
As you can see the number of invocations reduced significantly using your OOM Killer.
I did not know about this problem when I was moving the original ranking algorithm to userland. As Thomaz mentioned: invocation madness, reentrancy problems and those strange timers and counter as now, since, last, lastkill and count. I guess that now i can put some OOM Killer stuffs in userland in a safer manner with those problems solved, right?
BTW, will your OOM Killer be included in the kernel tree?
BR,
Mauricio Lin.
Quoted 13 lines · Mauricio Lin
As you know, Andrew generated the patch. Here goes some test results about your OOM Killer and the Original OOm Killer. We accomplished 10 experiments for each OOM Killer and below are average values.
"Invocations" is the number of times that
out_of_memoryfunction is called. "Selections" is the number of times thatselect_bad_processfunction is called and "Killed" is the number of killed process.Original OOM Killer
Invocations average = 51620/10 = 5162
Selections average = 30/10 = 3
Killed average = 38/10 = 3.8
Quoted 7 lines
Andrea OOM Killer
Invocations average = 213/10 = 21.3
Selections average = 213/10 = 21.3
Killed average = 52/10 = 5.2As you can see the number of invocations reduced significantly using your OOM Killer.
Yep, thanks for testing!
Quoted 6 lines
I did not know about this problem when I was moving the original ranking algorithm to userland. As Thomaz mentioned: invocation madness, reentrancy problems and those strange timers and counter as now, since, last, lastkill and count. I guess that now i can put some OOM Killer stuffs in userland in a safer manner with those problems solved, right?
Yep ;)
Quoted 1 line
BTW, will your OOM Killer be included in the kernel tree?
Yes, Andrew said it should go in the next few days, which is a great news, thanks everyone!
Quoted 9 lines · Mauricio Lin
Quoted 1 line
Can you please show the kernel messages ?
OK. We will try to reach a situation that the printk messages can be written entirely in the log file and show you the kernel messages. But as I said: usually the printks messages are not written in the log file using Andrea's patch. But using the original OOM Killer we can see the messages in the log file. The syslog.conf file is the same for both OOM Killer(Andrea and Original). Do you have any idea what is happening to log file?
Add "console=ttyS0,115200" to your commandline so you get all the messages on the serial console.
tglx
Quoted 8 lines · Mauricio Lin
Hi Andrea,
I applied your patch and I am checking your code. It is really a very interesting work. I have a question about the function
__set_current_state(TASK_INTERRUPTIBLE)you put inout_of_memoryfunction. Do not you think it would be better putset_current_stateinstead of__set_current_statefunction? AFAIK theset_current_statefunction is more feasible for SMP systems, right?
set_current_state is needed only when you need to place a memory barrier after __set_current_state. So it's needed in the usual wait_event loop, right after registering in the waitqueue. Example:
unsigned long flags; wait->flags &= ~WQ_FLAG_EXCLUSIVE; spin_lock_irqsave(&q->lock, flags); if (list_empty(&wait->task_list)) __add_wait_queue(q, wait); /* * don't alter the task state if this is just going to * queue an async wait queue callback */ if (is_sync_wait(wait)) set_current_state(state); spin_unlock_irqrestore(&q->lock, flags);
and even in the above is needed only because spin_unlock has inclusive semantics in ia64. In 2.4 there was no unlock at all after set_current_state and it was like this:
set_current_state(TASK_UNINTERRUPTIBLE); \ if (condition) \ break; \ schedule();
\
The rule of thumb is that if there's nothing between set_current_state and schedule() then __set_current_state is more efficient and equally safe to use. And the oom killer path I posted falls in this category, nothing in between set_current_state and schedule, so no reason to place memory barries in there.
Hope this helps ;)
Quoted 19 lines · Thomas Gleixner
Quoted 3 lines · Marcelo Tosatti
The feature is interesting - several similar patches have been around with similar functionality (people who need usually write their own, I've seen a few), but none has ever been merged, even though it is an important requirement for many users.
It's not a requirement for users. The current implementation in the kernel it's just broken, ugly code.
Quoted 2 lines
This is simple, an ordered list of candidate PIDs. IMO something similar to this should be merged. Andrew ?
I have no objections against the userspace provided candidate list option, but as long as the main sources of trouble
- invocation
- reentrancy
- timed, counted, blah ugly protection
- selection problem
Quoted 5 lines
are not fixed properly, we don't need to discuss the inclusion of a userspace provided candidate list.
Postpone this until the main problem is fixed. There is a proper confirmed fix for this available. It was posted more than once.
Agreed - haven't you and Andrea fixed those recently ?
Quoted 8 lines
Merging a fix which helps only 0,001 % of the users to hide the mess instead of fixing the real problem is a real interesting engineering aproach.
I don't deny, that after the source of trouble is fixed it is worth to think about the merging of this addon to allow interested users to define the culprits instead of relying on an always imperfect selection algorithm.
Yep.
Quoted 7 lines · Marcelo Tosatti
Quoted 5 lines
are not fixed properly, we don't need to discuss the inclusion of a userspace provided candidate list.
Postpone this until the main problem is fixed. There is a proper confirmed fix for this available. It was posted more than once.
Agreed - haven't you and Andrea fixed those recently ?
Yep. The fixes are around for quite a while and Andrea is bringing the fixes up to kernel current, if I understood one of his previous mails correctly.
tglx
Quoted 3 lines · Thomas Gleixner
Yep. The fixes are around for quite a while and Andrea is bringing the fixes up to kernel current, if I understood one of his previous mails correctly.
Yes, it's in my queue (unfortunately I've a few bits to do more urgently but Andrew said it's not in a huge rush ;).
Quoted 12 lines · Marcelo Tosatti
Quoted 7 lines · Mauricio Lin
Hi all,
We have done a comparison between the kernel version and user space version and apparently the behavior is similar. You can also get this patch and module to test it and compare with kernel OOM Killer. Here goes a patch and a module that moves the kernel space OOM Killer algorithm to user space. Let us know about your ideas.
No comments on the code itself - It is interesting to have certain pids "not selectable" by the OOM killer. Patches which have similar funcionality have been floating around.
Quoted 6 lines
The userspace OOM killer is dangerous though. You have to guarantee that allocations will NOT happen until the OOM killer is executed and the killed process is dead and has its pages freed - allocations under OOM can cause deadlocks.
"OOM-killer-in-userspace" is unreliable, not sure if its worth the effort making it reliable (mlock it, flagged as
PF_MEMALLOC, etc).
Actually its only unreliable if its called from OOM time.
The case here is you have a daemon which periodically writes to /proc/oom ?
Quoted 17 lines · Marcelo Tosatti
Quoted 16 lines · Marcelo Tosatti
Quoted 7 lines · Mauricio Lin
Hi all,
We have done a comparison between the kernel version and user space version and apparently the behavior is similar. You can also get this patch and module to test it and compare with kernel OOM Killer. Here goes a patch and a module that moves the kernel space OOM Killer algorithm to user space. Let us know about your ideas.
No comments on the code itself - It is interesting to have certain pids "not selectable" by the OOM killer. Patches which have similar funcionality have been floating around.
The userspace OOM killer is dangerous though. You have to guarantee that allocations will NOT happen until the OOM killer is executed and the killed process is dead and has its pages freed - allocations under OOM can cause deadlocks.
Quoted 7 lines
Quoted 2 lines
"OOM-killer-in-userspace" is unreliable, not sure if its worth the effort making it reliable (mlock it, flagged as
PF_MEMALLOC, etc).Actually its only unreliable if its called from OOM time.
The case here is you have a daemon which periodically writes to
/proc/oom?
Yes, let me explain the idea.
When the memory consumption reaches a percentage of usage, as 98% or something like that, we call this as red zone. So when red zone is reached, the ranking algorithm is started to select which processes could be killed whe out of memory happens. If the memory comsumption is less than this percentage (not in red zone), the ranking algorithm is not executed. So we have a loop the check the memory comsumption all the time and if the red zone is reached, the ranking algorithm is started before the system get the out of memory state.
BR,
Mauricio Lin.