Skip to contentSkip to conversation

taskstats: Improve cumulative time accounting

PATCH Superseded Michael Holzheu 5 participants 24 messages Nov 19, 2010 – Nov 29, 2010

Subsystems (7)
VFS Scheduler s390 exec and binfmt +3 more subsystems

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

cover taskstats: Improve cumulative time accounting

Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

To solve this problem, this patch set duplicates the cumulative accounting data in the signal_struct. In the second set (cdata_acct) the complete cumulative resource counters are stored. The new cumulative CPU time (utime and stime) is then exported via the taskstats interface.

PATCH SET OVERVIEW
------------------
Patches apply on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git

[1] Introduce "struct cdata"
[2] Introduce __account_cdata() function
[3] Introduce cdata_acct for complete cumulative accounting
[4] Export "cdata_acct" with taskstats

#1 lore raw

PATCH 1/4 taskstats: Introduce "struct cdata"

From: Michael Holzheu <[email protected]>

This patch introduces a new structure "struct cdata" that is used to store cumulative resource counters for dead child processes and threads.

Note that there is one asymmetry:
For "struct task_io_accounting" (ioc) there is no extra accounting field for dead threads. One field is used for both, dead processes and threads.

This patch introduces no functional change.

---
 fs/binfmt_elf.c           |    4 +-
 fs/exec.c                 |    2 -
 fs/proc/array.c           |   16 ++++----
 include/linux/sched.h     |   22 +++++++----
 kernel/exit.c             |   86 ++++++++++++++++++++++++----------------------
 kernel/posix-cpu-timers.c |   12 +++---
 kernel/sys.c              |   44 ++++++++++++-----------
 7 files changed, 100 insertions(+), 86 deletions(-)
fs/binfmt_elf.c +2 -2
@@ -1296,8 +1296,8 @@ static void fill_prstatus(struct elf_prs
 		cputime_to_timeval(p->utime, &prstatus->pr_utime);
 		cputime_to_timeval(p->stime, &prstatus->pr_stime);
 	}
-	cputime_to_timeval(p->signal->cutime, &prstatus->pr_cutime);
-	cputime_to_timeval(p->signal->cstime, &prstatus->pr_cstime);
+	cputime_to_timeval(p->signal->cdata_wait.utime, &prstatus->pr_cutime);
+	cputime_to_timeval(p->signal->cdata_wait.stime, &prstatus->pr_cstime);
 }
 
 static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p,
fs/exec.c +1 -1
@@ -894,7 +894,7 @@ static int de_thread(struct task_struct
 
 no_thread_group:
 	if (current->mm)
-		setmax_mm_hiwater_rss(&sig->maxrss, current->mm);
+		setmax_mm_hiwater_rss(&sig->cdata_threads.maxrss, current->mm);
 
 	exit_itimers(sig);
 	flush_itimer_signals();
fs/proc/array.c +8 -8
@@ -413,11 +413,11 @@ static int do_task_stat(struct seq_file
 		num_threads = get_nr_threads(task);
 		collect_sigign_sigcatch(task, &sigign, &sigcatch);
 
-		cmin_flt = sig->cmin_flt;
-		cmaj_flt = sig->cmaj_flt;
-		cutime = sig->cutime;
-		cstime = sig->cstime;
-		cgtime = sig->cgtime;
+		cmin_flt = sig->cdata_wait.min_flt;
+		cmaj_flt = sig->cdata_wait.maj_flt;
+		cutime = sig->cdata_wait.utime;
+		cstime = sig->cdata_wait.stime;
+		cgtime = sig->cdata_wait.gtime;
 		rsslim = ACCESS_ONCE(sig->rlim[RLIMIT_RSS].rlim_cur);
 
 		/* add up live thread stats at the group level */
@@ -430,10 +430,10 @@ static int do_task_stat(struct seq_file
 				t = next_thread(t);
 			} while (t != task);
 
-			min_flt += sig->min_flt;
-			maj_flt += sig->maj_flt;
+			min_flt += sig->cdata_threads.min_flt;
+			maj_flt += sig->cdata_threads.maj_flt;
 			thread_group_times(task, &utime, &stime);
-			gtime = cputime_add(gtime, sig->gtime);
+			gtime = cputime_add(gtime, sig->cdata_threads.gtime);
 		}
 
 		sid = task_session_nr_ns(task, ns);
include/linux/sched.h +14 -8
@@ -510,6 +510,17 @@ struct thread_group_cputimer {
 };
 
 /*
+ * Cumulative resource counters
+ */
+struct cdata {
+	cputime_t utime, stime, gtime;
+	unsigned long nvcsw, nivcsw;
+	unsigned long min_flt, maj_flt;
+	unsigned long inblock, oublock;
+	unsigned long maxrss;
+};
+
+/*
  * NOTE! "signal_struct" does not have it's own
  * locking, because a shared signal_struct always
  * implies a shared sighand_struct, so locking
@@ -582,17 +593,12 @@ struct signal_struct {
 	 * Live threads maintain their own counters and add to these
 	 * in __exit_signal, except for the group leader.
 	 */
-	cputime_t utime, stime, cutime, cstime;
-	cputime_t gtime;
-	cputime_t cgtime;
+	struct cdata cdata_wait;
+	struct cdata cdata_threads;
+	struct task_io_accounting ioac;
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
 	cputime_t prev_utime, prev_stime;
 #endif
-	unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
-	unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
-	unsigned long inblock, oublock, cinblock, coublock;
-	unsigned long maxrss, cmaxrss;
-	struct task_io_accounting ioac;
 
 	/*
 	 * Cumulative ns of schedule CPU time fo dead threads in the
kernel/exit.c +45 -41

124 lines not on this page · raw message

kernel/posix-cpu-timers.c +6 -6
@@ -232,12 +232,12 @@ static int cpu_clock_sample(const clocki
 
 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
 {
-	struct signal_struct *sig = tsk->signal;
+	struct cdata *tcd = &tsk->signal->cdata_threads;
 	struct task_struct *t;
 
-	times->utime = sig->utime;
-	times->stime = sig->stime;
-	times->sum_exec_runtime = sig->sum_sched_runtime;
+	times->utime = tcd->utime;
+	times->stime = tcd->stime;
+	times->sum_exec_runtime = tsk->signal->sum_sched_runtime;
 
 	rcu_read_lock();
 	/* make sure we can trust tsk->thread_group list */
@@ -516,8 +516,8 @@ void posix_cpu_timers_exit_group(struct
 	struct signal_struct *const sig = tsk->signal;
 
 	cleanup_timers(tsk->signal->cpu_timers,
-		       cputime_add(tsk->utime, sig->utime),
-		       cputime_add(tsk->stime, sig->stime),
+		       cputime_add(tsk->utime, sig->cdata_threads.utime),
+		       cputime_add(tsk->stime, sig->cdata_threads.stime),
 		       tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
 }
 
kernel/sys.c +24 -20

76 lines not on this page · raw message

Acked-by on v1 1/4
#2 lore raw

Re: [patch 1/4] taskstats: Introduce "struct cdata"

Quoted 12 lines · Michael Holzheu

From: Michael Holzheu <[email protected]>

This patch introduces a new structure "struct cdata" that is used to store cumulative resource counters for dead child processes and threads.

Note that there is one asymmetry:
For "struct task_io_accounting" (ioc) there is no extra accounting field for dead threads. One field is used for both, dead processes and threads.

This patch introduces no functional change.

Quoted 9 lines
---
 fs/binfmt_elf.c           |    4 +-
 fs/exec.c                 |    2 -
 fs/proc/array.c           |   16 ++++----
 include/linux/sched.h     |   22 +++++++----
 kernel/exit.c             |   86 ++++++++++++++++++++++++----------------------
 kernel/posix-cpu-timers.c |   12 +++---
 kernel/sys.c              |   44 ++++++++++++-----------
 7 files changed, 100 insertions(+), 86 deletions(-)

Looks good to me

Signature · 2 lines
	Three Cheers,
	Balbir
#3 lore raw

Re: [patch 1/4] taskstats: Introduce "struct cdata"

In reply to Michael Holzheu
Quoted 13 lines · Michael Holzheu

From: Michael Holzheu <[email protected]>

This patch introduces a new structure "struct cdata" that is used to store cumulative resource counters for dead child processes and threads.

Note that there is one asymmetry:
For "struct task_io_accounting" (ioc) there is no extra accounting field for dead threads. One field is used for both, dead processes and threads.

This patch introduces no functional change.

Quoted 9 lines
---
 fs/binfmt_elf.c           |    4 +-
 fs/exec.c                 |    2 -
 fs/proc/array.c           |   16 ++++----
 include/linux/sched.h     |   22 +++++++----
 kernel/exit.c             |   86 ++++++++++++++++++++++++----------------------
 kernel/posix-cpu-timers.c |   12 +++---
 kernel/sys.c              |   44 ++++++++++++-----------
 7 files changed, 100 insertions(+), 86 deletions(-)

Looks good. In fact, to me it looks like a cleanup.

But. You seem to forgot to change kernel/signal.c, no?

And cosmetic nit,

kernel/posix-cpu-timers.c static int cpu_clock_sample(const clocki · line 240

Quoted 12 lines

void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) {

kernel/posix-cpu-timers.c +4 -4
-	struct signal_struct *sig = tsk->signal;
+	struct cdata *tcd = &tsk->signal->cdata_threads;
 	struct task_struct *t;
 
-	times->utime = sig->utime;
-	times->stime = sig->stime;
-	times->sum_exec_runtime = sig->sum_sched_runtime;
+	times->utime = tcd->utime;
+	times->stime = tcd->stime;
+	times->sum_exec_runtime = tsk->signal->sum_sched_runtime;

Feel free to ignore, but I don't understand why you removed "sig". Afaics,

	-     times->utime = sig->utime;
	-     times->stime = sig->stime;
	+     times->utime = sig->cdata_threads->utime;
	+     times->stime = sig->cdata_threads->stime;

looks a bit better.

Oleg.

#4 lore raw

Re: [patch 1/4] taskstats: Introduce "struct cdata"

Hello Oleg,

Quoted 12 lines · Oleg Nesterov
Quoted 10 lines · Michael Holzheu
---
 fs/binfmt_elf.c           |    4 +-
 fs/exec.c                 |    2 -
 fs/proc/array.c           |   16 ++++----
 include/linux/sched.h     |   22 +++++++----
 kernel/exit.c             |   86 ++++++++++++++++++++++++----------------------
 kernel/posix-cpu-timers.c |   12 +++---
 kernel/sys.c              |   44 ++++++++++++-----------
 7 files changed, 100 insertions(+), 86 deletions(-)
Quoted 3 lines

Looks good. In fact, to me it looks like a cleanup.

But. You seem to forgot to change kernel/signal.c, no?

Yes, you are right.

Quoted 19 lines

And cosmetic nit,

Review

Quoted 12 lines

void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) {

(quoted diff) +4 -4
-	struct signal_struct *sig = tsk->signal;
+	struct cdata *tcd = &tsk->signal->cdata_threads;
 	struct task_struct *t;
 
-	times->utime = sig->utime;
-	times->stime = sig->stime;
-	times->sum_exec_runtime = sig->sum_sched_runtime;
+	times->utime = tcd->utime;
+	times->stime = tcd->stime;
+	times->sum_exec_runtime = tsk->signal->sum_sched_runtime;

Feel free to ignore, but I don't understand why you removed "sig". Afaics,

Quoted 6 lines
	-     times->utime = sig->utime;
	-     times->stime = sig->stime;
	+     times->utime = sig->cdata_threads->utime;
	+     times->stime = sig->cdata_threads->stime;

looks a bit better.

Yes, this looks better.

I attached the corrected patch.

Michael
---
Subject: taskstats: Introduce "struct cdata"

From: Michael Holzheu <[email protected]>

This patch introduces a new structure "struct cdata" that is used to store cumulative resource counters for dead child processes and threads.

Note that there is one asymmetry:
For "struct task_io_accounting" (ioc) there is no extra accounting field for dead threads. One field is used for both dead processes and threads.

This patch introduces no functional change.

---
 fs/binfmt_elf.c           |    4 +-
 fs/exec.c                 |    2 -
 fs/proc/array.c           |   16 ++++----
 include/linux/sched.h     |   22 +++++++----
 kernel/exit.c             |   86 ++++++++++++++++++++++++----------------------
 kernel/posix-cpu-timers.c |    8 ++--
 kernel/signal.c           |    4 +-
 kernel/sys.c              |   44 ++++++++++++-----------
 8 files changed, 100 insertions(+), 86 deletions(-)
fs/binfmt_elf.c +2 -2
@@ -1296,8 +1296,8 @@ static void fill_prstatus(struct elf_prs
 		cputime_to_timeval(p->utime, &prstatus->pr_utime);
 		cputime_to_timeval(p->stime, &prstatus->pr_stime);
 	}
-	cputime_to_timeval(p->signal->cutime, &prstatus->pr_cutime);
-	cputime_to_timeval(p->signal->cstime, &prstatus->pr_cstime);
+	cputime_to_timeval(p->signal->cdata_wait.utime, &prstatus->pr_cutime);
+	cputime_to_timeval(p->signal->cdata_wait.stime, &prstatus->pr_cstime);
 }
 
 static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p,
fs/exec.c +1 -1
@@ -894,7 +894,7 @@ static int de_thread(struct task_struct
 
 no_thread_group:
 	if (current->mm)
-		setmax_mm_hiwater_rss(&sig->maxrss, current->mm);
+		setmax_mm_hiwater_rss(&sig->cdata_threads.maxrss, current->mm);
 
 	exit_itimers(sig);
 	flush_itimer_signals();
fs/proc/array.c +8 -8
@@ -413,11 +413,11 @@ static int do_task_stat(struct seq_file
 		num_threads = get_nr_threads(task);
 		collect_sigign_sigcatch(task, &sigign, &sigcatch);
 
-		cmin_flt = sig->cmin_flt;
-		cmaj_flt = sig->cmaj_flt;
-		cutime = sig->cutime;
-		cstime = sig->cstime;
-		cgtime = sig->cgtime;
+		cmin_flt = sig->cdata_wait.min_flt;
+		cmaj_flt = sig->cdata_wait.maj_flt;
+		cutime = sig->cdata_wait.utime;
+		cstime = sig->cdata_wait.stime;
+		cgtime = sig->cdata_wait.gtime;
 		rsslim = ACCESS_ONCE(sig->rlim[RLIMIT_RSS].rlim_cur);
 
 		/* add up live thread stats at the group level */
@@ -430,10 +430,10 @@ static int do_task_stat(struct seq_file
 				t = next_thread(t);
 			} while (t != task);
 
-			min_flt += sig->min_flt;
-			maj_flt += sig->maj_flt;
+			min_flt += sig->cdata_threads.min_flt;
+			maj_flt += sig->cdata_threads.maj_flt;
 			thread_group_times(task, &utime, &stime);
-			gtime = cputime_add(gtime, sig->gtime);
+			gtime = cputime_add(gtime, sig->cdata_threads.gtime);
 		}
 
 		sid = task_session_nr_ns(task, ns);
include/linux/sched.h +14 -8
@@ -510,6 +510,17 @@ struct thread_group_cputimer {
 };
 
 /*
+ * Cumulative resource counters
+ */
+struct cdata {
+	cputime_t utime, stime, gtime;
+	unsigned long nvcsw, nivcsw;
+	unsigned long min_flt, maj_flt;
+	unsigned long inblock, oublock;
+	unsigned long maxrss;
+};
+
+/*
  * NOTE! "signal_struct" does not have it's own
  * locking, because a shared signal_struct always
  * implies a shared sighand_struct, so locking
@@ -582,17 +593,12 @@ struct signal_struct {
 	 * Live threads maintain their own counters and add to these
 	 * in __exit_signal, except for the group leader.
 	 */
-	cputime_t utime, stime, cutime, cstime;
-	cputime_t gtime;
-	cputime_t cgtime;
+	struct cdata cdata_wait;
+	struct cdata cdata_threads;
+	struct task_io_accounting ioac;
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
 	cputime_t prev_utime, prev_stime;
 #endif
-	unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw;
-	unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt;
-	unsigned long inblock, oublock, cinblock, coublock;
-	unsigned long maxrss, cmaxrss;
-	struct task_io_accounting ioac;
 
 	/*
 	 * Cumulative ns of schedule CPU time fo dead threads in the
kernel/exit.c +45 -41

124 lines not on this page · raw message

kernel/posix-cpu-timers.c +4 -4
@@ -235,8 +235,8 @@ void thread_group_cputime(struct task_st
 	struct signal_struct *sig = tsk->signal;
 	struct task_struct *t;
 
-	times->utime = sig->utime;
-	times->stime = sig->stime;
+	times->utime = sig->cdata_threads.utime;
+	times->stime = sig->cdata_threads.stime;
 	times->sum_exec_runtime = sig->sum_sched_runtime;
 
 	rcu_read_lock();
@@ -516,8 +516,8 @@ void posix_cpu_timers_exit_group(struct
 	struct signal_struct *const sig = tsk->signal;
 
 	cleanup_timers(tsk->signal->cpu_timers,
-		       cputime_add(tsk->utime, sig->utime),
-		       cputime_add(tsk->stime, sig->stime),
+		       cputime_add(tsk->utime, sig->cdata_threads.utime),
+		       cputime_add(tsk->stime, sig->cdata_threads.stime),
 		       tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
 }
 
kernel/signal.c +2 -2
@@ -1476,9 +1476,9 @@ int do_notify_parent(struct task_struct
 	rcu_read_unlock();
 
 	info.si_utime = cputime_to_clock_t(cputime_add(tsk->utime,
-				tsk->signal->utime));
+				tsk->signal->cdata_threads.utime));
 	info.si_stime = cputime_to_clock_t(cputime_add(tsk->stime,
-				tsk->signal->stime));
+				tsk->signal->cdata_threads.stime));
 
 	info.si_status = tsk->exit_code & 0x7f;
 	if (tsk->exit_code & 0x80)
kernel/sys.c +24 -20

76 lines not on this page · raw message

#5 lore raw

PATCH 2/4 taskstats: Introduce __account_cdata() function

In reply to Michael Holzheu

From: Michael Holzheu <[email protected]>

This patch introduces the function __account_cdata() that does the cummulative resource accounting for dead processes in sys_wait(). No functional changes are done. This patch is a preparation for cdata_acct accounting.

---
 kernel/exit.c |  133 +++++++++++++++++++++++++++++-----------------------------
 1 file changed, 68 insertions(+), 65 deletions(-)
kernel/exit.c +68 -65
@@ -74,6 +74,72 @@ static void __unhash_process(struct task
 	list_del_rcu(&p->thread_group);
 }
 
+static void __account_cdata(struct task_struct *p)
+{
+	struct cdata *cd, *pcd, *tcd;
+	unsigned long maxrss;
+	cputime_t tgutime, tgstime;
+
+	/*
+	 * The resource counters for the group leader are in its
+	 * own task_struct.  Those for dead threads in the group
+	 * are in its signal_struct, as are those for the child
+	 * processes it has previously reaped.  All these
+	 * accumulate in the parent's signal_struct c* fields.
+	 *
+	 * We don't bother to take a lock here to protect these
+	 * p->signal fields, because they are only touched by
+	 * __exit_signal, which runs with tasklist_lock
+	 * write-locked anyway, and so is excluded here.  We do
+	 * need to protect the access to parent->signal fields,
+	 * as other threads in the parent group can be right
+	 * here reaping other children at the same time.
+	 *
+	 * We use thread_group_times() to get times for the thread
+	 * group, which consolidates times for all threads in the
+	 * group including the group leader.
+	 */
+	thread_group_times(p, &tgutime, &tgstime);
+	spin_lock_irq(&p->real_parent->sighand->siglock);
+	pcd = &p->real_parent->signal->cdata_wait;
+	tcd = &p->signal->cdata_threads;
+	cd = &p->signal->cdata_wait;
+
+	pcd->utime =
+		cputime_add(pcd->utime,
+		cputime_add(tgutime,
+			    cd->utime));
+	pcd->stime =
+		cputime_add(pcd->stime,
+		cputime_add(tgstime,
+			    cd->stime));
+	pcd->gtime =
+		cputime_add(pcd->gtime,
+		cputime_add(p->gtime,
+		cputime_add(tcd->gtime,
+			    cd->gtime)));
+	pcd->min_flt +=
+		p->min_flt + tcd->min_flt + cd->min_flt;
+	pcd->maj_flt +=
+		p->maj_flt + tcd->maj_flt + cd->maj_flt;
+	pcd->nvcsw +=
+		p->nvcsw + tcd->nvcsw + cd->nvcsw;
+	pcd->nivcsw +=
+		p->nivcsw + tcd->nivcsw + cd->nivcsw;
+	pcd->inblock +=
+		task_io_get_inblock(p) +
+		tcd->inblock + cd->inblock;
+	pcd->oublock +=
+		task_io_get_oublock(p) +
+		tcd->oublock + cd->oublock;
+	maxrss = max(tcd->maxrss, cd->maxrss);
+	if (pcd->maxrss < maxrss)
+		pcd->maxrss = maxrss;
+	task_io_accounting_add(&p->real_parent->signal->ioac, &p->ioac);
+	task_io_accounting_add(&p->real_parent->signal->ioac, &p->signal->ioac);
+	spin_unlock_irq(&p->real_parent->sighand->siglock);
+}
+
 /*
  * This function expects the tasklist_lock write-locked.
  */
@@ -1226,71 +1292,8 @@ static int wait_task_zombie(struct wait_
 	 * It can be ptraced but not reparented, check
 	 * !task_detached() to filter out sub-threads.
 	 */
-	if (likely(!traced) && likely(!task_detached(p))) {
-		struct cdata *cd, *pcd, *tcd;
-		unsigned long maxrss;
-		cputime_t tgutime, tgstime;
-
-		/*
-		 * The resource counters for the group leader are in its
-		 * own task_struct.  Those for dead threads in the group
-		 * are in its signal_struct, as are those for the child
-		 * processes it has previously reaped.  All these
-		 * accumulate in the parent's signal_struct c* fields.
-		 *
-		 * We don't bother to take a lock here to protect these
-		 * p->signal fields, because they are only touched by
-		 * __exit_signal, which runs with tasklist_lock
-		 * write-locked anyway, and so is excluded here.  We do
-		 * need to protect the access to parent->signal fields,
-		 * as other threads in the parent group can be right
-		 * here reaping other children at the same time.
-		 *
-		 * We use thread_group_times() to get times for the thread
-		 * group, which consolidates times for all threads in the
-		 * group including the group leader.
-		 */
-		thread_group_times(p, &tgutime, &tgstime);
-		spin_lock_irq(&p->real_parent->sighand->siglock);
-		pcd = &p->real_parent->signal->cdata_wait;
-		tcd = &p->signal->cdata_threads;
-		cd = &p->signal->cdata_wait;
-
-		pcd->utime =
-			cputime_add(pcd->utime,
-			cputime_add(tgutime,
-				    cd->utime));
-		pcd->stime =
-			cputime_add(pcd->stime,
-			cputime_add(tgstime,
-				    cd->stime));
-		pcd->gtime =
-			cputime_add(pcd->gtime,
-			cputime_add(p->gtime,
-			cputime_add(tcd->gtime,
-				    cd->gtime)));
-		pcd->min_flt +=
-			p->min_flt + tcd->min_flt + cd->min_flt;
-		pcd->maj_flt +=
-			p->maj_flt + tcd->maj_flt + cd->maj_flt;
-		pcd->nvcsw +=
-			p->nvcsw + tcd->nvcsw + cd->nvcsw;
-		pcd->nivcsw +=
-			p->nivcsw + tcd->nivcsw + cd->nivcsw;
-		pcd->inblock +=
-			task_io_get_inblock(p) +
-			tcd->inblock + cd->inblock;
-		pcd->oublock +=
-			task_io_get_oublock(p) +
-			tcd->oublock + cd->oublock;
-		maxrss = max(tcd->maxrss, cd->maxrss);
-		if (pcd->maxrss < maxrss)
-			pcd->maxrss = maxrss;
-		task_io_accounting_add(&p->real_parent->signal->ioac, &p->ioac);
-		task_io_accounting_add(&p->real_parent->signal->ioac,
-				       &p->signal->ioac);
-		spin_unlock_irq(&p->real_parent->sighand->siglock);
-	}
+	if (likely(!traced) && likely(!task_detached(p)))
+		__account_cdata(p);
 
 	/*
 	 * Now we are sure this task is interesting, and no other
#6 lore raw

PATCH 3/4 taskstats: Introduce cdata_acct for complete cumulative accounting

In reply to Michael Holzheu

From: Michael Holzheu <[email protected]>

Currently the cumulative time accounting in Linux is not complete. Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

This patch adds a new set of cumulative time counters. We then have two cumulative counter sets:

* cdata_wait: Traditional cumulative time used e.g. by getrusage.
* cdata_acct: Cumulative time that also includes dead processes with parents that ignore SIGCHLD or have set SA_NOCLDWAIT. cdata_acct will be exported by taskstats.

TODO:
-----
With this patch we take the siglock twice. First for the dead task
and second for the parent of the dead task. This give the following
lockdep warning (probably a lockdep annotation is needed here):
=============================================
[ INFO: possible recursive locking detected ] 2.6.37-rc1-00116-g151f52f-dirty #19
---------------------------------------------
kworker/u:0/15 is trying to acquire lock:
(&(&sighand->siglock)->rlock){......}, at: [<000000000014a426>] __account_cdata+0x6e/0x444
but task is already holding lock:
(&(&sighand->siglock)->rlock){......}, at: [<000000000014b634>] release_task+0x160/0x6a0

---
 include/linux/sched.h |    2 ++
 kernel/exit.c         |   36 +++++++++++++++++++++++++-----------
 2 files changed, 27 insertions(+), 11 deletions(-)
include/linux/sched.h +2 -0
@@ -595,6 +595,8 @@ struct signal_struct {
 	 */
 	struct cdata cdata_wait;
 	struct cdata cdata_threads;
+	struct cdata cdata_acct;
+	struct task_io_accounting ioac_acct;
 	struct task_io_accounting ioac;
 #ifndef CONFIG_VIRT_CPU_ACCOUNTING
 	cputime_t prev_utime, prev_stime;
kernel/exit.c +25 -11
@@ -74,10 +74,10 @@ static void __unhash_process(struct task
 	list_del_rcu(&p->thread_group);
 }
 
-static void __account_cdata(struct task_struct *p)
+static void __account_cdata(struct task_struct *p, int wait)
 {
 	struct cdata *cd, *pcd, *tcd;
-	unsigned long maxrss;
+	unsigned long maxrss, flags;
 	cputime_t tgutime, tgstime;
 
 	/*
@@ -100,11 +100,16 @@ static void __account_cdata(struct task_
 	 * group including the group leader.
 	 */
 	thread_group_times(p, &tgutime, &tgstime);
-	spin_lock_irq(&p->real_parent->sighand->siglock);
-	pcd = &p->real_parent->signal->cdata_wait;
-	tcd = &p->signal->cdata_threads;
-	cd = &p->signal->cdata_wait;
-
+	spin_lock_irqsave(&p->real_parent->sighand->siglock, flags);
+	if (wait) {
+		pcd = &p->real_parent->signal->cdata_wait;
+		tcd = &p->signal->cdata_threads;
+		cd = &p->signal->cdata_wait;
+	} else {
+		pcd = &p->real_parent->signal->cdata_acct;
+		tcd = &p->signal->cdata_threads;
+		cd = &p->signal->cdata_acct;
+	}
 	pcd->utime =
 		cputime_add(pcd->utime,
 		cputime_add(tgutime,
@@ -135,9 +140,17 @@ static void __account_cdata(struct task_
 	maxrss = max(tcd->maxrss, cd->maxrss);
 	if (pcd->maxrss < maxrss)
 		pcd->maxrss = maxrss;
-	task_io_accounting_add(&p->real_parent->signal->ioac, &p->ioac);
-	task_io_accounting_add(&p->real_parent->signal->ioac, &p->signal->ioac);
-	spin_unlock_irq(&p->real_parent->sighand->siglock);
+	if (wait) {
+		task_io_accounting_add(&p->real_parent->signal->ioac, &p->ioac);
+		task_io_accounting_add(&p->real_parent->signal->ioac,
+				       &p->signal->ioac);
+	} else {
+		task_io_accounting_add(&p->real_parent->signal->ioac_acct,
+				       &p->ioac);
+		task_io_accounting_add(&p->real_parent->signal->ioac_acct,
+				       &p->signal->ioac_acct);
+	}
+	spin_unlock_irqrestore(&p->real_parent->sighand->siglock, flags);
 }
 
 /*
@@ -157,6 +170,7 @@ static void __exit_signal(struct task_st
 
 	posix_cpu_timers_exit(tsk);
 	if (group_dead) {
+		__account_cdata(tsk, 0);
 		posix_cpu_timers_exit_group(tsk);
 		tty = sig->tty;
 		sig->tty = NULL;
@@ -1293,7 +1307,7 @@ static int wait_task_zombie(struct wait_
 	 * !task_detached() to filter out sub-threads.
 	 */
 	if (likely(!traced) && likely(!task_detached(p)))
-		__account_cdata(p);
+		__account_cdata(p, 1);
 
 	/*
 	 * Now we are sure this task is interesting, and no other
#7 lore raw

Re: [patch 3/4] taskstats: Introduce cdata_acct for complete cumulative accounting

Quoted 8 lines · Michael Holzheu

Currently the cumulative time accounting in Linux is not complete. Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

Quoted 7 lines

This patch adds a new set of cumulative time counters. We then have two cumulative counter sets:

* cdata_wait: Traditional cumulative time used e.g. by getrusage.
* cdata_acct: Cumulative time that also includes dead processes with parents that ignore SIGCHLD or have set SA_NOCLDWAIT. cdata_acct will be exported by taskstats.

Looks correct at first glance. A couple of nits below.

Quoted 5 lines

TODO:
-----
With this patch we take the siglock twice. First for the dead task and second for the parent of the dead task. This give the following lockdep warning (probably a lockdep annotation is needed here):

And we already discussed this ;) We do not need 2 siglock's, only parent's. Just move the callsite in __exit_signal() down, under another (lockless) group_dead check.

Or I missed something?

include/linux/sched.h struct signal_struct { · line 600

Quoted 7 lines
include/linux/sched.h +2 -0
@@ -595,6 +595,8 @@ struct signal_struct {
 	 */
 	struct cdata cdata_wait;
 	struct cdata cdata_threads;
+	struct cdata cdata_acct;
+	struct task_io_accounting ioac_acct;
 	struct task_io_accounting ioac;

Given that task_io_accounting is Linux specific, perhaps we can use signal->ioac in both cases?

Yes, this is a user-visible change anyway. But, at least we can forget about POSIX.

kernel/exit.c static void __account_cdata(struct task_ · line 112

Quoted 10 lines
kernel/exit.c +10 -0
+	spin_lock_irqsave(&p->real_parent->sighand->siglock, flags);
+	if (wait) {
+		pcd = &p->real_parent->signal->cdata_wait;
+		tcd = &p->signal->cdata_threads;
+		cd = &p->signal->cdata_wait;
+	} else {
+		pcd = &p->real_parent->signal->cdata_acct;
+		tcd = &p->signal->cdata_threads;
+		cd = &p->signal->cdata_acct;
+	}

We can do this before taking ->siglock. Not that I think this really matters, but otherwise this looks a bit confusing imho, as if we need parent's ->siglock to pin something.

And thanks for splitting these changes. It was much, much easier to read now.

Oleg.

#8 lore raw

Re: [patch 3/4] taskstats: Introduce cdata_acct for complete cumulative accounting

Hello Oleg,

Quoted 7 lines · Oleg Nesterov
Quoted 5 lines · Michael Holzheu

TODO:
-----
With this patch we take the siglock twice. First for the dead task and second for the parent of the dead task. This give the following lockdep warning (probably a lockdep annotation is needed here):

Quoted 5 lines

And we already discussed this ;) We do not need 2 siglock's, only parent's. Just move the callsite in __exit_signal() down, under another (lockless) group_dead check.

Or I missed something?

The problem with moving this down to the second group_dead check is that after __unhash_process() is called, pid_alive(tsk) which is checked in thread_group_cputime() returns false. Therefore we always get zero CPU times.

So I probably have to introduce a second group_dead check at the beginning of __exit_signal():

@@ -150,6 +153,9 @@ static void __exit_signal(struct task_st

        struct sighand_struct *sighand;
        struct tty_struct *uninitialized_var(tty);

+ if (group_dead)
+ __account_cdata(...);
+

        sighand = rcu_dereference_check(tsk->sighand,
                                        rcu_read_lock_held() ||
Quoted 3 lines

We can do this before taking ->siglock. Not that I think this really matters, but otherwise this looks a bit confusing imho, as if we need parent's ->siglock to pin something.

ok

Quoted 4 lines

And thanks for splitting these changes. It was much, much easier to read now.

My personal feeling is that probably the only acceptable thing would be to make the new behavior configurable with a sysctl and define the default as it currently is (POSIX compliant).

This would only introduce two additional checks in __exit_signal() and wait_task_zombie() and would not add any new fields to the signal_struct.

Michael

#9 lore raw

Re: [patch 3/4] taskstats: Introduce cdata_acct for complete cumulative accounting

Quoted 15 lines · Michael Holzheu

Hello Oleg,

Quoted 11 lines · Oleg Nesterov
Quoted 5 lines · Michael Holzheu

TODO:
-----
With this patch we take the siglock twice. First for the dead task and second for the parent of the dead task. This give the following lockdep warning (probably a lockdep annotation is needed here):

And we already discussed this ;) We do not need 2 siglock's, only parent's. Just move the callsite in __exit_signal() down, under another (lockless) group_dead check.

Quoted 6 lines
Quoted 1 line

Or I missed something?

The problem with moving this down to the second group_dead check is that after __unhash_process() is called, pid_alive(tsk) which is checked in thread_group_cputime() returns false. Therefore we always get zero CPU times.

I see, thanks.

Quoted 2 lines

So I probably have to introduce a second group_dead check at the beginning of __exit_signal():

Probably...

But in fact this reminds we should cleanup this code somehow. By the time we call thread_group_times() there are no other threads.

Quoted 7 lines

My personal feeling is that probably the only acceptable thing would be to make the new behavior configurable with a sysctl and define the default as it currently is (POSIX compliant).

This would only introduce two additional checks in __exit_signal() and wait_task_zombie() and would not add any new fields to the signal_struct.

Yeah, it would be nice to avoid new fields.

Hmm. Somehow I forgot about 4/4, please see another email...

Oleg.

#10 lore raw

Re: [patch 3/4] taskstats: Introduce cdata_acct for complete cumulative accounting

Quoted 4 lines · Oleg Nesterov

On 11/25, Michael Holzheu wrote:
But in fact this reminds we should cleanup this code somehow. By the time we call thread_group_times() there are no other threads.

Couldn't we just replace thread_group_times() with the following:

kernel/exit.c +2 -1
@@ -101,12 +101,13 @@ static void __account_cdata(struct task_
         * group, which consolidates times for all threads in the
         * group including the group leader.
         */
-       thread_group_times(p, &tgutime, &tgstime);
        pcd = &p->real_parent->signal->cdata_wait;
        tcd = &p->signal->cdata_threads;
        cd = &p->signal->cdata_wait;
 
        spin_lock_irqsave(&p->real_parent->sighand->siglock, flags);
+       tgutime = cputime_add(tcd->utime, p->utime);
+       tgstime = cputime_add(tcd->stime, p->stime);
        pcd->utime =
                cputime_add(pcd->utime,
                cputime_add(tgutime,

Then we can move the __account_cdata() invocation down to the second group_dead check.

Michael

#11 lore raw

PATCH 4/4 taskstats: Export "cdata_acct" with taskstats

In reply to Michael Holzheu

From: Michael Holzheu <[email protected]>

With this patch the (full) cumulative CPU time is added to "struct taskstats". The CPU time is only returned for the thread group leader.

---
 include/linux/taskstats.h |    7 ++++++-
 kernel/tsacct.c           |    7 +++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
include/linux/taskstats.h +6 -1
@@ -33,7 +33,7 @@
  */
 
 
-#define TASKSTATS_VERSION	7
+#define TASKSTATS_VERSION	8
 #define TS_COMM_LEN		32	/* should be >= TASK_COMM_LEN
 					 * in linux/sched.h */
 
@@ -163,6 +163,11 @@ struct taskstats {
 	/* Delay waiting for memory reclaim */
 	__u64	freepages_count;
 	__u64	freepages_delay_total;
+	/* version 7 ends here */
+
+	/* All cumulative CPU time of dead children */
+	__u64	ac_cutime_acct;		/* User CPU time [usec] */
+	__u64	ac_cstime_acct;		/* System CPU time [usec] */
 };
 
 
kernel/tsacct.c +7 -0
@@ -31,6 +31,7 @@ void bacct_add_tsk(struct taskstats *sta
 	const struct cred *tcred;
 	struct timespec uptime, ts;
 	u64 ac_etime;
+	unsigned long flags;
 
 	BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN);
 
@@ -71,6 +72,12 @@ void bacct_add_tsk(struct taskstats *sta
 	stats->ac_majflt = tsk->maj_flt;
 
 	strncpy(stats->ac_comm, tsk->comm, sizeof(stats->ac_comm));
+	if (tsk->tgid == tsk->pid && lock_task_sighand(tsk, &flags)) {
+		struct cdata *cd = &tsk->signal->cdata_acct;
+		stats->ac_cutime_acct = cputime_to_usecs(cd->utime);
+		stats->ac_cstime_acct = cputime_to_usecs(cd->stime);
+		unlock_task_sighand(tsk, &flags);
+	}
 }
 
 
#12 lore raw

Re: [patch 4/4] taskstats: Export "cdata_acct" with taskstats

Review

Quoted 9 lines · Michael Holzheu

From: Michael Holzheu <[email protected]>

With this patch the (full) cumulative CPU time is added to "struct taskstats". The CPU time is only returned for the thread group leader.

...

(quoted diff) +1 -0
+	if (tsk->tgid == tsk->pid

thread_group_leader() ?

Quoted 1 line

&& lock_task_sighand(tsk, &flags)) {

Do you really need ->siglock? Starting from 2.6.35 it is always safe to access ->signal.

Oleg.

#13 lore raw

Re: [patch 4/4] taskstats: Export "cdata_acct" with taskstats

Hello Oleg,

Quoted 4 lines · Oleg Nesterov
Quoted 3 lines · Michael Holzheu

From: Michael Holzheu <[email protected]>

Quoted 8 lines

Review

Quoted 6 lines

With this patch the (full) cumulative CPU time is added to "struct taskstats". The CPU time is only returned for the thread group leader.

...

(quoted diff) +1 -0
+	if (tsk->tgid == tsk->pid

thread_group_leader() ?

Yes, that's better.

Quoted 4 lines
Quoted 1 line

&& lock_task_sighand(tsk, &flags)) {

Do you really need ->siglock? Starting from 2.6.35 it is always safe to access ->signal.

Hmmm, if you say that...

I just did it like it is done in e.g. fs/proc/base.c (proc_pid_limits). Can we remove the locking there, too?

Michael

#14 lore raw

Re: [patch 4/4] taskstats: Export "cdata_acct" with taskstats

Quoted 21 lines · Michael Holzheu

Hello Oleg,

Quoted 4 lines · Oleg Nesterov
Quoted 3 lines · Michael Holzheu

From: Michael Holzheu <[email protected]>

Quoted 8 lines

Review

Quoted 6 lines

With this patch the (full) cumulative CPU time is added to "struct taskstats". The CPU time is only returned for the thread group leader.

...

(quoted diff) +1 -0
+	if (tsk->tgid == tsk->pid

thread_group_leader() ?

Yes, that's better.

Quoted 2 lines
Quoted 1 line

&& lock_task_sighand(tsk, &flags)) {

Quoted 7 lines
Quoted 2 lines

Do you really need ->siglock? Starting from 2.6.35 it is always safe to access ->signal.

Hmmm, if you say that...

I just did it like it is done in e.g. fs/proc/base.c (proc_pid_limits). Can we remove the locking there, too?

We can certainly remove more siglock's which were previously needed to access ->signal.

This particular one is just wrong. We need task_lock(group_leader) to read signal->rlim atomically. However, it is not trivial to do this correctly. Probably we should ignore this minor problem.

In any case, this ->siglock buys nothing today.

Oleg.

#15 lore raw

Re: [patch 4/4] taskstats: Export "cdata_acct" with taskstats

Hello Oleg,

Quoted 11 lines · Oleg Nesterov
Quoted 2 lines

I just did it like it is done in e.g. fs/proc/base.c (proc_pid_limits). Can we remove the locking there, too?

We can certainly remove more siglock's which were previously needed to access ->signal.

This particular one is just wrong. We need task_lock(group_leader) to read signal->rlim atomically. However, it is not trivial to do this correctly. Probably we should ignore this minor problem.

In any case, this ->siglock buys nothing today.

But at least to get the two values cutime and cstime consistent we need the siglock? There could be a parallel update for tsk->signal->cutime/cstime, while the taskstats are created, no?

Michael

#16 lore raw

Re: [patch 4/4] taskstats: Export "cdata_acct" with taskstats

Quoted 14 lines · Michael Holzheu

Hello Oleg,

Quoted 10 lines · Oleg Nesterov
Quoted 2 lines

I just did it like it is done in e.g. fs/proc/base.c (proc_pid_limits). Can we remove the locking there, too?

We can certainly remove more siglock's which were previously needed to access ->signal.

This particular one is just wrong. We need task_lock(group_leader) to read signal->rlim atomically. However, it is not trivial to do this correctly. Probably we should ignore this minor problem.

Quoted 5 lines
Quoted 1 line

In any case, this ->siglock buys nothing today.

But at least to get the two values cutime and cstime consistent we need the siglock? There could be a parallel update for tsk->signal->cutime/cstime, while the taskstats are created, no?

I guess you mean 4/4 you sent. Probably you are right, I'll try to look tomorrow.

Oleg.

#17 lore raw

Re: [patch 4/4] taskstats: Export "cdata_acct" with taskstats

In reply to Michael Holzheu
Quoted 45 lines · Michael Holzheu

From: Michael Holzheu <[email protected]>

With this patch the (full) cumulative CPU time is added to "struct taskstats". The CPU time is only returned for the thread group leader.

---
 include/linux/taskstats.h |    7 ++++++-
 kernel/tsacct.c           |    7 +++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
include/linux/taskstats.h +6 -1
@@ -33,7 +33,7 @@
  */
 
 
-#define TASKSTATS_VERSION	7
+#define TASKSTATS_VERSION	8
 #define TS_COMM_LEN		32	/* should be >= TASK_COMM_LEN
 					 * in linux/sched.h */
 
@@ -163,6 +163,11 @@ struct taskstats {
 	/* Delay waiting for memory reclaim */
 	__u64	freepages_count;
 	__u64	freepages_delay_total;
+	/* version 7 ends here */
+
+	/* All cumulative CPU time of dead children */
+	__u64	ac_cutime_acct;		/* User CPU time [usec] */
+	__u64	ac_cstime_acct;		/* System CPU time [usec] */
 };
 
 
kernel/tsacct.c +1 -0
@@ -31,6 +31,7 @@ void bacct_add_tsk(struct taskstats *sta
 	const struct cred *tcred;
 	struct timespec uptime, ts;
 	u64 ac_etime;
+	unsigned long flags;
 
 	BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN);
 

kernel/tsacct.c void bacct_add_tsk(struct taskstats *sta · line 75

Quoted 4 lines
kernel/tsacct.c +1 -0
@@ -71,6 +72,12 @@ void bacct_add_tsk(struct taskstats *sta
 	stats->ac_majflt = tsk->maj_flt;
 
 	strncpy(stats->ac_comm, tsk->comm, sizeof(stats->ac_comm));
+	if (tsk->tgid == tsk->pid && lock_task_sighand(tsk, &flags)) {

I don't think referring to tgid and pid is a good idea in the context of namespaces, thread_group_leader makes more sense like Oleg pointed out.

Quoted 9 lines
kernel/tsacct.c +5 -0
+		struct cdata *cd = &tsk->signal->cdata_acct;
+		stats->ac_cutime_acct = cputime_to_usecs(cd->utime);
+		stats->ac_cstime_acct = cputime_to_usecs(cd->stime);
+		unlock_task_sighand(tsk, &flags);
+	}
 }
 
 
Signature · 2 lines
	Three Cheers,
	Balbir
#18 lore raw
In reply to Michael Holzheu
Quoted 10 lines · Michael Holzheu

Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

To solve this problem, this patch set duplicates the cumulative accounting data in the signal_struct. In the second set (cdata_acct) the complete cumulative resource counters are stored. The new cumulative CPU time (utime and stime) is then exported via the taskstats interface.

Maybe this has been treated earlier in the threads and I missed it, but the obvious solution doesn't get mentioned:

What would break if we violate this silly POSIX rule and account time of childs regardless of SIGCHLD/SA_NOCLDWAIT?

#19 lore raw
Quoted 8 lines · Peter Zijlstra
Quoted 6 lines · Michael Holzheu

Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

Quoted 7 lines
Quoted 4 lines

To solve this problem, this patch set duplicates the cumulative accounting data in the signal_struct. In the second set (cdata_acct) the complete cumulative resource counters are stored. The new cumulative CPU time (utime and stime) is then exported via the taskstats interface.

Maybe this has been treated earlier in the threads and I missed it, but the obvious solution doesn't get mentioned:

IIRC, the first version did this.

And it was me who spoiled this approach. But! only because I wasn't sure this user-visible change is acceptable, and because there was some misunderstanding. See http://marc.info/?l=linux-kernel&m=128552495203050&w=2

But,

Quoted 1 line

What would break if we

say, any test-case which does getrusage() after fork() with ignored SIGCHLD/SA_NOCLDWAIT?.

Quoted 2 lines

violate this silly POSIX rule and account time of
childs regardless of SIGCHLD/SA_NOCLDWAIT?

+1.

Personally, I'd certainly prefer this way, because I don't care about POSIX at all ;)

Still. Once again, this breaks the current rules, and we never do this without strong reason.

I think we should ask Roland. If he thinks this is OK, I'd certainly agree.

Oleg.

#20 lore raw
Quoted 39 lines · Oleg Nesterov
Quoted 8 lines · Peter Zijlstra
Quoted 6 lines · Michael Holzheu

Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

Quoted 7 lines
Quoted 4 lines

To solve this problem, this patch set duplicates the cumulative accounting data in the signal_struct. In the second set (cdata_acct) the complete cumulative resource counters are stored. The new cumulative CPU time (utime and stime) is then exported via the taskstats interface.

Maybe this has been treated earlier in the threads and I missed it, but the obvious solution doesn't get mentioned:

IIRC, the first version did this.

And it was me who spoiled this approach. But! only because I wasn't sure this user-visible change is acceptable, and because there was some misunderstanding. See http://marc.info/?l=linux-kernel&m=128552495203050&w=2

But,

Quoted 1 line

What would break if we

say, any test-case which does getrusage() after fork() with ignored SIGCHLD/SA_NOCLDWAIT?.

Quoted 2 lines

violate this silly POSIX rule and account time of
childs regardless of SIGCHLD/SA_NOCLDWAIT?

+1.

Personally, I'd certainly prefer this way, because I don't care about POSIX at all ;)

Quoted 6 lines

Still. Once again, this breaks the current rules, and we never do this without strong reason.

I think we should ask Roland. If he thinks this is OK, I'd certainly agree.

The Linux man page states

In Linux kernel versions before 2.6.9, if the disposition of SIGCHLD is set to SIG_IGN then the resource usages of child processes are automatically included in the value returned by RUSAGE_CHILDREN, although POSIX.1-2001 explicitly prohibits this. This nonconformance is rectified in Linux 2.6.9 and later.

Signature · 2 lines
	Three Cheers,
	Balbir
#21 lore raw
In reply to Peter Zijlstra
Quoted 12 lines · Peter Zijlstra
Quoted 10 lines · Michael Holzheu

Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

To solve this problem, this patch set duplicates the cumulative accounting data in the signal_struct. In the second set (cdata_acct) the complete cumulative resource counters are stored. The new cumulative CPU time (utime and stime) is then exported via the taskstats interface.

Quoted 5 lines

Maybe this has been treated earlier in the threads and I missed it, but the obvious solution doesn't get mentioned:

What would break if we violate this silly POSIX rule and account time of childs regardless of SIGCHLD/SA_NOCLDWAIT?

Or maybe we could add a sysctl that allows to switch between the two semantics.

Michael

#22 lore raw
Quoted 2 lines · Michael Holzheu

Or maybe we could add a sysctl that allows to switch between the two semantics.

Then patch 03/04 would be something like the following:
---
Subject: taskstats: Introduce complete cumulative accounting

From: Michael Holzheu <[email protected]>

Currently the cumulative time accounting in Linux is not complete. Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

This patch adds a new sysctl "kernel.full_cdata" that allows to switch between the POSIX behavior and complete cumulative accounting.

---
 include/linux/sched.h |    1 +
 kernel/exit.c         |   12 ++++++++----
 kernel/sysctl.c       |    7 +++++++
 3 files changed, 16 insertions(+), 4 deletions(-)
include/linux/sched.h +1 -0
@@ -1907,6 +1907,7 @@ enum sched_tunable_scaling {
 };
 extern enum sched_tunable_scaling sysctl_sched_tunable_scaling;
 
+extern unsigned int full_cdata_enabled;
 #ifdef CONFIG_SCHED_DEBUG
 extern unsigned int sysctl_sched_migration_cost;
 extern unsigned int sysctl_sched_nr_migrate;
kernel/exit.c +8 -4
@@ -57,6 +57,8 @@
 #include <asm/pgtable.h>
 #include <asm/mmu_context.h>
 
+unsigned int full_cdata_enabled = 1;
+
 static void exit_mm(struct task_struct * tsk);
 
 static void __unhash_process(struct task_struct *p, bool group_dead)
@@ -77,7 +79,7 @@ static void __unhash_process(struct task
 static void __account_cdata(struct task_struct *p)
 {
 	struct cdata *cd, *pcd, *tcd;
-	unsigned long maxrss;
+	unsigned long maxrss, flags;
 	cputime_t tgutime, tgstime;
 
 	/*
@@ -100,7 +102,7 @@ static void __account_cdata(struct task_
 	 * group including the group leader.
 	 */
 	thread_group_times(p, &tgutime, &tgstime);
-	spin_lock_irq(&p->real_parent->sighand->siglock);
+	spin_lock_irqsave(&p->real_parent->sighand->siglock, flags);
 	pcd = &p->real_parent->signal->cdata_wait;
 	tcd = &p->signal->cdata_threads;
 	cd = &p->signal->cdata_wait;
@@ -137,7 +139,7 @@ static void __account_cdata(struct task_
 		pcd->maxrss = maxrss;
 	task_io_accounting_add(&p->real_parent->signal->ioac, &p->ioac);
 	task_io_accounting_add(&p->real_parent->signal->ioac, &p->signal->ioac);
-	spin_unlock_irq(&p->real_parent->sighand->siglock);
+	spin_unlock_irqrestore(&p->real_parent->sighand->siglock, flags);
 }
 
 /*
@@ -157,6 +159,8 @@ static void __exit_signal(struct task_st
 
 	posix_cpu_timers_exit(tsk);
 	if (group_dead) {
+		if (full_cdata_enabled)
+			__account_cdata(tsk);
 		posix_cpu_timers_exit_group(tsk);
 		tty = sig->tty;
 		sig->tty = NULL;
@@ -1292,7 +1296,7 @@ static int wait_task_zombie(struct wait_
 	 * It can be ptraced but not reparented, check
 	 * !task_detached() to filter out sub-threads.
 	 */
-	if (likely(!traced) && likely(!task_detached(p)))
+	if (likely(!traced) && likely(!task_detached(p)) && !full_cdata_enabled)
 		__account_cdata(p);
 
 	/*
kernel/sysctl.c +7 -0
@@ -963,6 +963,13 @@ static struct ctl_table kern_table[] = {
 		.proc_handler	= proc_dointvec,
 	},
 #endif
+	{
+		.procname	= "full_cdata",
+		.data		= &full_cdata_enabled,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec,
+	},
 /*
  * NOTE: do not add new entries to this table unless you have read
  * Documentation/sysctl/ctl_unnumbered.txt
#23 lore raw
Quoted 11 lines · Michael Holzheu

Currently the cumulative time accounting in Linux is not complete. Due to POSIX POSIX.1-2001, the CPU time of processes is not accounted to the cumulative time of the parents, if the parents ignore SIGCHLD or have set SA_NOCLDWAIT. This behaviour has the major drawback that it is not possible to calculate all consumed CPU time of a system by looking at the current tasks. CPU time can be lost.

This patch adds a new sysctl "kernel.full_cdata" that allows to switch between the POSIX behavior and complete cumulative accounting.

+unsigned int full_cdata_enabled = 1;

This probably needs to default to "current kernel behavior" but allow sysadmins to change to the new behavior.