Skip to contentSkip to conversation

delay accounting taskstats interface send tgid once

RFC PATCH In review Shailabh Nagar 3 participants 5 messages Jun 26, 2006

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 delay accounting taskstats interface send tgid once

Send per-tgid data only once during exit of a thread group instead of once with each member thread exit.

Currently, when a thread exits, besides its per-tid data, the per-tgid data of its thread group is also sent out, if its thread group is non-empty. The per-tgid data sent consists of the sum of per-tid stats for all *remaining* threads of the thread group.

This patch modifies this sending in two ways:

- the per-tgid data is sent only when the last thread of a thread group exits. This cuts down heavily on the overhead of sending/receiving per-tgid data, especially when other exploiters of the taskstats interface aren't interested in per-tgid stats

- the semantics of the per-tgid data sent are changed. Instead of being the sum of per-tid data for remaining threads, the value now sent is the true total accumalated statistics for all threads that are/were part of the thread group.

The patch also addresses a minor issue where failure of one accounting subsystem to fill in the taskstats structure was causing the send of taskstats to not be sent at all.

The patch has been tested for stability and run cerberus for over 4 hours on an SMP.

 Documentation/accounting/delay-accounting.txt |   13 +--
 Documentation/accounting/taskstats.txt        |   33 +++-----
 MAINTAINERS                                   |   12 +++
 include/linux/sched.h                         |    4 +
 include/linux/taskstats_kern.h                |   67 +++++++++++++-----
 kernel/exit.c                                 |    8 +-
 kernel/fork.c                                 |    4 +
 kernel/taskstats.c                            |   96 +++++++++++++++++---------
 8 files changed, 155 insertions(+), 82 deletions(-)
include/linux/sched.h +4 -0
Index: linux-2.6.17/include/linux/sched.h
===================================================================
@@ -449,6 +449,10 @@ struct signal_struct {
 	struct key *session_keyring;	/* keyring inherited over fork */
 	struct key *process_keyring;	/* keyring private to this process */
 #endif
+#ifdef CONFIG_TASKSTATS
+	spinlock_t stats_lock;
+	struct taskstats *stats;
+#endif
 };
 
 /* Context switch must be unlocked if interrupts are to be enabled */
include/linux/taskstats_kern.h +51 -16

87 lines not on this page · raw message

kernel/fork.c +4 -0
Index: linux-2.6.17/kernel/fork.c
===================================================================
@@ -45,6 +45,7 @@
 #include <linux/acct.h>
 #include <linux/cn_proc.h>
 #include <linux/delayacct.h>
+#include <linux/taskstats_kern.h>
 
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
@@ -815,6 +816,7 @@ static inline int copy_signal(unsigned l
 	if (clone_flags & CLONE_THREAD) {
 		atomic_inc(&current->signal->count);
 		atomic_inc(&current->signal->live);
+		taskstats_tgid_alloc(current->signal);
 		return 0;
 	}
 	sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL);
@@ -859,6 +861,7 @@ static inline int copy_signal(unsigned l
 	INIT_LIST_HEAD(&sig->cpu_timers[0]);
 	INIT_LIST_HEAD(&sig->cpu_timers[1]);
 	INIT_LIST_HEAD(&sig->cpu_timers[2]);
+	taskstats_tgid_init(sig);
 
 	task_lock(current->group_leader);
 	memcpy(sig->rlim, current->signal->rlim, sizeof sig->rlim);
@@ -879,6 +882,7 @@ static inline int copy_signal(unsigned l
 void __cleanup_signal(struct signal_struct *sig)
 {
 	exit_thread_group_keys(sig);
+	taskstats_tgid_free(sig);
 	kmem_cache_free(signal_cachep, sig);
 }
 
kernel/taskstats.c +63 -33

155 lines not on this page · raw message

kernel/exit.c +4 -4
Index: linux-2.6.17/kernel/exit.c
===================================================================
@@ -850,7 +850,7 @@ static void exit_notify(struct task_stru
 fastcall NORET_TYPE void do_exit(long code)
 {
 	struct task_struct *tsk = current;
-	struct taskstats *tidstats, *tgidstats;
+	struct taskstats *tidstats;
 	int group_dead;
 
 	profile_task_exit(tsk);
@@ -889,7 +889,7 @@ fastcall NORET_TYPE void do_exit(long co
 				current->comm, current->pid,
 				preempt_count());
 
-	taskstats_exit_alloc(&tidstats, &tgidstats);
+	taskstats_exit_alloc(&tidstats);
 
 	acct_update_integrals(tsk);
 	if (tsk->mm) {
@@ -910,8 +910,8 @@ fastcall NORET_TYPE void do_exit(long co
 #endif
 	if (unlikely(tsk->audit_context))
 		audit_free(tsk);
-	taskstats_exit_send(tsk, tidstats, tgidstats);
-	taskstats_exit_free(tidstats, tgidstats);
+	taskstats_exit_send(tsk, tidstats, group_dead);
+	taskstats_exit_free(tidstats);
 	delayacct_tsk_exit(tsk);
 
 	exit_mm(tsk);
Documentation/accounting/delay-accounting.txt +4 -9
Index: linux-2.6.17/Documentation/accounting/delay-accounting.txt
===================================================================
@@ -48,9 +48,10 @@ counter (say cpu_delay_total) for a task
 experienced by the task waiting for the corresponding resource
 in that interval.
 
-When a task exits, records containing the per-task and per-process statistics
-are sent to userspace without requiring a command. More details are given in
-the taskstats interface description.
+When a task exits, records containing the per-task statistics
+are sent to userspace without requiring a command. If it is the last exiting
+task of a thread group, the per-tgid statistics are also sent. More details
+are given in the taskstats interface description.
 
 The getdelays.c userspace utility in this directory allows simple commands to
 be run and the corresponding delay statistics to be displayed. It also serves
@@ -107,9 +108,3 @@ IO	count	delay total
 	0	0
 MEM	count	delay total
 	0	0
-
-
-
-
-
-
Documentation/accounting/taskstats.txt +13 -20
Index: linux-2.6.17/Documentation/accounting/taskstats.txt
===================================================================
@@ -32,12 +32,11 @@ The response contains statistics for a t
 statistics for all tasks of the process (if tgid is specified).
 
 To obtain statistics for tasks which are exiting, userspace opens a multicast
-netlink socket. Each time a task exits, two records are sent by the kernel to
-each listener on the multicast socket. The first the per-pid task's statistics
-and the second is the sum for all tasks of the process to which the task
-belongs (the task does not need to be the thread group leader). The need for
-per-tgid stats to be sent for each exiting task is explained in the per-tgid
-stats section below.
+netlink socket. Each time a task exits, its per-pid statistics is always sent
+by the kernel to each listener on the multicast socket. In addition, if it is
+the last thread exiting its thread group, an additional record containing the
+per-tgid stats are also sent. The latter contains the sum of per-pid stats for
+all threads in the thread group, both past and present.
 
 getdelays.c is a simple utility demonstrating usage of the taskstats interface
 for reporting delay accounting statistics.
@@ -104,20 +103,14 @@ stats in userspace alone is inefficient
 of atomicity).
 
 However, maintaining per-process, in addition to per-task stats, within the
-kernel has space and time overheads. Hence the taskstats implementation
-dynamically sums up the per-task stats for each task belonging to a process
-whenever per-process stats are needed.
-
-Not maintaining per-tgid stats creates a problem when userspace is interested
-in getting these stats when the process dies i.e. the last thread of
-a process exits. It isn't possible to simply return some aggregated per-process
-statistic from the kernel.
-
-The approach taken by taskstats is to return the per-tgid stats *each* time
-a task exits, in addition to the per-pid stats for that task. Userspace can
-maintain task<->process mappings and use them to maintain the per-process stats
-in userspace, updating the aggregate appropriately as the tasks of a process
-exit.
+kernel has space and time overheads. To address this, the taskstats code
+accumalates each exiting task's statistics into a process-wide data structure.
+When the last task of a process exits, the process level data accumalated also
+gets sent to userspace (along with the per-task data).
+
+When a user queries to get per-tgid data, the sum of all other live threads in
+the group is added up and added to the accumalated total for previously exited
+threads of the same thread group.
 
 Extending taskstats
 -------------------
MAINTAINERS +12 -0
Index: linux-2.6.17/MAINTAINERS
===================================================================
@@ -2185,6 +2185,12 @@ M:	[email protected]
 L:	[email protected]
 S:	Maintained
 
+PER-TASK DELAY ACCOUNTING
+P:	Shailabh Nagar
+M:	[email protected]
+L:	[email protected]
+S:	Maintained
+
 PERSONALITY HANDLING
 P:	Christoph Hellwig
 M:	[email protected]
@@ -2701,6 +2707,12 @@ P:	Christoph Hellwig
 M:	[email protected]
 S:	Maintained
 
+TASKSTATS STATISTICS INTERFACE
+P:	Shailabh Nagar
+M:	[email protected]
+L:	[email protected]
+S:	Maintained
+
 TI PARALLEL LINK CABLE DRIVER
 P:     Romain Lievin
 M:     [email protected]
#1 lore raw

Shailabh,

Is this patch supposed to go on top of all other patches? Or is it supposed to replace any? I had failure applying this patch on top of all previously applied.

- jay

Quoted 28 lines · Shailabh Nagar

Send per-tgid data only once during exit of a thread group instead of once with each member thread exit.

Currently, when a thread exits, besides its per-tid data, the per-tgid data of its thread group is also sent out, if its thread group is non-empty. The per-tgid data sent consists of the sum of per-tid stats for all *remaining* threads of the thread group.

This patch modifies this sending in two ways:

- the per-tgid data is sent only when the last thread of a thread group exits. This cuts down heavily on the overhead of sending/receiving per-tgid data, especially when other exploiters of the taskstats interface aren't interested in per-tgid stats

- the semantics of the per-tgid data sent are changed. Instead of being the sum of per-tid data for remaining threads, the value now sent is the true total accumalated statistics for all threads that are/were part of the thread group.

The patch also addresses a minor issue where failure of one accounting subsystem to fill in the taskstats structure was causing the send of taskstats to not be sent at all.

The patch has been tested for stability and run cerberus for over 4 hours on an SMP.

#2 lore raw
Quoted 5 lines · Jay Lan

Shailabh,

Is this patch supposed to go on top of all other patches? Or is it supposed to replace any? I had failure applying this patch on top of all previously applied.

This is my series file (effectively). So it should have applied on top of the new lock patch (which is now integrated into -mm2). But I'll send out another one that's explicitly on top of the -mm2 patchset. Since we've been privately exchanging patches, the series wasn't clear.

--Shailabh

Series for this patch:

#********** 2.6.17-mm1 patchset, ported to 2.6.17, start
per-task-delay-accounting-setup.patch
per-task-delay-accounting-setup-fix-1.patch
per-task-delay-accounting-setup-fix-2.patch per-task-delay-accounting-sync-block-i-o-and-swapin-delay-collection.patch per-task-delay-accounting-sync-block-i-o-and-swapin-delay-collection-fix-1.patch per-task-delay-accounting-cpu-delay-collection-via-schedstats.patch per-task-delay-accounting-cpu-delay-collection-via-schedstats-fix-1.patch per-task-delay-accounting-utilities-for-genetlink-usage.patch per-task-delay-accounting-taskstats-interface.patch per-task-delay-accounting-taskstats-interface-fix-1.patch per-task-delay-accounting-taskstats-interface-fix-2.patch per-task-delay-accounting-taskstats-interface-fix-exit-race-in-per-task-delay-accounting.patch per-task-delay-accounting-delay-accounting-usage-of-taskstats-interface.patch per-task-delay-accounting-delay-accounting-usage-of-taskstats-interface-use-portable-cputime-api-in-__delayacct_add_tsk.patch per-task-delay-accounting-delay-accounting-usage-of-taskstats-interface-fix-return-value-of-delayacct_add_tsk.patch per-task-delay-accounting-documentation.patch per-task-delay-accounting-proc-export-of-aggregated-block-i-o-delays.patch per-task-delay-accounting-proc-export-of-aggregated-block-i-o-delays-warning-fix.patch
#*********** 2.6.17-mm1 patchset, ported to 2.6.17, end taskstats-interface-revised-exit-race-locking.patch
taskstats-interface-send-tgid-once.patch

#3 lore raw
In reply to Jay Lan
Quoted 3 lines · On Mon, 26 Jun 2006 12:04:16 -0700 Jay Lan <[email protected]> wrote:

Is this patch supposed to go on top of all other patches? Or is it supposed to replace any? I had failure applying this patch on top of all previously applied.

It would have got tangled up with the task-watchers patches.

ahem. Documentation/SubmitChecklist, item 2:

In file included from kernel/exit.c:29:
include/linux/taskstats_kern.h: In function 'taskstats_exit_send':
include/linux/taskstats_kern.h:80: error: parameter name omitted In file included from include/linux/delayacct.h:21,

                 from kernel/fork.c:47:

include/linux/taskstats_kern.h: In function 'taskstats_exit_send':
include/linux/taskstats_kern.h:80: error: parameter name omitted make[1]: *** [kernel/exit.o] Error 1
make: *** [kernel/exit.o] Error 2
make: *** Waiting for unfinished jobs....
make[1]: *** [kernel/fork.o] Error 1
make: *** [kernel/fork.o] Error 2

include/linux/taskstats_kern.h renamed from include/linux/taskstats_kern.h~delay-accounting-taskstats-interface-send-tgid-once-fixes +2 -1
diff -puN include/linux/taskstats_kern.h~delay-accounting-taskstats-interface-send-tgid-once-fixes include/linux/taskstats_kern.h
@@ -77,7 +77,8 @@ static inline void taskstats_exit_alloc(
 static inline void taskstats_exit_free(struct taskstats *ptidstats)
 {}
 static inline void taskstats_exit_send(struct task_struct *tsk,
-				       struct taskstats *tidstats, int)
+				       struct taskstats *tidstats,
+				       int group_dead)
 {}
 static inline void taskstats_tgid_init(struct signal_struct *sig)
 {}

_

http://www.zip.com.au/~akpm/linux/patches/stuff/x.bz2 contains the current -mm rollup up to and including this patch. It's against 2.6.17.

#4 lore raw
Quoted 21 lines · Andrew Morton
Quoted 5 lines · On Mon, 26 Jun 2006 12:04:16 -0700 Jay Lan <[email protected]> wrote:

Is this patch supposed to go on top of all other patches? Or is it supposed to replace any? I had failure applying this patch on top of all previously applied.

It would have got tangled up with the task-watchers patches.

ahem. Documentation/SubmitChecklist, item 2:

In file included from kernel/exit.c:29:
include/linux/taskstats_kern.h: In function 'taskstats_exit_send':
include/linux/taskstats_kern.h:80: error: parameter name omitted In file included from include/linux/delayacct.h:21,

Quoted 10 lines
                from kernel/fork.c:47:

include/linux/taskstats_kern.h: In function 'taskstats_exit_send':
include/linux/taskstats_kern.h:80: error: parameter name omitted make[1]: *** [kernel/exit.o] Error 1
make: *** [kernel/exit.o] Error 2
make: *** Waiting for unfinished jobs....
make[1]: *** [kernel/fork.o] Error 1
make: *** [kernel/fork.o] Error 2

Oops. Thanks for the catch. It didn't get flagged on my compiles for some reason.