new CSA patchset for 2.6.8

PATCH In review Jay Lan 8 participants 27 messages Aug 26, 2004 – Sep 8, 2004 3 messages not in the archive

Subsystems (9)

Memory management VFS Block layer MM core +5 more subsystems

lore

Share & tools

lore

Jay Lan

author

2004-08-26 00:29 UTC #0 lore raw

PATCH 1/1 new CSA patchset for 2.6.8

Hi,

I have broken up one big CSA kernel patch into four smaller ones as attached:

    csa_io     - collects io accounting data
    csa_mm     - collects mm accounting data
    csa_eop    - provides a hook to perform end-of-process accounting

csa_module - builds csa loadable module

There are no functional changes in this set of csa patches compared to the 2.6.7 patch linux-2.6.7.csa.patch.

Patches csa_io, csa_mm, and csa_eop are independent of each other. You may apply any one, any two or all three and you will be able to build a functional kernel. However, data collected needs an agent to use it. The csa_module is one agent that takes advangtage of the feature and it works with csa-2.0.0 (or later) to report system accounting data of the host system. The csa-2.0.0 rpm can be downloaded from ftp://oss.sgi.com/projects/csa/download

The csa_module patch requires all three accounting data patches to be fully functional.

This set of csa patches has been tested with the pagg and job kernel patches to linux 2.6.8 kernel. The information of pagg and job project can be found at http://oss.sgi.com/projects/pagg/

The csa_module requires the pagg and job kernel patches.

Feedback, bug reports, and comments are very welcome!

Signed-off-by Jay Lan <[email protected]>

---
Jay Lan - Linux System Software
Silicon Graphics Inc., Mountain View, CA

drivers/block/ll_rw_blk.c +4 -0

Index: linux/drivers/block/ll_rw_blk.c
===================================================================
@@ -1674,6 +1674,7 @@
 {
 	DEFINE_WAIT(wait);
 	struct request *rq;
+	unsigned long start_wait = jiffies;
 
 	generic_unplug_device(q);
 	do {

@@ -1702,6 +1703,7 @@
 		finish_wait(&rl->wait[rw], &wait);
 	} while (!rq);
 
+	current->bwtime += (unsigned long) jiffies - start_wait;
 	return rq;
 }

@@ -1948,10 +1950,12 @@
 
 	if (rw == READ) {
 		disk_stat_add(rq->rq_disk, read_sectors, nr_sectors);
+		current->rblk += nr_sectors;
 		if (!new_io)
 			disk_stat_inc(rq->rq_disk, read_merges);
 	} else if (rw == WRITE) {
 		disk_stat_add(rq->rq_disk, write_sectors, nr_sectors);
+		current->wblk += nr_sectors;
 		if (!new_io)
 			disk_stat_inc(rq->rq_disk, write_merges);
 	}

fs/read_write.c +23 -2

Index: linux/fs/read_write.c
===================================================================
@@ -216,8 +216,11 @@
 				ret = file->f_op->read(file, buf, count, pos);
 			else
 				ret = do_sync_read(file, buf, count, pos);
-			if (ret > 0)
+			if (ret > 0) {
 				dnotify_parent(file->f_dentry, DN_ACCESS);
+				current->rchar += ret;
+			}
+			current->syscr++;
 		}
 	}

@@ -260,8 +263,11 @@
 				ret = file->f_op->write(file, buf, count, pos);
 			else
 				ret = do_sync_write(file, buf, count, pos);
-			if (ret > 0)
+			if (ret > 0) {
 				dnotify_parent(file->f_dentry, DN_MODIFY);
+				current->wchar += ret;
+			}
+			current->syscw++;
 		}
 	}

@@ -540,6 +546,10 @@
 		fput_light(file, fput_needed);
 	}
 
+	if (ret > 0) {
+		current->rchar += ret;
+	}
+	current->syscr++;
 	return ret;
 }

@@ -558,6 +568,10 @@
 		fput_light(file, fput_needed);
 	}
 
+	if (ret > 0) {
+		current->wchar += ret;
+	}
+	current->syscw++;
 	return ret;
 }

@@ -636,6 +650,13 @@
 
 	retval = in_file->f_op->sendfile(in_file, ppos, count, file_send_actor, out_file);
 
+	if (retval > 0) {
+		current->rchar += retval;
+		current->wchar += retval;
+	}
+	current->syscr++;
+	current->syscw++;
+
 	if (*ppos > max)
 		retval = -EOVERFLOW;

kernel/fork.c +2 -0

Index: linux/kernel/fork.c
===================================================================
@@ -966,6 +966,8 @@
 
 	p->utime = p->stime = 0;
 	p->cutime = p->cstime = 0;
+	p->rchar = p->wchar = p->rblk = p->wblk = p->syscr = p->syscw = 0;
+	p->bwtime = 0;
 	p->lock_depth = -1;		/* -1 = no lock */
 	p->start_time = get_jiffies_64();
 	p->security = NULL;

include/linux/sched.h +3 -0

Index: linux/include/linux/sched.h
===================================================================
@@ -523,6 +523,9 @@
 	unsigned long ptrace_message;
 	siginfo_t *last_siginfo; /* For ptrace use.  */
 
+/* i/o counters(bytes read/written, blocks read/written, #syscalls, waittime */
+        unsigned long rchar, wchar, rblk, wblk, syscr, syscw, bwtime;
+
 #ifdef CONFIG_NUMA
   	struct mempolicy *mempolicy;
   	short il_next;		/* could be shared with used_math */

fs/exec.c +4 -0

Index: linux/fs/exec.c
===================================================================
@@ -47,6 +47,7 @@
 #include <linux/syscalls.h>
 #include <linux/rmap.h>
 #include <linux/pagg.h>
+#include <linux/csa_internal.h>
 
 #include <asm/uaccess.h>
 #include <asm/mmu_context.h>

@@ -1145,6 +1146,9 @@
 
 		/* execve success */
 		security_bprm_free(&bprm);
+		/* no-op if CONFIG_CSA not set */
+                csa_update_integrals();
+                update_mem_hiwater();
 		return retval;
 	}

include/linux/csa_internal.h new file +70 -0

70 lines not on this page · raw message

include/linux/sched.h +19 -0

Index: linux/include/linux/sched.h
===================================================================
@@ -229,6 +229,8 @@
 	struct kioctx		*ioctx_list;
 
 	struct kioctx		default_kioctx;
+
+	unsigned long hiwater_rss, hiwater_vm;
 };
 
 extern int mmlist_nr;

@@ -525,6 +527,10 @@
 
 /* i/o counters(bytes read/written, blocks read/written, #syscalls, waittime */
         unsigned long rchar, wchar, rblk, wblk, syscr, syscw, bwtime;
+#if defined(CONFIG_CSA) || defined(CONFIG_CSA_MODULE)
+	unsigned long csa_rss_mem1, csa_vm_mem1;
+	clock_t csa_stimexpd;
+#endif
 
 #ifdef CONFIG_NUMA
   	struct mempolicy *mempolicy;

@@ -873,6 +879,19 @@
 /* Remove the current tasks stale references to the old mm_struct */
 extern void mm_release(struct task_struct *, struct mm_struct *);
 
+/* Update highwater values */
+static inline void update_mem_hiwater(void)
+{
+	if (current->mm) {
+		if (current->mm->hiwater_rss < current->mm->rss) {
+			current->mm->hiwater_rss = current->mm->rss;
+		}
+		if (current->mm->hiwater_vm < current->mm->total_vm) {
+			current->mm->hiwater_vm = current->mm->total_vm;
+		}
+	}
+}
+
 extern int  copy_thread(int, unsigned long, unsigned long, unsigned long, struct task_struct *, struct pt_regs *);
 extern void flush_thread(void);
 extern void exit_thread(void);

kernel/exit.c +4 -0

Index: linux/kernel/exit.c
===================================================================
@@ -23,6 +23,7 @@
 #include <linux/mount.h>
 #include <linux/proc_fs.h>
 #include <linux/mempolicy.h>
+#include <linux/csa_internal.h>
 
 #include <asm/uaccess.h>
 #include <asm/unistd.h>

@@ -820,6 +821,9 @@
 		ptrace_notify((PTRACE_EVENT_EXIT << 8) | SIGTRAP);
 	}
 
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
+	update_mem_hiwater();
 	acct_process(code);
 	__exit_mm(tsk);

kernel/fork.c +6 -1

Index: linux/kernel/fork.c
===================================================================
@@ -37,7 +37,7 @@
 #include <linux/audit.h>
 #include <linux/rmap.h>
 #include <linux/pagg.h>
-
+#include <linux/csa_internal.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/uaccess.h>

@@ -578,6 +578,9 @@
 	if (retval)
 		goto free_pt;
 
+	mm->hiwater_rss = mm->rss;
+	mm->hiwater_vm = mm->total_vm;	
+
 good_mm:
 	tsk->mm = mm;
 	tsk->active_mm = mm;

@@ -968,6 +971,8 @@
 	p->cutime = p->cstime = 0;
 	p->rchar = p->wchar = p->rblk = p->wblk = p->syscr = p->syscw = 0;
 	p->bwtime = 0;
+	/* no-op if CONFIG_CSA not set */
+	csa_clear_integrals(p);
 	p->lock_depth = -1;		/* -1 = no lock */
 	p->start_time = get_jiffies_64();
 	p->security = NULL;

mm/memory.c +19 -2

Index: linux/mm/memory.c
===================================================================
@@ -44,6 +44,7 @@
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
 #include <linux/rmap.h>
+#include <linux/csa_internal.h>
 #include <linux/module.h>
 #include <linux/init.h>

@@ -605,6 +606,8 @@
 	tlb = tlb_gather_mmu(mm, 0);
 	unmap_vmas(&tlb, mm, vma, address, end, &nr_accounted, details);
 	tlb_finish_mmu(tlb, address, end);
+	/* no-op unless CONFIG_CSA is set */
+        csa_update_integrals();
 	spin_unlock(&mm->page_table_lock);
 }

@@ -1095,9 +1098,12 @@
 	spin_lock(&mm->page_table_lock);
 	page_table = pte_offset_map(pmd, address);
 	if (likely(pte_same(*page_table, pte))) {
-		if (PageReserved(old_page))
+		if (PageReserved(old_page)) {
 			++mm->rss;
-		else
+			/* no-op if CONFIG_CSA not set */
+			csa_update_integrals();
+			update_mem_hiwater();
+		} else
 			page_remove_rmap(old_page);
 		break_cow(vma, new_page, address, page_table);
 		lru_cache_add_active(new_page);

@@ -1378,6 +1384,10 @@
 		remove_exclusive_swap_page(page);
 
 	mm->rss++;
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
+	update_mem_hiwater();
+  
 	pte = mk_pte(page, vma->vm_page_prot);
 	if (write_access && can_share_swap_page(page)) {
 		pte = maybe_mkwrite(pte_mkdirty(pte), vma);

@@ -1443,6 +1453,9 @@
 			goto out;
 		}
 		mm->rss++;
+		/* no-op if CONFIG_CSA not set */
+		csa_update_integrals();
+		update_mem_hiwater();
 		entry = maybe_mkwrite(pte_mkdirty(mk_pte(page,
 							 vma->vm_page_prot)),
 				      vma);

@@ -1552,6 +1565,10 @@
 	if (pte_none(*page_table)) {
 		if (!PageReserved(new_page))
 			++mm->rss;
+		/* no-op if CONFIG_CSA not set */
+		csa_update_integrals();
+		update_mem_hiwater();
+
 		flush_icache_page(vma, new_page);
 		entry = mk_pte(new_page, vma->vm_page_prot);
 		if (write_access)

mm/mmap.c +10 -0

Index: linux/mm/mmap.c
===================================================================
@@ -20,6 +20,7 @@
 #include <linux/hugetlb.h>
 #include <linux/profile.h>
 #include <linux/module.h>
+#include <linux/csa_internal.h>
 #include <linux/mount.h>
 #include <linux/mempolicy.h>
 #include <linux/rmap.h>

@@ -988,6 +989,9 @@
 					pgoff, flags & MAP_NONBLOCK);
 		down_write(&mm->mmap_sem);
 	}
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
+	update_mem_hiwater();
 	return addr;
 
 unmap_and_free_vma:

@@ -1227,6 +1231,9 @@
 	vma->vm_mm->total_vm += grow;
 	if (vma->vm_flags & VM_LOCKED)
 		vma->vm_mm->locked_vm += grow;
+	/* no-op if CONFIG_CSA_JOB_ACCT not set */
+	csa_update_integrals();
+	update_mem_hiwater();
 	anon_vma_unlock(vma);
 	return 0;
 }

@@ -1688,6 +1695,9 @@
 		mm->locked_vm += len >> PAGE_SHIFT;
 		make_pages_present(addr, addr + len);
 	}
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
+	update_mem_hiwater();
 	return addr;
 }

mm/mremap.c +8 -0

Index: linux/mm/mremap.c
===================================================================
@@ -16,6 +16,7 @@
 #include <linux/fs.h>
 #include <linux/highmem.h>
 #include <linux/security.h>
+#include <linux/csa_internal.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>

@@ -231,6 +232,10 @@
 					   new_addr + new_len);
 	}
 
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
+	update_mem_hiwater();
+
 	return new_addr;
 }

@@ -363,6 +368,9 @@
 				make_pages_present(addr + old_len,
 						   addr + new_len);
 			}
+			/* no-op if CONFIG_CSA not set */
+			csa_update_integrals();
+			update_mem_hiwater();
 			ret = addr;
 			goto out;
 		}

mm/rmap.c +5 -0

Index: linux/mm/rmap.c
===================================================================
@@ -29,6 +29,7 @@
 #include <linux/swapops.h>
 #include <linux/slab.h>
 #include <linux/init.h>
+#include <linux/csa_internal.h>
 #include <linux/rmap.h>
 
 #include <asm/tlbflush.h>

@@ -515,6 +516,8 @@
 	mm->rss--;
 	BUG_ON(!page->mapcount);
 	page->mapcount--;
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
 	page_cache_release(page);
 
 out_unmap:

@@ -614,6 +617,8 @@
 
 		page_remove_rmap(page);
 		page_cache_release(page);
+		/* no-op if CONFIG_CSA not set */
+		csa_update_integrals();
 		mm->rss--;
 		(*mapcount)--;
 	}

mm/swapfile.c +4 -0

Index: linux/mm/swapfile.c
===================================================================
@@ -24,6 +24,7 @@
 #include <linux/module.h>
 #include <linux/rmap.h>
 #include <linux/security.h>
+#include <linux/csa_internal.h>
 #include <linux/backing-dev.h>
 
 #include <asm/pgtable.h>

@@ -435,6 +436,9 @@
 	set_pte(dir, pte_mkold(mk_pte(page, vma->vm_page_prot)));
 	page_add_anon_rmap(page, vma, address);
 	swap_free(entry);
+	/* no-op if CONFIG_CSA not set */
+	csa_update_integrals();
+	update_mem_hiwater();
 }
 
 /* vma->vm_mm->page_table_lock is held */

kernel/exit.c +5 -0

Index: linux/kernel/exit.c
===================================================================
@@ -32,6 +32,8 @@
 
 extern void sem_exit (void);
 extern struct task_struct *child_reaper;
+void (*do_eop_acct) (int, struct task_struct *) = NULL;
+EXPORT_SYMBOL(do_eop_acct);
 
 int getrusage(struct task_struct *, int, struct rusage __user *);

@@ -825,6 +827,9 @@
 	csa_update_integrals();
 	update_mem_hiwater();
 	acct_process(code);
+	/* Handle end-of-process accounting */
+	if (do_eop_acct != NULL)
+		do_eop_acct(code, tsk);
 	__exit_mm(tsk);
 
 	exit_sem(tsk);

include/linux/acct_eop.h new file +38 -0

Index: linux/include/linux/acct_eop.h
===================================================================
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2004 Silicon Graphics, Inc All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it would be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+ *
+ * Contact information:  Silicon Graphics, Inc., 1500 Crittenden Lane,
+ * Mountain View, CA  94043, or:
+ *
+ * http://www.sgi.com
+ */
+
+/*
+ *  End-of-process Accounting for Linux
+ *
+ *  This header file contains the hook needed for processing of 
+ *  end-of-process accounting record
+ *
+ */
+
+#ifndef _LINUX_ACCT_EOP_H
+#define _LINUX_ACCT_EOP_H
+
+
+extern void (*do_eop_acct) (int, struct task_struct *);
+
+#endif	/* _LINUX_CSA_INTERNAL_H */

include/linux/csa.h new file +526 -0

526 lines not on this page · raw message

kernel/Makefile +1 -0

Index: linux/kernel/Makefile
===================================================================
@@ -20,6 +20,7 @@
 obj-$(CONFIG_COMPAT) += compat.o
 obj-$(CONFIG_PAGG) += pagg.o
 obj-$(CONFIG_PAGG_JOB) += job.o
+obj-$(CONFIG_CSA) += csa.o
 obj-$(CONFIG_IKCONFIG) += configs.o
 obj-$(CONFIG_IKCONFIG_PROC) += configs.o
 obj-$(CONFIG_STOP_MACHINE) += stop_machine.o

kernel/csa.c new file +1665 -0

1,665 lines not on this page · raw message

init/Kconfig +24 -0

Index: linux/init/Kconfig
===================================================================
@@ -158,6 +158,30 @@
 	  a module, select this entry using M.  If you do not want support
 	  for jobs, select N.
 
+config CSA
+	tristate "  CSA Job Accounting"
+	depends on PAGG_JOB
+	help
+	  Comprehensive System Accounting (CSA) provides job level
+	  accounting of resource usage.  The accounting records are
+	  written by the kernel into a file.  CSA user level scripts
+	  and commands process the binary accounting records and
+	  combine them by job identifier within system boot uptime
+	  periods.  These accounting records are then used to produce
+	  reports and charge fees to users.
+
+	  Say Y here if you want job level accounting to be compiled
+	  into the kernel.  Say M here if you want the writing of
+	  accounting records portion of this feature to be a loadable
+	  module.  Say N here if you do not want job level accounting
+	  (the default).
+	
+	  The CSA commands and scripts package needs to be installed
+	  to process the CSA accounting records.  See
+	  http://oss.sgi.com/projects/csa for further information
+	  about CSA and download instructions for the CSA commands
+	  package and documentation.
+
 config SYSCTL
 	bool "Sysctl support"
 	---help---

Andrew Morton

2004-08-26 05:22 UTC #1 lore raw

Quoted 8 lines · Jay Lan

I have broken up one big CSA kernel patch into four smaller ones
as attached:
      csa_io     - collects io accounting data
      csa_mm     - collects mm accounting data
      csa_eop    - provides a hook to perform end-of-process accounting
csa_module - builds csa loadable module

Please don't send patches as attachments, and please don't send more than one patch per email.

If you stick to the one-patch-per-email rule, you also get an opportunity to nicely describe each patch within its email.

Coding style nit:

	if (foo)
		bar();

is preferable to

	if (foo) {
		bar();
	}

More broadly: Help!

I am 100% not in a position to judge whether Linux needs Comprehensive System Accounting, nor am I able to define what the requirements for such a thing should be. All I can tell from your patch is the quality of its implementation, and that's leaping far, far ahead of where we should be.

We're going to need help from you, and from all the other stakeholders in judging how useful this feature is to Linux implementors and how well this implementation meets the (unknown) requirements. See my problem?

I've cc'ed lse-tech, where enterprise folks hang out. I would request that the people who are stakeholders in this feature

a) stick their hands up

b) let us know how important this kind of feature is for their users

c) review the offered feature set against their requirements

d) let us know how well the implementation fits that requirement and

e) inform us of any competing implementations. Compare and contrast.

Thanks.

Quoted 22 lines

There are no functional changes in this set of csa patches compared to the 2.6.7 patch linux-2.6.7.csa.patch.
Patches csa_io, csa_mm, and csa_eop are independent of each other. You may apply any one, any two or all three and you will be able to build a functional kernel. However, data collected needs an agent to use it. The csa_module is one agent that takes advangtage of the feature and it works with csa-2.0.0 (or later) to report system accounting data of the host system. The csa-2.0.0 rpm can be downloaded from ftp://oss.sgi.com/projects/csa/download
The csa_module patch requires all three accounting data patches to be fully functional.
This set of csa patches has been tested with the pagg and job kernel patches to linux 2.6.8 kernel. The information of pagg and job project can be found at http://oss.sgi.com/projects/pagg/
The csa_module requires the pagg and job kernel patches.
Feedback, bug reports, and comments are very welcome!

Tim Schmielau

2004-08-26 17:19 UTC #2 lore raw

Quoted 16 lines · Andrew Morton

More broadly: Help!
I am 100% not in a position to judge whether Linux needs Comprehensive System Accounting, nor am I able to define what the requirements for such a thing should be. All I can tell from your patch is the quality of its implementation, and that's leaping far, far ahead of where we should be.
We're going to need help from you, and from all the other stakeholders in judging how useful this feature is to Linux implementors and how well this implementation meets the (unknown) requirements. See my problem?
I've cc'ed lse-tech, where enterprise folks hang out. I would request that the people who are stakeholders in this feature
a) stick their hands up

Quoted 7 lines

b) let us know how important this kind of feature is for their users
c) review the offered feature set against their requirements
d) let us know how well the implementation fits that requirement and
e) inform us of any competing implementations. Compare and contrast.

Judging from the feedback during it's stay in -mm (none at all!), general interest in BSD accounting seems quite limited. The rate of downloads of the updated userspace tools is hardly distinguishable from background noise. (This might change with the correct URL in the help text now, but even that was broken for months and nobody cared).
Also general interest in the user space tools is low, the latest release of the GNU acct package is from 1998 (and yes, there _are_ problems warranting updates).

Funnily enough, with three competing implementation even interest from developers seems larger than that from users (This statement includes me, I did a patch but am not a user of it).) But communication between developers is poor. I for myself only recently learned about ELSA and CSA.

Therefore I've Cc:ed some people from whom I got valuable feedback on the BSD accounting format patch.

IMHO CSA, ELSA and BSD accounting are too similar to have more than one of them in the kernel. We should either improve BSD accounting to do the job, or kill it in favor of a different implementation.

Tim

Jay Lan

author

2004-08-26 18:38 UTC #3 lore raw

The reason for breaking up one CSA patch into four patches was so that the only CSA (http://oss.sgi.com/projects/csa/) specific thing is the csa_module. My intention is to improve the system accounting data collection and make the data available to any clients that can use the data. The three areas of accounting data we try to improve are io, mm, and per-process area.

As Tim said the problem of BSD accounting was that it has been inactive for a long time. I do not mind incoporating the three accounting data collection patches i submitted into BSD or others as long as the data made available to modules that plan to make use of the data. :)

Thanks,
- jay

Quoted 57 lines · Tim Schmielau

Quoted 16 lines · Andrew Morton
More broadly: Help!
I am 100% not in a position to judge whether Linux needs Comprehensive System Accounting, nor am I able to define what the requirements for such a thing should be. All I can tell from your patch is the quality of its implementation, and that's leaping far, far ahead of where we should be.
We're going to need help from you, and from all the other stakeholders in judging how useful this feature is to Linux implementors and how well this implementation meets the (unknown) requirements. See my problem?
I've cc'ed lse-tech, where enterprise folks hang out. I would request that the people who are stakeholders in this feature
a) stick their hands up
Quoted 7 lines
b) let us know how important this kind of feature is for their users
c) review the offered feature set against their requirements
d) let us know how well the implementation fits that requirement and
e) inform us of any competing implementations. Compare and contrast.
Judging from the feedback during it's stay in -mm (none at all!), general
interest in BSD accounting seems quite limited. The rate of downloads of
the updated userspace tools is hardly distinguishable from background
noise. (This might change with the correct URL in the help text now, but
even that was broken for months and nobody cared).
Also general interest in the user space tools is low, the latest release of the GNU acct package is from 1998 (and yes, there _are_ problems warranting updates).
Funnily enough, with three competing implementation even interest from developers seems larger than that from users (This statement includes me, I did a patch but am not a user of it).) But communication between developers is poor. I for myself only recently learned about ELSA and CSA.
Therefore I've Cc:ed some people from whom I got valuable feedback on the BSD accounting format patch.
IMHO CSA, ELSA and BSD accounting are too similar to have more than one of them in the kernel. We should either improve BSD accounting to do the job, or kill it in favor of a different implementation.
Tim
-------------------------------------------------------
SF.Net email is sponsored by Shop4tech.com-Lowest price on Blank Media 100pk Sonic DVD-R 4x for only $29 -100pk Sonic DVD+R for only $33 Save 50% off Retail on Ink & Toner - Free Shipping and Free Gift.
http://www.shop4tech.com/z/Inkjet_Cartridges/9_108_r285
_______________________________________________
Lse-tech mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/lse-tech

Arthur Corliss

2004-08-26 19:49 UTC #4 lore raw

Quoted 7 lines · Jay Lan

The reason for breaking up one CSA patch into four patches was so that the only CSA (http://oss.sgi.com/projects/csa/) specific thing is the csa_module. My intention is to improve the system accounting data collection and make the data available to any clients that can use the data. The three areas of accounting data we try to improve are io, mm, and per-process area.

Quoted 5 lines

As Tim said the problem of BSD accounting was that it has been inactive for a long time. I do not mind incoporating the three accounting data collection patches i submitted into BSD or others as long as the data made available to modules that plan to make use of the data. :)

All right, I'm going to shut up now. I had no idea that SGI had gone behind my back and started porting CSA to Linux. :-P Before I shut up, though, as a user of these tools I'd vote for a unified data collection method, as suggested above. There has to be at least five of us who have volunteered at one point or another to help make the GNU utilities conform. That should make everyone happy.

--Arthur Corliss
Bolverk's Lair -- http://arthur.corlissfamily.org/
Digital Mages -- http://www.digitalmages.com/
"Live Free or Die, the Only Way to Live" -- NH State Motto

Arthur Corliss

2004-08-26 19:34 UTC #5 lore raw

In reply to Tim Schmielau

On Thu, 26 Aug 2004, Tim Schmielau wrote:

<snip>

Quoted 6 lines

Therefore I've Cc:ed some people from whom I got valuable feedback on the BSD accounting format patch.
IMHO CSA, ELSA and BSD accounting are too similar to have more than one of them in the kernel. We should either improve BSD accounting to do the job, or kill it in favor of a different implementation.

I would be very interested in a CSA implementation similar to what I have on IRIX. I will also plead guilty to not having downloaded the updated patches for either the kernel or the tools. I'm continuing to use my poor hack until a permanent solution gets accepted into the kernel, at which point I'll adopt that.

And if it counters the impression at all, I'm not a kernel developer, I proposed my hack out of need as a user of the tools. I also try to stay away from modified kernels, so I'm running Marcelos' 2.4 stable branch with only the 32bit u/gid_t hack applied. That's why I haven't had any feedback on the -mm branch.

In short, for my use BSD accounting is sufficient, but I'd love to see CSA in Linux as well. Linux hasn't moved too far into roles where it's a necessity (for what I'm doing, anyway), but I see CSA as something that would certainly help it assume those roles.

--Arthur Corliss
Bolverk's Lair -- http://arthur.corlissfamily.org/
Digital Mages -- http://www.digitalmages.com/
"Live Free or Die, the Only Way to Live" -- NH State Motto

Tim Schmielau

2004-08-26 20:13 UTC #6 lore raw

Quoted 5 lines · Arthur Corliss

I would be very interested in a CSA implementation similar to what I have on IRIX. I will also plead guilty to not having downloaded the updated patches for either the kernel or the tools. I'm continuing to use my poor hack until a permanent solution gets accepted into the kernel, at which point I'll adopt that.

That's ok, we carefully discussed the changes to make sure no new tools are required ;-)

Quoted 5 lines

And if it counters the impression at all, I'm not a kernel developer, I proposed my hack out of need as a user of the tools. I also try to stay away from modified kernels, so I'm running Marcelos' 2.4 stable branch with only the 32bit u/gid_t hack applied. That's why I haven't had any feedback on the -mm branch.

I haven't even tried to get a patch into 2.4, since Marcelo is (rightly) quite resilent to new features.

Quoted 5 lines

In short, for my use BSD accounting is sufficient, but I'd love to see CSA in Linux as well. Linux hasn't moved too far into roles where it's a necessity (for what I'm doing, anyway), but I see CSA as something that would certainly help it assume those roles.

Does this mean you would want to have both in the same kernel, potentially turning on both at the same time?

Ok, let me summarize what I learned until now:

It should be easy to combine the data collection enhancements from CSA and ELSA to provide a common superset of information.

Output file formats vary, but might be unified if projects don't insist too much.
Main difference between CSA and ELSA on the one hand and BSD acct on the other is that the latter writes one record per process, while the former write one per job.
With the new BSD acct v3 format, it should be possible to do per job accounting entirely from userspace, using pid and ppid information to reconstruct the process tree and some userland database for the pid -> job mapping. It would, however, be greatly simplified if the accounting records provided some kind of job id, and some indicator whether or not this process was the last of a job (group).

CSA and ELSA might even be more lightweight since fewer accounting records are actually written.

Sounds like it should be possible to fulfill the different needs by having loadable modules for the different output formats, or by a /proc entry that controls some aspects like whether records are written per job or per process.

Comments?

Jay Lan

author

2004-08-26 20:59 UTC #7 lore raw

I do like to see a common data collection method in the kernl. Kernel does not need to decide how the data to be presented to the user space. An accounting loadable module such as CSA or ELSA will take care of how the data to be presented to meet the needs of different users.

Sounds reasonable?

Regards,
- jay

Quoted 60 lines · Tim Schmielau

Quoted 5 lines · Arthur Corliss
I would be very interested in a CSA implementation similar to what I have on IRIX. I will also plead guilty to not having downloaded the updated patches for either the kernel or the tools. I'm continuing to use my poor hack until a permanent solution gets accepted into the kernel, at which point I'll adopt that.
That's ok, we carefully discussed the changes to make sure no new tools are required ;-)
Quoted 5 lines
And if it counters the impression at all, I'm not a kernel developer, I proposed my hack out of need as a user of the tools. I also try to stay away from modified kernels, so I'm running Marcelos' 2.4 stable branch with only the 32bit u/gid_t hack applied. That's why I haven't had any feedback on the -mm branch.
I haven't even tried to get a patch into 2.4, since Marcelo is (rightly) quite resilent to new features.
Quoted 4 lines
In short, for my use BSD accounting is sufficient, but I'd love to see CSA in Linux as well. Linux hasn't moved too far into roles where it's a necessity (for what I'm doing, anyway), but I see CSA as something that would certainly help it assume those roles.
Does this mean you would want to have both in the same kernel, potentially turning on both at the same time?
Ok, let me summarize what I learned until now:
It should be easy to combine the data collection enhancements from CSA and ELSA to provide a common superset of information.
Output file formats vary, but might be unified if projects don't insist too much.
Main difference between CSA and ELSA on the one hand and BSD acct on the other is that the latter writes one record per process, while the former write one per job.
With the new BSD acct v3 format, it should be possible to do per job accounting entirely from userspace, using pid and ppid information to reconstruct the process tree and some userland database for the pid -> job mapping. It would, however, be greatly simplified if the accounting records provided some kind of job id, and some indicator whether or not this process was the last of a job (group).
CSA and ELSA might even be more lightweight since fewer accounting records are actually written.
Sounds like it should be possible to fulfill the different needs by having loadable modules for the different output formats, or by a /proc entry that controls some aspects like whether records are written per job or per process.
Comments?

Arthur Corliss

2004-08-28 01:28 UTC #8 lore raw

Quoted 7 lines · Jay Lan

I do like to see a common data collection method in the kernl. Kernel does not need to decide how the data to be presented to the user space. An accounting loadable module such as CSA or ELSA will take care of how the data to be presented to meet the needs of different users.
Sounds reasonable?

Seconded.

--Arthur Corliss
Bolverk's Lair -- http://arthur.corlissfamily.org/
Digital Mages -- http://www.digitalmages.com/
"Live Free or Die, the Only Way to Live" -- NH State Motto

Guillaume Thouvenin

2004-08-30 12:28 UTC #9 lore raw

Quoted 11 lines · Arthur Corliss

Quoted 7 lines · Jay Lan
I do like to see a common data collection method in the kernl. Kernel does not need to decide how the data to be presented to the user space. An accounting loadable module such as CSA or ELSA will take care of how the data to be presented to meet the needs of different users.
Sounds reasonable?
Seconded.

Thus, to be clear, the enhanced accounting can be divided into
three parts:

1) A common data collection method in the kernel.
We could start from BSD-accounting and add CSA information. Could it be something like BSD version4?

2) A module that will manage a job history. I mean, it will manage a structure in which we will keep the relationship between processes and "containers" and also between process and its children. The property needed here is that a child belongs to the same "job" as its parent. This allows to do per-job accounting. I will have a look to PAGG and JOB.

Can it be done in userspace? Is the lost of data (as observed by Arthur Corliss with SAR) can be avoided?

3) Finally we need module that will be in charge of datas presentation. This will allow to be easily compatible with other applications.

I will have a look to the per-job accounting patch. If I understand
well, this patch falls into the second requirement (manage groups of processes).

Regards,
Guillaume

Tim Schmielau

2004-08-31 14:21 UTC #10 lore raw

Quoted 6 lines · Guillaume Thouvenin

Thus, to be clear, the enhanced accounting can be divided into
three parts:
1) A common data collection method in the kernel.
We could start from BSD-accounting and add CSA information. Could it be something like BSD version4?

I've had a quick look at the CSA data collection patches. To get the discussion started, here are my comments:

Quoted 5 lines

drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000 -0700 renamed from drivers/block/ll_rw_blk.c 2004-08-13 22:36:16.000000000 -0700 +0 -0
@@ -1948,10 +1950,12 @@
 
        if (rw == READ) {

drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000 -0700 line 1961

Quoted 10 lines

drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000 -0700 renamed from drivers/block/ll_rw_blk.c 2004-08-13 22:36:16.000000000 -0700 +2 -0

                disk_stat_add(rq->rq_disk, read_sectors, nr_sectors);
+               current->rblk += nr_sectors;
                if (!new_io)
                        disk_stat_inc(rq->rq_disk, read_merges);
        } else if (rw == WRITE) {
                disk_stat_add(rq->rq_disk, write_sectors, nr_sectors);
+               current->wblk += nr_sectors;
                if (!new_io)
                        disk_stat_inc(rq->rq_disk, write_merges);
        }

Andi Kleen's comment on the ELSA patch also applies here - most writes will get accounted to pdflushd. See

http://www.lib.uaa.alaska.edu/linux-kernel/archive/2004-Week-31/0047.html

for his comment.

Quoted 2 lines

include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 -0700 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -0

[...]

include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 -0700

Quoted 5 lines

include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 -0700 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +5 -0
+#else  /* CONFIG_CSA || CONFIG_CSA_MODULE */
+
+#define csa_update_integrals()         do { } while (0);
+#define csa_clear_integrals(task)      do { } while (0);
+#endif /* CONFIG_CSA || CONFIG_CSA_MODULE */

I suppose the semicolons are unintentional.

Quoted 2 lines

include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 renamed from include/linux/sched.h 2004-08-19 15:17:52.000000000 -0700 +0 -0

[...]

include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 line 533

Quoted 8 lines

include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 renamed from include/linux/sched.h 2004-08-19 15:17:52.000000000 -0700 +4 -0

@@ -525,6 +527,10 @@
 
 /* i/o counters(bytes read/written, blocks read/written, #syscalls, waittime */
         unsigned long rchar, wchar, rblk, wblk, syscr, syscw, bwtime;
+#if defined(CONFIG_CSA) || defined(CONFIG_CSA_MODULE)
+       unsigned long csa_rss_mem1, csa_vm_mem1;
+       clock_t csa_stimexpd;
+#endif

These probably need to be u64, otherwise they might easily overflow within a view seconds on 32 bit platforms.

include/linux/acct_eop.h 2004-08-19 18:48:44.000000000 -0700

Quoted 2 lines

include/linux/acct_eop.h 2004-08-19 18:48:44.000000000 -0700 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -0

This should probably be unified with BSD accounting to a general accounting hook.

Tim

Jay Lan

author

2004-08-31 23:06 UTC #11 lore raw

Adding [email protected], the CSA user group mailing list, to Cc.

Quoted 34 lines · Tim Schmielau

Quoted 6 lines · Guillaume Thouvenin
Thus, to be clear, the enhanced accounting can be divided into
three parts:
1) A common data collection method in the kernel.
We could start from BSD-accounting and add CSA information. Could it be something like BSD version4?
I've had a quick look at the CSA data collection patches. To get the discussion started, here are my comments:
Quoted 5 lines
drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000 -0700 renamed from drivers/block/ll_rw_blk.c 2004-08-13 22:36:16.000000000 -0700 +0 -0
@@ -1948,10 +1950,12 @@
 
       if (rw == READ) {
drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000 -0700 line 1961
Quoted 10 lines
drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000 -0700 renamed from drivers/block/ll_rw_blk.c 2004-08-13 22:36:16.000000000 -0700 +2 -0
               disk_stat_add(rq->rq_disk, read_sectors, nr_sectors);
+               current->rblk += nr_sectors;
               if (!new_io)
                       disk_stat_inc(rq->rq_disk, read_merges);
       } else if (rw == WRITE) {
               disk_stat_add(rq->rq_disk, write_sectors, nr_sectors);
+               current->wblk += nr_sectors;
               if (!new_io)
                       disk_stat_inc(rq->rq_disk, write_merges);
       }
Andi Kleen's comment on the ELSA patch also applies here - most writes will get accounted to pdflushd. See

Quoted 3 lines

http://www.lib.uaa.alaska.edu/linux-kernel/archive/2004-Week-31/0047.html
for his comment.

I need more time on this. :)

Quoted 7 lines

Quoted 4 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 -0700 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -0
[...]

Quoted 8 lines

include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 -0700
Quoted 5 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 -0700 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +5 -0
+#else  /* CONFIG_CSA || CONFIG_CSA_MODULE */
+
+#define csa_update_integrals()         do { } while (0);
+#define csa_clear_integrals(task)      do { } while (0);
+#endif /* CONFIG_CSA || CONFIG_CSA_MODULE */
I suppose the semicolons are unintentional.

Good catch! I fixed this in our internal tree.

Quoted 9 lines

Quoted 4 lines
include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 renamed from include/linux/sched.h 2004-08-19 15:17:52.000000000 -0700 +0 -0
[...]
Quoted 2 lines
include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 renamed from include/linux/sched.h 2004-08-19 15:17:52.000000000 -0700 +0 -0
@@ -525,6 +527,10 @@
 

Quoted 10 lines

include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 line 533
Quoted 6 lines
include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700 renamed from include/linux/sched.h 2004-08-19 15:17:52.000000000 -0700 +4 -0
 /* i/o counters(bytes read/written, blocks read/written, #syscalls, waittime */
        unsigned long rchar, wchar, rblk, wblk, syscr, syscw, bwtime;
+#if defined(CONFIG_CSA) || defined(CONFIG_CSA_MODULE)
+       unsigned long csa_rss_mem1, csa_vm_mem1;
+       clock_t csa_stimexpd;
+#endif
These probably need to be u64, otherwise they might easily overflow within a view seconds on 32 bit platforms.

Will fix it.

Quoted 8 lines

include/linux/acct_eop.h 2004-08-19 18:48:44.000000000 -0700
Quoted 4 lines
include/linux/acct_eop.h 2004-08-19 18:48:44.000000000 -0700 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -0
This should probably be unified with BSD accounting to a general accounting hook.

Do you suggest to merge acct_eop.h into acct.h? It sounds good to me!

Thanks!
- jay

Quoted 14 lines

Tim
-------------------------------------------------------
This SF.Net email is sponsored by BEA Weblogic Workshop FREE Java Enterprise J2EE developer tools!
Get your free copy of BEA WebLogic Workshop 8.1 today.
http://ads.osdn.com/?ad_id=5047&alloc_id=10808&op=click
_______________________________________________
Lse-tech mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/lse-tech

Jay Lan

author

2004-09-08 18:38 UTC #12 lore raw

The block IO data was not used in billing for CSA customers.

Nobody has ever charged for this data. It is more for accountability and resource consumption tracking. Accuracy changes as the program runs for hours or days in our market. Repeatable billing is critical and block IO is one that is not repeatable.

The write blocks was useful and with the type of programs running for hours and days (which are the ones of interest in block IO data). The delayed write after the process terminates is ignored because that is insignificant for a process doing io for hours or days. We use the bytes transferred vs blocks transferred to see programs dominating and polluting the cache. Processes big in one and not the other are of interest to profile more closely.

However, if nobody else wants to have this feature, we can pull it out until we can find a way of doing this that makes people happy.

I will submit a new set of patch based on 2.6.8.1 later.

Thanks!
- jay

Quoted 141 lines · Jay Lan

Adding [email protected], the CSA user group mailing list, to Cc.
Quoted 38 lines · Tim Schmielau
Quoted 6 lines · Guillaume Thouvenin
Thus, to be clear, the enhanced accounting can be divided into
three parts:
1) A common data collection method in the kernel.
We could start from BSD-accounting and add CSA information. Could it be something like BSD version4?
I've had a quick look at the CSA data collection patches. To get the discussion started, here are my comments:
Quoted 7 lines
--- linux.orig/drivers/block/ll_rw_blk.c 2004-08-13
22:36:16.000000000 -0700
+++ linux/drivers/block/ll_rw_blk.c 2004-08-18 12:07:10.000000000
-0700
(quoted diff) +0 -0
@@ -1948,10 +1950,12 @@
 
       if (rw == READ) {
Review line 1961
Quoted 10 lines
(quoted diff) +2 -0
               disk_stat_add(rq->rq_disk, read_sectors, nr_sectors);
+               current->rblk += nr_sectors;
               if (!new_io)
                       disk_stat_inc(rq->rq_disk, read_merges);
       } else if (rw == WRITE) {
               disk_stat_add(rq->rq_disk, write_sectors, nr_sectors);
+               current->wblk += nr_sectors;
               if (!new_io)
                       disk_stat_inc(rq->rq_disk, write_merges);
       }
Andi Kleen's comment on the ELSA patch also applies here - most writes will get accounted to pdflushd. See
Quoted 3 lines
http://www.lib.uaa.alaska.edu/linux-kernel/archive/2004-Week-31/0047.html
for his comment.
I need more time on this. :)
Quoted 9 lines
Quoted 5 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -1
-0700
[...]
Quoted 9 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000
Quoted 5 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +5 -0
+#else  /* CONFIG_CSA || CONFIG_CSA_MODULE */
+
+#define csa_update_integrals()         do { } while (0);
+#define csa_clear_integrals(task)      do { } while (0);
+#endif /* CONFIG_CSA || CONFIG_CSA_MODULE */
I suppose the semicolons are unintentional.
Good catch! I fixed this in our internal tree.
Quoted 11 lines
Quoted 5 lines
--- linux.orig/include/linux/sched.h 2004-08-19 15:17:52.000000000
-0700
+++ linux/include/linux/sched.h 2004-08-19 15:19:05.000000000 -0700
[...]
Quoted 2 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -0
@@ -525,6 +527,10 @@
 
Quoted 11 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 line 533
Quoted 6 lines
include/linux/csa_internal.h 2004-08-19 15:19:05.000000000 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +4 -0
 /* i/o counters(bytes read/written, blocks read/written, #syscalls, waittime */
        unsigned long rchar, wchar, rblk, wblk, syscr, syscw, bwtime;
+#if defined(CONFIG_CSA) || defined(CONFIG_CSA_MODULE)
+       unsigned long csa_rss_mem1, csa_vm_mem1;
+       clock_t csa_stimexpd;
+#endif
These probably need to be u64, otherwise they might easily overflow within a view seconds on 32 bit platforms.
Will fix it.
Quoted 10 lines
include/linux/acct_eop.h 2004-08-19 18:48:44.000000000
Quoted 5 lines
include/linux/acct_eop.h 2004-08-19 18:48:44.000000000 renamed from dev/null 1970-01-01 00:00:00.000000000 +0000 +0 -1
-0700
This should probably be unified with BSD accounting to a general accounting hook.
Do you suggest to merge acct_eop.h into acct.h? It sounds good to me!
Thanks!
- jay
Quoted 5 lines
Tim
Quoted 9 lines
-------------------------------------------------------
This SF.Net email is sponsored by BEA Weblogic Workshop FREE Java Enterprise J2EE developer tools!
Get your free copy of BEA WebLogic Workshop 8.1 today.
http://ads.osdn.com/?ad_id=5047&alloc_id=10808&op=click
_______________________________________________
Lse-tech mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/lse-tech
-------------------------------------------------------
This SF.Net email is sponsored by BEA Weblogic Workshop FREE Java Enterprise J2EE developer tools!
Get your free copy of BEA WebLogic Workshop 8.1 today.
http://ads.osdn.com/?ad_id=5047&alloc_id=10808&op=click
_______________________________________________
Lse-tech mailing list [email protected]
https://lists.sourceforge.net/lists/listinfo/lse-tech

A message that is not in the archive (sent off-list or not yet received): [email protected]

John Hesterberg

2004-08-27 19:42 UTC #14 lore raw

Quoted 19 lines · Guillaume Thouvenin

Quoted 3 lines · Tim Schmielau
It should be easy to combine the data collection enhancements from CSA and ELSA to provide a common superset of information.
ELSA uses current BSD accounting. The only difference with BSD is that accounting is done for a group of processes. I didn't use PAGG and rewrite something because I thought (I was wrong) that PAGG project wasn't maintained. I continue to maintain ELSA just because there is, until today, no solution for doing job accounting.
So, the data collection enhancements from ELSA is not very useful.
Quoted 6 lines
With the new BSD acct v3 format, it should be possible to do per job accounting entirely from userspace, using pid and ppid information to reconstruct the process tree and some userland database for the pid -> job mapping. It would, however, be greatly simplified if the accounting records provided some kind of job id, and some indicator whether or not this process was the last of a job (group).

Quoted 5 lines

I like this solution.
In fact what I proposed was to have PAGG and a modified BSD accounting that can be used with PAGG as both are already in the -mm tree. But manage group of processes from userspace is, IMHO, a better solution as modifications in the kernel will be minimal.

The kernel part of linux-job is a module that uses PAGG, and isn't difficult. We've been running it in production for a couple years.

I don't think a kernel-based job is a requirement, though, so I'd like to hear more about how you'd do it otherwise.

The other comments about only one acct record per job vs one per process might be important, and that might mean the kernel has to know about the job.

Quoted 6 lines

Therefore the solution could be to enhance BSD accounting with data
collection from CSA and provide per job accounting with a userspace mechanism. Sounds great to me...
Best,
Guillaume

How does the BSD accounting define jobs? What determines the job that a process is part of?

An important aspect of linux-job (ie the job part of the pagg/job/csa stack) is that it is inescapable. The user doesn't get to determine or change their job (unlike process groups). For true accounting, that determines the real $$$ chargebacks on shared machines, this is necessary.

Another aspect of jobs that isn't directly related to accounting is that it gives users and admins a way to query, and kill :-), all the processes that are part of the job. The inescapable part is again important...you can't fork off a process and detach it from the job to hide it. In fact, I've heard that some sites use pagg/job without CSA for this reason. It might have been an ISP or ASP, and they liked the containment linux-job provided.

John

Tim Schmielau

2004-08-30 08:30 UTC #15 lore raw

Quoted 16 lines · John Hesterberg

Quoted 9 lines · Guillaume Thouvenin
Quoted 6 lines · Tim Schmielau
With the new BSD acct v3 format, it should be possible to do per job accounting entirely from userspace, using pid and ppid information to reconstruct the process tree and some userland database for the pid -> job mapping. It would, however, be greatly simplified if the accounting records provided some kind of job id, and some indicator whether or not this process was the last of a job (group).
Quoted 5 lines
I like this solution.
In fact what I proposed was to have PAGG and a modified BSD accounting that can be used with PAGG as both are already in the -mm tree. But manage group of processes from userspace is, IMHO, a better solution as modifications in the kernel will be minimal.

Quoted 3 lines

The kernel part of linux-job is a module that uses PAGG, and isn't difficult. We've been running it in production for a couple years.

Well, I'm rethinking my opinion of not wanting two accounting methods in the kernel. Make them share as much code as possible, with the only remaining difference being the format of the record and wether it is written per process or per job. Then we just have to make sure that both mechanisms get exercised regulary, to prevent bit-rot.

Quoted 6 lines

I don't think a kernel-based job is a requirement, though, so I'd like to hear more about how you'd do it otherwise.
The other comments about only one acct record per job vs one per process might be important, and that might mean the kernel has to know about the job.

Yes, it would probably be easier if the kernel knows about the job and could stuff a job ID into the acct record. If that means going from 64 byte records to 128 bytes, this would again double the already larger overhead of BSD accounting, however. This lightweightness of CSA is why I am not opposed to its inclusion.

On the other hand, there are a few uses (and users) of per-process accounting records, i.e. for security auditing, so we should not back it out of the kernel.

Quoted 2 lines

How does the BSD accounting define jobs? What determines the job that a process is part of?

BSD accounting doesn't have the concept of a job at all. When we discussed the v3 format, we considered adding a job ID field from PAGG, but a) nobody answered and b) there wasn't any space left in the record anyways. So a decision was postponed for a future 128 byte acct v4 structure.

Quoted 5 lines

An important aspect of linux-job (ie the job part of the pagg/job/csa stack) is that it is inescapable. The user doesn't get to determine or change their job (unlike process groups). For true accounting, that determines the real $$$ chargebacks on shared machines, this is necessary.

My proposed solution for a userspace method would also be inescapable:
To start a new job, just write out it's pid to a file that is only ever appended to, and consider all children of it as belonging to the same job.

Inescapeable, but probably some overhead in userspace.

Oh, wait - I think there is a problem in current BSD accounting, if the parent process dies before the child, and the child gets reparented to init. I should investigate that...

Quoted 7 lines

Another aspect of jobs that isn't directly related to accounting is that it gives users and admins a way to query, and kill :-), all the processes that are part of the job. The inescapable part is again important...you can't fork off a process and detach it from the job to hide it. In fact, I've heard that some sites use pagg/job without CSA for this reason. It might have been an ISP or ASP, and they liked the containment linux-job provided.

Yes, it's probably a lot easier if you don't have to search accounting files to do that.

So from my view, we might turn the discussion from whether we want CSA to how we integrate it, i.e. do some code review.

Tim

Jay Lan

2004-08-27 20:27 UTC #16 lore raw

In reply to a message not in the archive

Hi Guillaume,

Please visit http://oss.sgi.com/projects/pagg/
The page has been updated to provide information on a per job accounting project called 'job' based on PAGG.

There is one userspace rpm and one kernel module for job. This may provide what you are looking for. It is a mature product as well. I am sure Limin(job) and Erik(pagg) would appreciate any input you can provide to make 'job' more useful.

Regards,
- jay

Quoted 43 lines · Guillaume Thouvenin

Quoted 2 lines · Tim Schmielau
It should be easy to combine the data collection enhancements from CSA and ELSA to provide a common superset of information.
ELSA uses current BSD accounting. The only difference with BSD is that
accounting is done for a group of processes. I didn't use PAGG and
rewrite something because I thought (I was wrong) that PAGG project
wasn't maintained. I continue to maintain ELSA just because there is, until today, no solution for doing job accounting. So, the data collection enhancements from ELSA is not very useful.
Quoted 6 lines
With the new BSD acct v3 format, it should be possible to do per job accounting entirely from userspace, using pid and ppid information to reconstruct the process tree and some userland database for the pid -> job mapping. It would, however, be greatly simplified if the accounting records provided some kind of job id, and some indicator whether or not this process was the last of a job (group).
I like this solution.
In fact what I proposed was to have PAGG and a modified BSD accounting that can be used with PAGG as both are already in the -mm tree. But manage group of processes from userspace is, IMHO, a better solution as modifications in the kernel will be minimal.
Therefore the solution could be to enhance BSD accounting with data
collection from CSA and provide per job accounting with a userspace mechanism. Sounds great to me...
Best,
Guillaume
-------------------------------------------------------
SF.Net email is sponsored by Shop4tech.com-Lowest price on Blank Media 100pk Sonic DVD-R 4x for only $29 -100pk Sonic DVD+R for only $33 Save 50% off Retail on Ink & Toner - Free Shipping and Free Gift.
http://www.shop4tech.com/z/Inkjet_Cartridges/9_108_r285
_______________________________________________
Lse-tech mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/lse-tech

Guillaume Thouvenin

2004-08-31 09:17 UTC #17 lore raw

Quoted 8 lines · Jay Lan

Please visit http://oss.sgi.com/projects/pagg/
The page has been updated to provide information on a per job accounting project called 'job' based on PAGG.
There is one userspace rpm and one kernel module for job. This may provide what you are looking for. It is a mature product as well. I am sure Limin(job) and Erik(pagg) would appreciate any input you can provide to make 'job' more useful.

I have a question about job. If I understand how it works, you can not
add a process in a job. I mean when you start a session, a container is created and it's the only way to create it. If I'm right, I think that it could be interesting to add a process using ioctl and /proc interface. For example, if I want to know how resources are used by a compilation, I need to add the process gcc in a container. Any comments?

Best,
Guillaume

Guillaume Thouvenin

2004-08-31 10:32 UTC #18 lore raw

Quoted 10 lines · Guillaume Thouvenin

Quoted 8 lines · Jay Lan
Please visit http://oss.sgi.com/projects/pagg/
The page has been updated to provide information on a per job accounting project called 'job' based on PAGG.
There is one userspace rpm and one kernel module for job. This may provide what you are looking for. It is a mature product as well. I am sure Limin(job) and Erik(pagg) would appreciate any input you can provide to make 'job' more useful.

Quoted 4 lines

I have a question about job. If I understand how it works, you can not
add a process in a job. I mean when you start a session, a container is created and it's the only way to create it. If I'm right, I think that it could be interesting to add a process using ioctl and /proc interface.

I think that I'm not very clear. You can add a process to a container using the /proc/csa interface but it seems that currently this feature is not available with job-1.2.1 package. Therefore, maybe we can add a command called jattach that will attach a process to a given jid...

Best,
Guillaume

Limin Gu

2004-08-31 16:05 UTC #19 lore raw

Quoted 17 lines · Guillaume Thouvenin

Quoted 10 lines · Jay Lan
Quoted 8 lines
Please visit http://oss.sgi.com/projects/pagg/
The page has been updated to provide information on a per job accounting project called 'job' based on PAGG.
There is one userspace rpm and one kernel module for job. This may provide what you are looking for. It is a mature product as well. I am sure Limin(job) and Erik(pagg) would appreciate any input you can provide to make 'job' more useful.
Quoted 4 lines
I have a question about job. If I understand how it works, you can not
add a process in a job. I mean when you start a session, a container is created and it's the only way to create it. If I'm right, I think that it could be interesting to add a process using ioctl and /proc interface.

Quoted 4 lines

I think that I'm not very clear. You can add a process to a container using the /proc/csa interface but it seems that currently this feature is not available with job-1.2.1 package. Therefore, maybe we can add a command called jattach that will attach a process to a given jid...

The current job package is job-1.4.0-1, you can find it at ftp://oss.sgi.com/projects/pagg/download

And it is correct that JOB_ATTACH ioctl is not implemented right now. We could implement that ioctl and also add a user command like jattach.

We are trying to push the job kernel module to the community, but the ioctl and /proc binary interface seems not an appropriate kernel and user space communication interface. When job gets more users other CSA and there are more interest about job, maybe we could request a new syscall number in linux.

--Limin

Quoted 4 lines

Best,
Guillaume

John Hesterberg

2004-09-01 21:54 UTC #20 lore raw

In reply to Guillaume Thouvenin

Quoted 5 lines · Guillaume Thouvenin

Quoted 4 lines · Jay Lan
Please visit http://oss.sgi.com/projects/pagg/
The page has been updated to provide information on a per job accounting project called 'job' based on PAGG.

Quoted 8 lines

Quoted 4 lines
There is one userspace rpm and one kernel module for job. This may provide what you are looking for. It is a mature product as well. I am sure Limin(job) and Erik(pagg) would appreciate any input you can provide to make 'job' more useful.
I have a question about job. If I understand how it works, you can not
add a process in a job. I mean when you start a session, a container is created and it's the only way to create it.

Right, that's the current implementation. Any privileged process can create a job, though, it doesn't *have* to be at the start of a session. I believe job is currently hardwired that the initial member process is the creator, and the only other way in is via inheritance, and there's no way out of the job other than exiting or creating your own job.

Quoted 2 lines

If I'm right, I think that it could be interesting to add a process using ioctl and /proc interface.

We're planning on changing that interface, but I think your question applies regardless of what interface is used.

Quoted 6 lines

For example, if I want to know how resources are used by a compilation, I need to add the process gcc in a container. Any comments?
Best,
Guillaume

It sounds like a slightly different kind of job.

The gcc process should already be in a job via it's parent. If it's already in a job, we don't let it out, as jobs are designed to be inescapable so that users can't sneak processes outside their job. If the only client of job is accounting, that might not be required. Maybe as long as they become a member of another job, and the usage is tracked, that would be OK. I'm not sure what that would do to the current CSA tools, though.

On IRIX, I think jobs are also used to do resource limits, and that's probably where the hard requirement for jobs being inescapable came from.

The ISP/ASP non-acct uses of job would probably want it to be inescapable.

Different inheritance and creation policies could be implemented in job. Or, since it's just a loadable module, different job modules could be written to implement different styles of job as are required for different uses.

John

Arthur Corliss

2004-08-28 01:35 UTC #21 lore raw

In reply to a message not in the archive

Quoted 9 lines · Guillaume Thouvenin

I like this solution.
In fact what I proposed was to have PAGG and a modified BSD accounting that can be used with PAGG as both are already in the -mm tree. But manage group of processes from userspace is, IMHO, a better solution as modifications in the kernel will be minimal.
Therefore the solution could be to enhance BSD accounting with data
collection from CSA and provide per job accounting with a userspace mechanism. Sounds great to me...

The only concern I have with a userspace solution is that you run the risk of losing that data. What happens if a process on the box drives drives it out of memory and paging space? The box would still be working, it just wouldn't be able to fork new processes, and those already running that aren't purposely made high priority may not get much of a chance to execute as well. I've lost SAR data that way.

--Arthur Corliss
Bolverk's Lair -- http://arthur.corlissfamily.org/
Digital Mages -- http://www.digitalmages.com/
"Live Free or Die, the Only Way to Live" -- NH State Motto

Arthur Corliss

2004-08-28 01:28 UTC #22 lore raw

In reply to Tim Schmielau

Quoted 2 lines · Tim Schmielau

That's ok, we carefully discussed the changes to make sure no new tools are required ;-)

Understood, and appreciated.

Quoted 2 lines

Does this mean you would want to have both in the same kernel, potentially turning on both at the same time?

I think that's definitely the route to go. Not everyone needs the level of visibility that CSA provides, so I think that with a unified data collection methodology the users can decide for themselves what they need, and the kernel stays in good shape with no implementation-specific bloat.

Quoted 7 lines

Ok, let me summarize what I learned until now:
It should be easy to combine the data collection enhancements from CSA and ELSA to provide a common superset of information.
Output file formats vary, but might be unified if projects don't insist too much.

I don't think we'd necessarly want a unified accounting *file*. BSD account files grow pretty damned quick as it is, and that's with a lot less visibility than if you included the extract counters that CSA provides. Make the data available the logging module(s) or what have you via a common struct, but let each module decide what to actually commit to disk, and when (on job exit, etc.).

Quoted 10 lines

Main difference between CSA and ELSA on the one hand and BSD acct on the other is that the latter writes one record per process, while the former write one per job.
With the new BSD acct v3 format, it should be possible to do per job accounting entirely from userspace, using pid and ppid information to reconstruct the process tree and some userland database for the pid -> job mapping. It would, however, be greatly simplified if the accounting records provided some kind of job id, and some indicator whether or not this process was the last of a job (group).

Quoted 7 lines

CSA and ELSA might even be more lightweight since fewer accounting records are actually written.
Sounds like it should be possible to fulfill the different needs by having loadable modules for the different output formats, or by a /proc entry that controls some aspects like whether records are written per job or per process.

On one hand, loadable modules would be more helpful for people doing side-by-side comparisons of the accounting systems, but the /proc method would be better for dynamic adjustments of a single system. I don't think I have a horse in this race, either way.

--Arthur Corliss
Bolverk's Lair -- http://arthur.corlissfamily.org/
Digital Mages -- http://www.digitalmages.com/
"Live Free or Die, the Only Way to Live" -- NH State Motto

John Hesterberg

2004-08-27 03:03 UTC #23 lore raw

In reply to Tim Schmielau

Quoted 6 lines · Tim Schmielau

...
IMHO CSA, ELSA and BSD accounting are too similar to have more than one of them in the kernel. We should either improve BSD accounting to do the job, or kill it in favor of a different implementation.
Tim

We should at least have common data collection in the kernel.

I could more easily understand different accounting packages on top of that that might meet different needs of different classes of users.

John

Tim Schmielau

2004-08-27 08:32 UTC #24 lore raw

Quoted 6 lines · John Hesterberg

Quoted 5 lines · Tim Schmielau
...
IMHO CSA, ELSA and BSD accounting are too similar to have more than one of them in the kernel. We should either improve BSD accounting to do the job, or kill it in favor of a different implementation.

Quoted 6 lines

Quoted 1 line
Tim
We should at least have common data collection in the kernel.
I could more easily understand different accounting packages on top of that that might meet different needs of different classes of users.

Sorry, that is of course what I meant - I am only talking about kernel code.

Tim

John Hesterberg

2004-08-26 18:54 UTC #25 lore raw

In reply to Andrew Morton

Quoted 10 lines · Andrew Morton

Quoted 8 lines · Jay Lan
I have broken up one big CSA kernel patch into four smaller ones
as attached:
      csa_io     - collects io accounting data
      csa_mm     - collects mm accounting data
      csa_eop    - provides a hook to perform end-of-process accounting
csa_module - builds csa loadable module

Quoted 6 lines

More broadly: Help!
I am 100% not in a position to judge whether Linux needs Comprehensive System Accounting, nor am I able to define what the requirements for such a thing should be. All I can tell from your patch is the quality of its implementation, and that's leaping far, far ahead of where we should be.

Linux needs something beyond what it has today, at least for the HPC market SGI is familiar with. We believe it will more generally benefit Linux HPC and enterprise markets, which is one reason we've released the whole CSA stack as open source.

Quoted 9 lines

We're going to need help from you, and from all the other stakeholders in judging how useful this feature is to Linux implementors and how well this implementation meets the (unknown) requirements. See my problem?
I've cc'ed lse-tech, where enterprise folks hang out. I would request that the people who are stakeholders in this feature
a) stick their hands up

We're running CSA in production on Altix (our Itanium/Linux platform) for several years now.

Quoted 2 lines

b) let us know how important this kind of feature is for their users

A substantial number of our customers require it. CSA has been developed over the years on SGI's HPC systems in response to our customers needs. It's been reimplemented and opensourced for Linux, originally as an SGI/LANL collaboration.

Quoted 3 lines

c) review the offered feature set against their requirements
d) let us know how well the implementation fits that requirement and

It fits. :-)

Actually, one secondary feature on our wishlist is 'projects'.

Our customers are tied into the current CSA user interface. However, there is lots of room for cooperation under that, particularly in the kernel. We can always consider a migration project as well.

John

A message that is not in the archive (sent off-list or not yet received): [email protected]

Jay Lan

2004-08-27 20:37 UTC #28 lore raw

My proposed patchset includes three accounting data collection patches and one CSA loadable module that make use of the collected data. The three data collection patches (in the area of io, mm and end-of-process) do not conflict with BSD accounting or ELSA. They can be viewed as "enhancement" to both BSD and ELSA as Guillaume, the maintainer of ELSA put it:

Therefore the solution could be to enhance BSD accounting with data collection from CSA and provide per job accounting with a userspace mechanism. Sounds great to me...

It would be then up to BSD, ELSA or CSA to decide how to process the collected data and present it to users.

All response so far seems to favor this unified data collection method proposal.

The ELSA approach appears to make changes to kernel/acct.c while CSA will provide its own loadable module.

Of the three data collection patches, csa_io and csa_eop are very much CSA-independent. I certainly can make csa_mm more so as well.

I think it is a everyone-win situation for all three projects. :)

Regards,
- jay

Quoted 22 lines · Andrew Morton

Thanks, guys. So we now know that there are three potential implementations which do much the same thing, yes?
I didn't get a sense of a preferred direction, but at least nobody is flaming anybody else yet ;)
It strikes me that CSA is the most actively developed and is the furthest along. But that enhancing BSD accounting might be the least intrusive and most back-compatible approach.
Is that a fair summary? If not, what should I have said?
-------------------------------------------------------
SF.Net email is sponsored by Shop4tech.com-Lowest price on Blank Media 100pk Sonic DVD-R 4x for only $29 -100pk Sonic DVD+R for only $33 Save 50% off Retail on Ink & Toner - Free Shipping and Free Gift.
http://www.shop4tech.com/z/Inkjet_Cartridges/9_108_r285
_______________________________________________
Lse-tech mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/lse-tech

John Hesterberg

2004-08-30 19:18 UTC #29 lore raw

In reply to a message not in the archive

Quoted 2 lines · Andrew Morton

Thanks, guys. So we now know that there are three potential implementations which do much the same thing, yes?

I believe CSA does than the others.

Quoted 8 lines

I didn't get a sense of a preferred direction, but at least nobody is flaming anybody else yet ;)
It strikes me that CSA is the most actively developed and is the furthest along. But that enhancing BSD accounting might be the least intrusive and most back-compatible approach.
Is that a fair summary? If not, what should I have said?

Does anyone know if CSA is a super-set of BSD accounting and ELSA? What would be missing?

I'm unconvinced that enhancing BSD accounting to encompass the capabilities of CSA is appropriate.

I think we can make the data collection additions common. That should encompass the bulk of the invasive changes that are required by at least CSA proper (ie there are still the PAGG changes for job support that we can discuss separately). Not sure about BSD accounting and ELSA.

With that cooperation, we can then either proceed with further cooperation, or if the goals and users of the different accounting approaches dictate different kernel modules and user support, I'd propose that might be OK.

John