proc-based cpu affinity user interface
Subsystems (5)
I've been working on a patch that is a roll up of Andrew Morton's patch, and also of some work by Nick Pollitt to use a prctl() interface to the cpus_allowed bitmask. It also includes a new bitmask 'launch_policy' which allows a process to set a launch policy which is used to determine it's childrens' cpus_allowed bitmasks prior to launch.
The patch is against 2.4.10. It can be found on
http://sourceforge.com/projects/lse and I'll also attatch the most recent version below.
Robert,
There are two reasons not to use the cpus_online_map.
1. It isn't (or at least wasn't as of 2.4.10) available on all architectures.
2. If cpus are added, or removed, the bitmask the user originally requested is
effectively ignored. ie: if the user tries to set cpus_allowed to 0x0000ffff and we mask that against the current online map (0x00000003 in your example), and then another cpu comes online, the cpus_allowed bitmask will not be updated to reflect this change.
Ingo,
I'm going to look into using the set-affinity patch you posted with my patch.
I also agree that /proc is not the best way to access this data, but I *do* believe it is *a* good way. I think that having both interfaces (/proc and a syscall) gives us the best of both worlds.
Enjoy!
-matt
P.S.-please CC any replies, as I'm not on the list.
fs/proc/array.c +65 -2
@@ -50,6 +50,10 @@* Al Viro & Jeff Garzik : moved most of the thing into base.c and * : proc_misc.c. The rest may eventually go into * : base.c too. +* +* Andrew Morton : cpus_allowed +* +* Matthew Dobson : launch_policy (Thanks to Andrew Morton for inspiraton) */ #include <linux/config.h>
@@ -343,7 +347,7 @@read_unlock (&tasklist_lock); res = sprintf(buffer,"%d (%s) %c %d %d %d %d %d %lu %lu \ %lu %lu %lu %lu %lu %ld %ld %ld %ld %ld %ld %lu %lu %ld %lu %lu %lu %lu %lu \ -%lu %lu %lu %lu %lu %lu %lu %lu %d %d\n", +%lu %lu %lu %lu %lu %lu %lu %lu %d %d %lu %lu\n", task->pid, task->comm, state,
@@ -386,7 +390,9 @@ task->nswap, task->cnswap, task->exit_signal, - task->processor); + task->processor, + task->cpus_allowed, + task->launch_policy);if (mm)mmput (mm);return res;
@@ -691,5 +697,62 @@ task->per_cpu_stime[cpu_logical_map(i)]);return len; +} + +static inline int proc_pid_cpu_bitmask_read (char * buffer,unsigned long *bitmask) +{ +int len; + + len =sprintf (buffer,"%08lx\n" , *bitmask); +return len; +} + +/* +* FIXME: cpu_online_map should be used, but not all archs have it. +*/ + +static inline int proc_pid_cpu_bitmask_write (char * buffer,size_t nbytes,unsigned long *bitmask) +{ +unsigned long new_mask; +char *endp; +int ret; +unsigned long flags; + + ret = -EPERM ; +if (!capable (CAP_SYS_ADMIN )) + goto out; + + new_mask =simple_strtoul (buffer, &endp,16 ); + ret = endp - buffer; + +spin_lock_irqsave (&runqueue_lock, flags);/* token effort to not be racy */ +if ((((1 << smp_num_cpus) -1 ) & new_mask) ==0 ) + ret = -EINVAL ; +else + *bitmask = new_mask; +spin_unlock_irqrestore (&runqueue_lock, flags); + +out: +return ret; +} + +int proc_pid_cpus_allowed_read (struct task_struct *task,char * buffer) +{ +return proc_pid_cpu_bitmask_read (buffer, &task->cpus_allowed); +} + +int proc_pid_cpus_allowed_write (struct task_struct *task,char * buffer,size_t nbytes) +{ +return proc_pid_cpu_bitmask_write (buffer, nbytes, &task->cpus_allowed); +} + +int proc_pid_launch_policy_read (struct task_struct *task,char * buffer) +{ +return proc_pid_cpu_bitmask_read (buffer, &task->launch_policy); +} + +int proc_pid_launch_policy_write (struct task_struct *task,char * buffer,size_t nbytes) +{ +return proc_pid_cpu_bitmask_write (buffer, nbytes, &task->launch_policy); }#endif
fs/proc/base.c +66 -12
111 lines not on this page · raw message
include/linux/capability.h +2 -0
@@ -231,6 +231,8 @@/* Allow enabling/disabling tagged queuing on SCSI controllers and sending arbitrary SCSI commands */ /* Allow setting encryption key on loopback filesystem */ +/* Allow bonding of tasks to CPUs */ +/* Allow setting of launch policies */ #define CAP_SYS_ADMIN 21
include/linux/prctl.h +8 -0
@@ -20,4 +20,12 @@#define PR_GET_KEEPCAPS 7#define PR_SET_KEEPCAPS 8 +/* Get/set cpus allowed */ +#define PR_GET_CPUS_ALLOWED 13 +#define PR_SET_CPUS_ALLOWED 14 + +/* Get/set launch policies */ +#define PR_GET_LAUNCH_POLICY 15 +#define PR_SET_LAUNCH_POLICY 16 +#endif /* _LINUX_PRCTL_H */
include/linux/proc_fs_i.h +2 -1
@@ -1,9 +1,10 @@struct proc_inode_info {struct task_struct *task;int type; -union { +struct {int (*proc_get_link)(struct inode *,struct dentry **,struct vfsmount **);int (*proc_read)(struct task_struct *task,char *page); +int (*proc_write)(struct task_struct *task,char *page,size_t nbytes); } op;struct file *file; };
include/linux/sched.h +2 -0
@@ -344,6 +344,7 @@struct task_struct *pidhash_next;struct task_struct **pidhash_pprev; +unsigned long launch_policy;/* for *fork*() & exec() */ wait_queue_head_t wait_chldexit;/* for wait4() */ struct completion *vfork_done;/* for vfork() */ unsigned long rt_priority;
@@ -467,6 +468,7 @@ p_opptr: &tsk, \ p_pptr: &tsk, \ thread_group: LIST_HEAD_INIT(tsk.thread_group), \ + launch_policy: -1, \ wait_chldexit: __WAIT_QUEUE_HEAD_INITIALIZER(tsk.wait_chldexit),\ real_timer: { \ function: it_real_fn \
kernel/fork.c +1 -0
@@ -646,6 +646,7 @@ spin_lock_init(&p->sigmask_lock); }#endif + p->cpus_allowed = p->launch_policy;/* launch_policy is inherited from parent */ p->lock_depth =-1 ;/* -1 = no lock */ p->start_time = jiffies;
kernel/sys.c +27 -0
@@ -1256,6 +1256,33 @@ } current->keep_capabilities = arg2;break ; +case PR_GET_CPUS_ALLOWED : + error =put_user (current->cpus_allowed, (long *)arg2); +break ; +case PR_SET_CPUS_ALLOWED : +/* +* FIXME: cpu_online_map should be used, but not all archs have it. +*/ +if ((((1 << smp_num_cpus) -1 ) & arg2) ==0 ) + error = -EINVAL ; +else { + current->cpus_allowed = arg2; +if (!((1 <<smp_processor_id ()) & arg2)) + current->need_resched =1 ; + } +break ; +case PR_GET_LAUNCH_POLICY : + error =put_user (current->launch_policy, (long *)arg2); +break ; +case PR_SET_LAUNCH_POLICY : +/* +* FIXME: cpu_online_map should be used, but not all archs have it. +*/ +if ((((1 << smp_num_cpus) -1 ) & arg2) ==0 ) + error = -EINVAL ; +else + current->launch_policy = arg2; +break ;default : error = -EINVAL ;break ;
Quoted 5 lines
I've been working on a patch that is a roll up of Andrew Morton's patch, and also of some work by Nick Pollitt to use a
prctl()interface to thecpus_allowedbitmask. It also includes a new bitmask 'launch_policy' which allows a process to set a launch policy which is used to determine it's childrens'cpus_allowedbitmasks prior to launch.
It looks like you are limiting the number of CPUs to sizeof(long). Must you? Using "%lx" would be better in any case. Considering that you may outgrow the format, maybe this info doesn't belong in the /proc/*/stat files at all. For "ps" usage, a simple flag to indicate if the process is locked to a CPU would be OK. There are 3 cases of interest:
1. can run on all CPUs
2. can run on only the currently used CPU
3. can run on some subset of the available CPUs
(defined sanely for 1-CPU and 2-CPU systems of course)
Quoted 31 lines
diff -Nur
linux-2.4.10/fs/proc/array.clinux-2.4.10-launch_policy/fs/proc/array.cfs/proc/array.c +8
-2@@ -50,6 +50,10 @@* Al Viro & Jeff Garzik : moved most of the thing into base.c and * : proc_misc.c. The rest may eventually go into * : base.c too. +* +* Andrew Morton : cpus_allowed +* +* Matthew Dobson : launch_policy (Thanks to Andrew Morton for inspiraton) */ #include <linux/config.h> @@ -343,7 +347,7 @@read_unlock (&tasklist_lock); res = sprintf(buffer,"%d (%s) %c %d %d %d %d %d %lu %lu \ %lu %lu %lu %lu %lu %ld %ld %ld %ld %ld %ld %lu %lu %ld %lu %lu %lu %lu %lu \ -%lu %lu %lu %lu %lu %lu %lu %lu %d %d\n", +%lu %lu %lu %lu %lu %lu %lu %lu %d %d %lu %lu\n", task->pid, task->comm, state,@@ -386,7 +390,9 @@ task->nswap, task->cnswap, task->exit_signal, - task->processor); + task->processor, + task->cpus_allowed, + task->launch_policy);
Quoted 6 lines · Albert D. Cahalan
It looks like you are limiting the number of CPUs to sizeof(long). Must you? Using "%lx" would be better in any case. Considering that you may outgrow the format, maybe this info doesn't belong in the
/proc/*/statfiles at all. For "ps" usage, a simple flag to indicate if the process is locked to a CPU would be OK. There are 3 cases of interest:
We already limit it... we use cpus_allowed and cpus_runnable which are unsigned long.
Robert Love
Quoted 11 lines · Robert Love
Quoted 6 lines · Albert D. Cahalan
It looks like you are limiting the number of CPUs to sizeof(long). Must you? Using "%lx" would be better in any case. Considering that you may outgrow the format, maybe this info doesn't belong in the
/proc/*/statfiles at all. For "ps" usage, a simple flag to indicate if the process is locked to a CPU would be OK. There are 3 cases of interest:We already limit it... we use
cpus_allowedandcpus_runnablewhich are unsigned long.
Those can be changed without screwing up user code. You need to be very careful about what you put in /proc, because user apps will come to rely on whatever you choose.
Look at what happened with signals in /proc/*/stat. They were originally reported as "%lu", just like you did. Then the kernel was extended to support more than 32 signals on 32-bit hardware. The format was briefly changed to be 16 hex chars, but this broke lots of stuff. So now /proc/*/stat just reports the low numbered signals as it did before, and /proc/*/status has the rest of the signals.
So let's predict:
Around 2.5.42, IBM will have 60-way x86 or 80-way ppc64. Then the kernel will be hacked to report a hex string of arbitrary length, and user apps will die left and right. Somebody will then hack things back to decimal, reporting only the low sizeof(long)*8 processors in /proc/*/stat. Any additional processors may be found as a hex string in /proc/*/status, with the /proc/*/stat data useless on a system with very many processors.