Skip to contentSkip to conversation

2.5.64-mm2->4 hangs on contest

DISCUSSION Ed Tomlinson 3 participants 6 messages Mar 10, 2003 2 messages not in the archive

Subsystems (5)
lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

A message that is not in the archive (sent off-list or not yet received): [email protected]

#2 lore raw
Quoted 5 lines · Mike Galbraith
Quoted 4 lines · Con Kolivas
Quoted 1 line · Mike Galbraith

Ahem. Attached this time.

I assume this is against bk? I'll massage it into 2.5.64-mm4

Suspect that the interactivity changes have make the problem that my ptg patch is designed to fix easier to hit. Con where is the latest contest (a quick google does not help)? Mike what version of irman are you using? The one I have has problems parsing /proc/mem in mm.

Here is the ptg patch for 64-mm4. To apply it reverse the schedule-tunables-fix and schedule-tuneables and then apply the ptg and update tuneables patch.

Ed Tomlinson

----------- ptg-D3-mm2 (applies to mm4) ----------
# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:

#                  ChangeSet    1.1087  -> 1.1088 
#       include/linux/sched.h   1.137   -> 1.138  
#              kernel/fork.c    1.112   -> 1.113  
#              kernel/user.c    1.6     -> 1.7    
#             kernel/sched.c    1.163   -> 1.164  

#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 03/03/07 [email protected] 1.1088
# Add user and thread group governors to prevent either from monoplizing

# the system.  The governors work by limiting the sum of the timeslices
# of active tasks in a group to <n> timeslices.  The defaults set <n> to
# 1.5 for thread groups and to 10 for user tasks.
# 
# For numa systems the governors are per node.  Idealy the storage
# should also be local to the node however, we do not have dynamic per
# node storage yet...
# --------------------------------------------
#
diff -Nru a/include/linux/sched.h b/include/linux/sched.h
--- a/include/linux/sched.h     Sat Mar  8 14:18:48 2003
+++ b/include/linux/sched.h     Sat Mar  8 14:18:48 2003
@@ -178,6 +178,11 @@

#include <linux/aio.h>

+struct ptg_struct { /* pseudo thread groups */
+ atomic_t active[MAX_NUMNODES];
+ atomic_t count; /* number of refs */
+};
+
struct mm_struct {
struct vm_area_struct * mmap; /* list of VMAs */
struct rb_root mm_rb;
@@ -278,6 +283,7 @@
struct user_struct {
atomic_t __count; /* reference count */
atomic_t processes; /* How many processes does this user have? */
+ atomic_t active[MAX_NUMNODES];
atomic_t files; /* How many open files does this user have? */

/* Hash table maintenance information */
@@ -344,6 +350,8 @@

        struct list_head ptrace_list;
 
        struct mm_struct *mm, *active_mm;
+       struct ptg_struct * ptgroup;            /* pseudo thread group for this task */
+       atomic_t *governor;                     /* the atomic_t that governs this task */

/* task state */

        struct linux_binfmt *binfmt;
kernel/fork.c Sat Mar 8 14:18:48 2003 +45 -0
diff -Nru a/kernel/fork.c b/kernel/fork.c
@@ -96,12 +96,24 @@
        }
 }
 
+void free_ptgroup(struct task_struct *tsk)
+{
+       if (tsk->ptgroup && atomic_sub_and_test(1,&tsk->ptgroup->count)) {
+                kfree(tsk->ptgroup);
+                tsk->ptgroup = NULL;
+                tsk->governor = &tsk->user->active[cpu_to_node(task_cpu(tsk))];
+                if (tsk == current)
+                        atomic_inc(tsk->governor);
+        }
+}
+
 void __put_task_struct(struct task_struct *tsk)
 {
        WARN_ON(!(tsk->state & (TASK_DEAD | TASK_ZOMBIE)));
        WARN_ON(atomic_read(&tsk->usage));
        WARN_ON(tsk == current);
 
+       free_ptgroup(tsk);
        security_task_free(tsk);
        free_uid(tsk->user);
        free_task_struct(tsk);
@@ -469,6 +481,7 @@
 
        tsk->mm = NULL;
        tsk->active_mm = NULL;
+       tsk->ptgroup = NULL;
 
        /*
         * Are we cloning a kernel thread?
@@ -734,6 +747,32 @@
        p->flags = new_flags;
 }
 
+static inline int setup_governor(unsigned long clone_flags, struct task_struct *p)
+{
+       if ( ((clone_flags & CLONE_VM) && (clone_flags & CLONE_FILES)) ||
+            (clone_flags & CLONE_THREAD)) {
+               if (current->ptgroup)
+                       atomic_inc(&current->ptgroup->count);
+               else {
+                       int i;
+                       current->ptgroup = kmalloc(sizeof(struct ptg_struct), GFP_ATOMIC);
+                       if (!current->ptgroup)
+                               return 1;
+                       /* printk(KERN_INFO "ptgroup - pid %u\n",current->pid); */
+                       atomic_set(&current->ptgroup->count,2);
+                       for(i=0; i < MAX_NUMNODES; i++)
+                               atomic_set(&current->ptgroup->active[i], 0);
+                       atomic_set(&current->ptgroup->active[numa_node_id()], 1);
+                       atomic_dec(current->governor);
+                       current->governor = &current->ptgroup->active[numa_node_id()];
+               }
+               p->ptgroup = current->ptgroup;
+               p->governor = &p->ptgroup->active[numa_node_id()];
+       } else
+               p->governor = &p->user->active[numa_node_id()];
+       return 0;
+}
+
 asmlinkage int sys_set_tid_address(int *tidptr)
 {
        current->clear_child_tid = tidptr;
@@ -876,6 +915,12 @@
                goto bad_fork_cleanup_mm;
        retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs);
        if (retval)
+               goto bad_fork_cleanup_namespace;
+       /*
+        * Setup the governor pointer for the new process, allocating a new ptg as
+        * required if the process is a thread. 
+        */
+       if (setup_governor(clone_flags, p))
                goto bad_fork_cleanup_namespace;
 
        if (clone_flags & CLONE_CHILD_SETTID)
kernel/sched.c Sat Mar 8 14:18:48 2003 +42 -10

115 lines not on this page · raw message

kernel/user.c Sat Mar 8 14:18:48 2003 +9 -0
diff -Nru a/kernel/user.c b/kernel/user.c
@@ -30,6 +30,7 @@
 struct user_struct root_user = {
        .__count        = ATOMIC_INIT(1),
        .processes      = ATOMIC_INIT(1),
+       .active         = {[0 ...MAX_NUMNODES-1] = ATOMIC_INIT(0)},
        .files          = ATOMIC_INIT(0)
 };
 
@@ -89,6 +90,7 @@
 
        if (!up) {
                struct user_struct *new;
+               int i;
 
                new = kmem_cache_alloc(uid_cachep, SLAB_KERNEL);
                if (!new)
@@ -96,6 +98,8 @@
                new->uid = uid;
                atomic_set(&new->__count, 1);
                atomic_set(&new->processes, 0);
+               for(i=0; i < MAX_NUMNODES; i++)
+                       atomic_set(&new->active[i], 0);
                atomic_set(&new->files, 0);
 
                /*
@@ -130,6 +134,11 @@
        atomic_inc(&new_user->processes);
        atomic_dec(&old_user->processes);
        current->user = new_user;
+       if (!current->ptgroup) {
+               atomic_dec(current->governor);
+               current->governor = &current->user->active[numa_node_id()];
+               atomic_inc(current->governor);
+       }
        free_uid(old_user);
 }
 

----------- schedule-tunables for ptg -------------
# This is a BitKeeper generated patch for the following project:
# Project Name: Linux kernel tree
# This patch format is intended for GNU patch command version 2.5 or higher. # This patch includes the following deltas:

#                  ChangeSet    1.1088  -> 1.1089 
#            kernel/sysctl.c    1.39    -> 1.40   
#       Documentation/filesystems/proc.txt      1.14    -> 1.15   
#       include/linux/sysctl.h  1.42    -> 1.43   
#             kernel/sched.c    1.164   -> 1.165  

#
# The following is the BitKeeper ChangeSet Log
# --------------------------------------------
# 03/03/07 [email protected] 1.1089
# + schedule tunables for ptg
# --------------------------------------------
#

Documentation/filesystems/proc.txt Sat Mar 8 14:19:12 2003 +108 -0

120 lines not on this page · raw message

include/linux/sysctl.h Sat Mar 8 14:19:12 2003 +17 -1
diff -Nru a/include/linux/sysctl.h b/include/linux/sysctl.h
@@ -66,7 +66,8 @@
        CTL_DEV=7,              /* Devices */
        CTL_BUS=8,              /* Busses */
        CTL_ABI=9,              /* Binary emulation */
-       CTL_CPU=10              /* CPU stuff (speed scaling, etc) */
+       CTL_CPU=10,             /* CPU stuff (speed scaling, etc) */
+       CTL_SCHED=11,           /* scheduler tunables */
 };
 
 /* CTL_BUS names: */
@@ -157,6 +158,21 @@
        VM_LOWER_ZONE_PROTECTION=20,/* Amount of protection of lower zones */
 };
 
+/* Tunable scheduler parameters in /proc/sys/sched/ */
+enum {
+       SCHED_MIN_TIMESLICE=1,          /* minimum process timeslice */
+       SCHED_MAX_TIMESLICE=2,          /* maximum process timeslice */
+       SCHED_CHILD_PENALTY=3,          /* penalty on fork to child */
+       SCHED_PARENT_PENALTY=4,         /* penalty on fork to parent */
+       SCHED_EXIT_WEIGHT=5,            /* penalty to parent of CPU hog child */
+       SCHED_PRIO_BONUS_RATIO=6,       /* percent of max prio given as bonus */
+       SCHED_INTERACTIVE_DELTA=7,      /* delta used to scale interactivity */
+       SCHED_MAX_SLEEP_AVG=8,          /* maximum sleep avg attainable */
+       SCHED_STARVATION_LIMIT=9,       /* no re-active if expired is starved */
+       SCHED_NODE_THRESHOLD=10,        /* NUMA imbalance threshold */
+       SCHED_THREAD_GOVERNOR=11,       /* govern threadgroups when needed */
+       SCHED_USER_GOVERNOR=12,         /* govern users when needed  */
+};
 
 /* CTL_NET names: */
 enum
kernel/sched.c Sat Mar 8 14:19:12 2003 +28 -12
diff -Nru a/kernel/sched.c b/kernel/sched.c
@@ -57,19 +57,35 @@
  * Minimum timeslice is 10 msecs, default timeslice is 100 msecs,
  * maximum timeslice is 200 msecs. Timeslices get refilled after
  * they expire.
+ *
+ * They are configurable via /proc/sys/sched
  */
-#define MIN_TIMESLICE          ( 10 * HZ / 1000)
-#define MAX_TIMESLICE          (200 * HZ / 1000)
-#define CHILD_PENALTY          50
-#define PARENT_PENALTY         100
-#define EXIT_WEIGHT            3
-#define PRIO_BONUS_RATIO       25
-#define INTERACTIVE_DELTA      2
-#define MAX_SLEEP_AVG          (10*HZ)
-#define STARVATION_LIMIT       (10*HZ)
-#define NODE_THRESHOLD         125
-#define THREAD_GOVERNOR                15      /* allow threads groups 1.5 full timeslices */
-#define USER_GOVERNOR          100     /* allow user 10 full timeslices */
+
+int min_timeslice = (10 * HZ) / 1000;
+int max_timeslice = (200 * HZ) / 1000;
+int child_penalty = 50;
+int parent_penalty = 100;
+int exit_weight = 3;
+int prio_bonus_ratio = 25;
+int interactive_delta = 2;
+int max_sleep_avg = 10 * HZ;
+int starvation_limit = 10 * HZ;
+int node_threshold = 125;
+int thread_governor = 15;
+int user_governor = 100;
+
+#define MIN_TIMESLICE          (min_timeslice)
+#define MAX_TIMESLICE          (max_timeslice)
+#define CHILD_PENALTY          (child_penalty)
+#define PARENT_PENALTY         (parent_penalty)
+#define EXIT_WEIGHT            (exit_weight)
+#define PRIO_BONUS_RATIO       (prio_bonus_ratio)
+#define INTERACTIVE_DELTA      (interactive_delta)
+#define MAX_SLEEP_AVG          (max_sleep_avg)
+#define STARVATION_LIMIT       (starvation_limit)
+#define NODE_THRESHOLD         (node_threshold)
+#define THREAD_GOVERNOR                (thread_governor)
+#define USER_GOVERNOR          (user_governor)
 
 
 /*
kernel/sysctl.c Sat Mar 8 14:19:12 2003 +55 -1

80 lines not on this page · raw message

-----------

#3 lore raw
Quoted 10 lines · Ed Tomlinson
Quoted 5 lines · Mike Galbraith
Quoted 4 lines · Con Kolivas
Quoted 1 line · Mike Galbraith

Ahem. Attached this time.

I assume this is against bk? I'll massage it into 2.5.64-mm4

Suspect that the interactivity changes have make the problem that my ptg patch is designed to fix easier to hit. Con where is the latest contest (a quick google does not help)? Mike what version of irman

http://contest.kolivas.org

Con

#4 lore raw
In reply to Ed Tomlinson
Quoted 4 lines · Ed Tomlinson
Quoted 2 lines · Mike Galbraith
Quoted 1 line · Con Kolivas

On Mon, 10 Mar 2003 21:31, Mike Galbraith wrote:

Quoted 8 lines
Quoted 3 lines
Quoted 3 lines
Quoted 1 line

Ahem. Attached this time.

I assume this is against bk? I'll massage it into 2.5.64-mm4

Suspect that the interactivity changes have make the problem that my ptg patch is designed to fix easier to hit. Con where is the latest contest (a quick google does not help)? Mike what version of irman are you using? The one I have has problems parsing /proc/mem in mm.

Version 0.5. (also has parse problem)

#5 lore raw
Quoted 6 lines · Mike Galbraith
Quoted 5 lines · Ed Tomlinson
Quoted 4 lines · Mike Galbraith
Quoted 3 lines · Con Kolivas
Quoted 1 line · Mike Galbraith

Ahem. Attached this time.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line
Quoted 1 line

I assume this is against bk? I'll massage it into 2.5.64-mm4

Suspect that the interactivity changes have make the problem that my ptg patch is designed to fix easier to hit. Con where is the latest contest (a quick google does not help)? Mike what version of irman are you using? The one I have has problems parsing /proc/mem in mm.

Version 0.5. (also has parse problem)

If you guys play with the ptg patch there are two tunables you should be aware of. The first is theard_governor, which probably will not need to be touched. The second is user_governor. This one you may want to reduce from 100 to 50 - meaning start reducing a user's timeslices when there are more than 5 in run queue tasks for that user.

Ed

#6 lore raw
Quoted 14 lines · Ed Tomlinson
Quoted 13 lines · Mike Galbraith
Quoted 11 lines · Ed Tomlinson
Quoted 5 lines · Mike Galbraith
Quoted 4 lines · Con Kolivas
Quoted 1 line · Mike Galbraith

Ahem. Attached this time.

I assume this is against bk? I'll massage it into 2.5.64-mm4

Suspect that the interactivity changes have make the problem that my ptg patch is designed to fix easier to hit. Con where is the latest contest (a quick google does not help)? Mike what version of irman are you using? The one I have has problems parsing /proc/mem in mm.

Quoted 7 lines
Quoted 1 line

Version 0.5. (also has parse problem)

If you guys play with the ptg patch there are two tunables you should be aware of. The first is theard_governor, which probably will not need to be touched. The second is user_governor. This one you may want to reduce from 100 to 50 - meaning start reducing a user's timeslices when there are more than 5 in run queue tasks for that user.

In my experiments, reducing timeslice does horrible things to throughput. The hind of testing I do is mostly throughput oriented... latency is secondary when you're running full throttle ;-)

-Mike

#7 lore raw
In reply to Ed Tomlinson
Quoted 10 lines · Ed Tomlinson
Quoted 5 lines · Mike Galbraith
Quoted 4 lines · Con Kolivas
Quoted 1 line · Mike Galbraith

Ahem. Attached this time.

I assume this is against bk? I'll massage it into 2.5.64-mm4

Suspect that the interactivity changes have make the problem that my ptg patch is designed to fix easier to hit.

I think the problem I'm hitting is that the HUGE context switching that irman does causes the process load proggies to constantly max out their sleep average. There are no threads involved in irman, it's just plain old fork().

-Mike