Add credit2 scheduler (EXPERIMENTAL)
PATCH 1/1 Add credit2 scheduler (EXPERIMENTAL)
This patch series introduces the credit2 scheduler. The first two patches introduce changes necessary to allow the credit2 shared runqueue functionality to work properly; the last two implement the functionality itself.
The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas and status updates.
19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28
PATCH 1/1 credit2: Add context_saved scheduler callback
2 files changed, 3 insertions(+) xen/common/schedule.c | 2 ++ xen/include/xen/sched-if.h | 1 +
Because credit2 shares a runqueue between several cpus, it needs to know when a scheduled-out process has finally been context-switched away so that it can be added to the runqueue again. (Otherwise it may be grabbed by another processor before the context has been properly saved.)
xen/common/schedule.c +2 -0
@@ -923,6 +923,8 @@/* Check for migration request /after/ clearing running flag. */ smp_mb (); +SCHED_OP (context_saved, prev); +if (unlikely (test_bit (_VPF_migrating, &prev->pause_flags)) )vcpu_migrate (prev); }
xen/include/xen/sched-if.h +1 -0
@@ -70,6 +70,7 @@void (*sleep) (struct vcpu *);void (*wake) (struct vcpu *); +void (*context_saved) (struct vcpu *);struct task_slice (*do_schedule) (s_time_t );
PATCH 1/1 credit2: Flexible cpu-to-schedule-spinlock mappings
In reply to George Dunlap4 files changed, 40 insertions(+), 19 deletions(-) xen/arch/ia64/vmx/vmmu.c | 6 +++--- xen/common/sched_credit.c | 8 ++++---- xen/common/schedule.c | 18 ++++++++++-------- xen/include/xen/sched-if.h | 27 +++++++++++++++++++++++----
Credit2 shares a runqueue between several cpus. Rather than have double locking and dealing with the cpu-to-runqueue races, allow the scheduler to redefine the sched_lock-to-cpu mapping.
xen/arch/ia64/vmx/vmmu.c +3 -3
@@ -394,7 +394,7 @@if (cpu != current->processor)return ;local_irq_save (flags); -if (!spin_trylock (&per_cpu (schedule_data, cpu).schedule_lock)) +if (!spin_trylock (per_cpu (schedule_data, cpu).schedule_lock)) goto bail2;if (v->processor != cpu) goto bail1;
@@ -416,7 +416,7 @@ia64_dv_serialize_data (); args->vcpu =NULL ; bail1: -spin_unlock (&per_cpu (schedule_data, cpu).schedule_lock); +spin_unlock (per_cpu (schedule_data, cpu).schedule_lock); bail2:local_irq_restore (flags); }
@@ -446,7 +446,7 @@ do { cpu = v->processor; if (cpu != current->processor) { - spin_barrier(&per_cpu(schedule_data, cpu).schedule_lock); + spin_barrier(per_cpu(schedule_data, cpu).schedule_lock); /* Flush VHPT on remote processors. */ smp_call_function_single(cpu, &ptc_ga_remote_func, &args, 1); } else {
xen/common/sched_credit.c +4 -4
@@ -789,7 +789,7 @@ spc->runq_sort_last = sort_epoch; -spin_lock_irqsave (&per_cpu (schedule_data, cpu).schedule_lock, flags); +spin_lock_irqsave (per_cpu (schedule_data, cpu).schedule_lock, flags); runq = &spc->runq; elem = runq->next;
@@ -814,7 +814,7 @@ elem = next; } -spin_unlock_irqrestore (&per_cpu (schedule_data, cpu).schedule_lock, flags); +spin_unlock_irqrestore (per_cpu (schedule_data, cpu).schedule_lock, flags); } static void
@@ -1130,7 +1130,7 @@* cause a deadlock if the peer CPU is also load balancing and trying * to lock this CPU. */ -if ( !spin_trylock (&per_cpu (schedule_data, peer_cpu).schedule_lock) ) +if ( !spin_trylock (per_cpu (schedule_data, peer_cpu).schedule_lock) ) {CSCHED_STAT_CRANK (steal_trylock_failed);continue ;
@@ -1140,7 +1140,7 @@* Any work over there to steal? */ speer =csched_runq_steal (peer_cpu, cpu, snext->pri); -spin_unlock (&per_cpu (schedule_data, peer_cpu).schedule_lock); +spin_unlock (per_cpu (schedule_data, peer_cpu).schedule_lock);if ( speer !=NULL )return speer; }
xen/common/schedule.c +10 -8
@@ -131,7 +131,7 @@s_time_t delta;ASSERT (v->runstate.state != new_state); -ASSERT (spin_is_locked (&per_cpu (schedule_data,v->processor).schedule_lock)); +ASSERT (spin_is_locked (per_cpu (schedule_data,v->processor).schedule_lock));vcpu_urgent_count_update (v);
@@ -340,7 +340,7 @@/* Switch to new CPU, then unlock old CPU. */ v->processor = new_cpu;spin_unlock_irqrestore ( - &per_cpu (schedule_data, old_cpu).schedule_lock, flags); +per_cpu (schedule_data, old_cpu).schedule_lock, flags);/* Wake on new CPU. */ vcpu_wake (v);
@@ -846,7 +846,7 @@ sd = &this_cpu (schedule_data); -spin_lock_irq (&sd->schedule_lock); +spin_lock_irq (sd->schedule_lock);stop_timer (&sd->s_timer);
@@ -862,7 +862,7 @@if (unlikely (prev == next) ) { -spin_unlock_irq (&sd->schedule_lock); +spin_unlock_irq (sd->schedule_lock);trace_continue_running (next);return continue_running (prev); }
@@ -900,7 +900,7 @@ASSERT (!next->is_running); next->is_running =1 ; -spin_unlock_irq (&sd->schedule_lock); +spin_unlock_irq (sd->schedule_lock);perfc_incr (sched_ctx);
@@ -968,7 +968,9 @@for_each_possible_cpu ( i ) { -spin_lock_init (&per_cpu (schedule_data, i).schedule_lock); +spin_lock_init (&per_cpu (schedule_data, i)._lock); +per_cpu (schedule_data, i).schedule_lock + = &per_cpu (schedule_data, i)._lock;init_timer (&per_cpu (schedule_data, i).s_timer, s_timer_fn,NULL , i); }
@@ -1005,10 +1007,10 @@for_each_online_cpu ( i ) { -spin_lock (&per_cpu (schedule_data, i).schedule_lock); +spin_lock (per_cpu (schedule_data, i).schedule_lock);printk ("CPU[%02d] " , i);SCHED_OP (dump_cpu_state, i); -spin_unlock (&per_cpu (schedule_data, i).schedule_lock); +spin_unlock (per_cpu (schedule_data, i).schedule_lock); }local_irq_restore (flags);
xen/include/xen/sched-if.h +23 -4
@@ -10,8 +10,19 @@#include <xen/percpu.h> +/* +* In order to allow a scheduler to remap the lock->cpu mapping, +* we have a per-cpu pointer, along with a pre-allocated set of +* locks. The generic schedule init code will point each schedule lock +* pointer to the schedule lock; if the scheduler wants to remap them, +* it can simply modify the schedule locks. +* +* For cache betterness, keep the actual lock in the same cache area +* as the rest of the struct. Just have the scheduler point to the +* one it wants (This may be the one right in front of it).*/ struct schedule_data { -spinlock_t schedule_lock;/* spinlock protecting curr */ +spinlock_t *schedule_lock, + _lock;struct vcpu *curr;/* current task */ struct vcpu *idle;/* idle task for this cpu */ void *sched_priv;
@@ -27,11 +38,19 @@for ( ; ; ) { +/* NB: For schedulers with multiple cores per runqueue, +* a vcpu may change processor w/o changing runqueues; +* so we may release a lock only to grab it again. +* +* If that is measured to be an issue, then the check +* should be changed to checking if the locks pointed to +* by cpu and v->processor are still the same. +*/ cpu = v->processor; -spin_lock (&per_cpu (schedule_data, cpu).schedule_lock); +spin_lock (per_cpu (schedule_data, cpu).schedule_lock);if (likely (v->processor == cpu) )break ; -spin_unlock (&per_cpu (schedule_data, cpu).schedule_lock); +spin_unlock (per_cpu (schedule_data, cpu).schedule_lock); } }
@@ -42,7 +61,7 @@static inline void vcpu_schedule_unlock (struct vcpu *v) { -spin_unlock (&per_cpu (schedule_data, v->processor).schedule_lock); +spin_unlock (per_cpu (schedule_data, v->processor).schedule_lock); }#define vcpu_schedule_unlock_irq (v) \
PATCH 1/1 credit2: Add a scheduler-specific schedule trace class
In reply to George Dunlap1 file changed, 1 insertion(+)
xen/include/public/trace.h | 1 +
xen/include/public/trace.h +1 -0
@@ -53,6 +53,7 @@#define TRC_HVM_HANDLER 0x00082000/* various HVM handlers */ #define TRC_SCHED_MIN 0x00021000/* Just runstate changes */ +#define TRC_SCHED_CLASS 0x00022000/* Scheduler-specific */ #define TRC_SCHED_VERBOSE 0x00028000/* More inclusive scheduling */ /* Trace events per class */
PATCH 1/1 credit2: Add credit2 scheduler to hypervisor
In reply to George Dunlap4 files changed, 1132 insertions(+) xen/common/Makefile | 1 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++++++++++++++++ xen/common/schedule.c | 2 xen/include/public/domctl.h | 4
This is the core credit2 patch. It adds the new credit2 scheduler to the hypervisor, as the non-default scheduler. It should be emphasized that this is still in the development phase, and is probably still unstable. It is known to be suboptimal for multi-socket systems.
xen/common/Makefile +1 -0
@@ -13,6 +13,7 @@ obj-y += page_alloc.o obj-y += rangeset.o obj-y += sched_credit.o +obj-y += sched_credit2.o obj-y += sched_sedf.o obj-y += schedule.o obj-y += shutdown.o
xen/common/sched_credit2.c new file +1125 -0
1,125 lines not on this page · raw message
xen/common/schedule.c +2 -0
@@ -56,9 +56,11 @@extern const struct scheduler sched_sedf_def;extern const struct scheduler sched_credit_def; +extern const struct scheduler sched_credit2_def;static const struct scheduler *__initdata schedulers[] = { &sched_sedf_def, &sched_credit_def, + &sched_credit2_def,NULL };
xen/include/public/domctl.h +4 -0
@@ -303,6 +303,7 @@/* Scheduler types. */ #define XEN_SCHEDULER_SEDF 4#define XEN_SCHEDULER_CREDIT 5 +#define XEN_SCHEDULER_CREDIT2 6/* Set or get info? */ #define XEN_DOMCTL_SCHEDOP_putinfo 0#define XEN_DOMCTL_SCHEDOP_getinfo 1
@@ -321,6 +322,9 @@uint16_t weight;uint16_t cap; } credit; +struct xen_domctl_sched_credit2 { +uint16_t weight; + } credit2; } u; };typedef struct xen_domctl_scheduler_op xen_domctl_scheduler_op_t ;
PATCH 1/1 credit2: Add toolstack options to control credit2 scheduler parameters
In reply to George Dunlap11 files changed, 277 insertions(+), 2 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 +++++++++++++++++ tools/libxc/xenctrl.h | 8 ++ tools/python/xen/lowlevel/xc/xc.c | 58 ++++++++++++++++++++ tools/python/xen/xend/XendAPI.py | 3 - tools/python/xen/xend/XendDomain.py | 54 +++++++++++++++++++ tools/python/xen/xend/XendDomainInfo.py | 4 + tools/python/xen/xend/XendNode.py | 4 + tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 ++++ tools/python/xen/xm/main.py | 82 +++++++++++++++++++++++++++++
tools/libxc/Makefile +1 -0
@@ -17,6 +17,7 @@ CTRL_SRCS-y += xc_private.c CTRL_SRCS-y += xc_sedf.c CTRL_SRCS-y += xc_csched.c +CTRL_SRCS-y += xc_csched2.c CTRL_SRCS-y += xc_tbuf.c CTRL_SRCS-y += xc_pm.c CTRL_SRCS-y += xc_cpu_hotplug.c
tools/libxc/xc_csched2.c new file +50 -0
50 lines not on this page · raw message
tools/libxc/xenctrl.h +8 -0
@@ -475,6 +475,14 @@uint32_t domid, struct xen_domctl_sched_credit *sdom); +int xc_sched_credit2_domain_set (int xc_handle, +uint32_t domid, +struct xen_domctl_sched_credit2 *sdom); + +int xc_sched_credit2_domain_get (int xc_handle, +uint32_t domid, +struct xen_domctl_sched_credit2 *sdom); +/** * This function sends a trigger to a domain. *
tools/python/xen/lowlevel/xc/xc.c +58 -0
76 lines not on this page · raw message
tools/python/xen/xend/XendAPI.py +1 -2
@@ -1626,8 +1626,7 @@ if 'weight' in xeninfo.info['vcpus_params'] \ and 'cap' in xeninfo.info['vcpus_params']: weight = xeninfo.info['vcpus_params']['weight'] - cap = xeninfo.info['vcpus_params']['cap'] - xendom.domain_sched_credit_set(xeninfo.getDomid(), weight, cap) + xendom.domain_sched_credit2_set(xeninfo.getDomid(), weight) def VM_set_VCPUs_number_live(self, _, vm_ref, num): dom = XendDomain.instance().get_vm_by_uuid(vm_ref)
tools/python/xen/xend/XendDomain.py +54 -0
60 lines not on this page · raw message
tools/python/xen/xend/XendDomainInfo.py +4 -0
@@ -2811,6 +2811,10 @@ XendDomain.instance().domain_sched_credit_set(self.getDomid(), self.getWeight(), self.getCap()) + elif XendNode.instance().xenschedinfo() == 'credit2': + from xen.xend import XendDomain + XendDomain.instance().domain_sched_credit2_set(self.getDomid(), + self.getWeight()) def _initDomain(self): log.debug('XendDomainInfo.initDomain: %s %s',
tools/python/xen/xend/XendNode.py +4 -0
@@ -779,6 +779,8 @@ return 'sedf' elif sched_id == xen.lowlevel.xc.XEN_SCHEDULER_CREDIT: return 'credit' + elif sched_id == xen.lowlevel.xc.XEN_SCHEDULER_CREDIT2: + return 'credit2' else: return 'unknown'
@@ -988,6 +990,8 @@ return 'sedf' elif sched_id == xen.lowlevel.xc.XEN_SCHEDULER_CREDIT: return 'credit' + elif sched_id == xen.lowlevel.xc.XEN_SCHEDULER_CREDIT2: + return 'credit2' else: return 'unknown'
tools/python/xen/xend/XendVMMetrics.py +1 -0
@@ -129,6 +129,7 @@ params_live['cpumap%i' % i] = \ ",".join(map(str, info['cpumap'])) + # FIXME: credit2?? params_live.update(xc.sched_credit_domain_get(domid)) return params_live
tools/python/xen/xend/server/SrvDomain.py +14 -0
@@ -163,6 +163,20 @@ val = fn(req.args, {'dom': self.dom.getName()}) return val + def op_domain_sched_credit2_get(self, _, req): + fn = FormFn(self.xd.domain_sched_credit2_get, + [['dom', 'str']]) + val = fn(req.args, {'dom': self.dom.getName()}) + return val + + + def op_domain_sched_credit2_set(self, _, req): + fn = FormFn(self.xd.domain_sched_credit2_set, + [['dom', 'str'], + ['weight', 'int']]) + val = fn(req.args, {'dom': self.dom.getName()}) + return val + def op_maxmem_set(self, _, req): return self.call(self.dom.setMemoryMaximum, [['memory', 'int']],
tools/python/xen/xm/main.py +82 -0
112 lines not on this page · raw message
Keir has checked the patches in, so if you wait a bit, they should show up on the public repository.
The tool patch is only necessary for adjusting the weight; if you're OK using the default weight, just adding "sched=credit2" on the xen command-line should be fine.
Don't forget that this isn't meant to perform well on multiple sockets yet. :-)
-George
Quoted 64 lines · Dan Magenheimer
Hi George --
I'm seeing some problems applying the patches (such as "malformed
patch"). If you could send me a monolithic patch in an attachment
and tell me what cset in http://xenbits.xensource.com/xen-unstable.hg that it successfully applies against, I will try to give my workload a test against it to see if it has the same
symptoms.Also, do I need to apply the tools patch if I don't intend to specify any parameters, or is the xen patch + "sched=credit2" in a boot param sufficient?
Thanks,
DanQuoted 48 lines
-----Original Message-----
From: George Dunlap [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 4:26 AM
To: [email protected]
Cc: [email protected]
Subject: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
The patches are already available from the staging tree. They will get automatically pushed to the main tree when they pass the regression tests.
K.
Quoted 84 lines · George Dunlap
Keir has checked the patches in, so if you wait a bit, they should show up on the public repository.
The tool patch is only necessary for adjusting the weight; if you're OK using the default weight, just adding "sched=credit2" on the xen command-line should be fine.
Don't forget that this isn't meant to perform well on multiple sockets yet. :-)
-George
Quoted 9 lines · Dan Magenheimer
Hi George --
I'm seeing some problems applying the patches (such as "malformed patch"). If you could send me a monolithic patch in an attachment and tell me what cset in http://xenbits.xensource.com/xen-unstable.hg
that it successfully applies against, I will try to give my workload a test against it to see if it has the same symptoms.Quoted 56 lines
Also, do I need to apply the tools patch if I don't intend to specify any parameters, or is the xen patch + "sched=credit2" in a boot param sufficient?
Thanks,
DanQuoted 48 lines
-----Original Message-----
From: George Dunlap [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 4:26 AM
To: [email protected]
Cc: [email protected]
Subject: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Thanks. Unfortunately, after updating both hypervisor and tools to cs21173 (from staging), xend seems to start fine, but attempting to launch a domain yields:
Xend has probably crashed! Invalid or missing HTTP status code.
An immediate "xm list" shows that xend has not crashed (or perhaps silently and successfully restarted), but re-attempting to launch a domain yields the same message.
(George, no need to point out that this is probably unrelated to the credit2 scheduler... but that would imply that xen-unstable-not-staging is also broken.)
Keir, if staging passes your regression tests successfully without the problem I am seeing, please let me know. (And I'll try rolling back to 4.0 for now.)
Quoted 110 lines
-----Original Message-----
From: Keir Fraser [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 8:53 AM
To: George Dunlap; Dan Magenheimer
Cc: [email protected]
Subject: Re: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)The patches are already available from the staging tree. They will get automatically pushed to the main tree when they pass the regression tests.
K.
Quoted 1 line · George Dunlap
Keir has checked the patches in, so if you wait a bit, they should
show
Quoted 3 lines
up on the public repository.
The tool patch is only necessary for adjusting the weight; if you're
OK
Quoted 4 lines
using the default weight, just adding "sched=credit2" on the xen command-line should be fine.
Don't forget that this isn't meant to perform well on multiple
sockets
Quoted 10 lines
yet. :-)
-George
Quoted 5 lines · Dan Magenheimer
Hi George --
I'm seeing some problems applying the patches (such as "malformed patch"). If you could send me a monolithic patch in an attachment and tell me what cset in http://xenbits.xensource.com/xen-
unstable.hg
Quoted 20 lines
Quoted 20 lines
that it successfully applies against, I will try to give my workload a test against it to see if it has the same symptoms.
Also, do I need to apply the tools patch if I don't intend to specify any parameters, or is the xen patch + "sched=credit2" in a boot param sufficient?
Thanks,
DanQuoted 8 lines
-----Original Message-----
From: George Dunlap [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 4:26 AM
To: [email protected]
Cc: [email protected]
Subject: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development;
Quoted 1 line
Quoted 1 line
Quoted 1 line
George Dunlap <[email protected]> or check out the wiki
page
Quoted 37 lines
Quoted 26 lines
Quoted 26 lines
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28Quoted 5 lines
Quoted 5 lines
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 7 lines · Dan Magenheimer
(George, no need to point out that this is probably unrelated to the credit2 scheduler... but that would imply that xen-unstable-not-staging is also broken.)
Keir, if staging passes your regression tests successfully without the problem I am seeing, please let me know. (And I'll try rolling back to 4.0 for now.)
No, it crashes for me too. I think it's related to the recent NUMA patches.
-- Keir
Quoted 1 line
No, it crashes for me too. I think it's related to the recent NUMA patches.
Keir, which numa patches
-dulloor
Quoted 21 lines · Keir Fraser
Quoted 7 lines · Dan Magenheimer
(George, no need to point out that this is probably unrelated to the credit2 scheduler... but that would imply that xen-unstable-not-staging is also broken.)
Keir, if staging passes your regression tests successfully without the problem I am seeing, please let me know. (And I'll try rolling back to 4.0 for now.)
No, it crashes for me too. I think it's related to the recent NUMA patches.
-- Keir
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 2 lines · Dulloor
Quoted 1 line
No, it crashes for me too. I think it's related to the recent NUMA patches.
Keir, which numa patches
Nitin's interface-changing patch, and the ensuing patch to remove sockets_per_node. Not that I'm certain, but it was around then that the crashes began, and that's what touched the Xc Python extension package (and C extensions to Python code are usually what make Python programs crash).
-- Keir
Quoted 10 lines
Quoted 7 lines
(George, no need to point out that this is probably unrelated to the credit2 scheduler... but that would imply that xen-unstable-not-staging is also broken.)
Keir, if staging passes your regression tests successfully without the problem I am seeing, please let me know. (And I'll try rolling back to 4.0 for now.)
No, it crashes for me too. I think it's related to the recent NUMA patches.
OK. George, I've applied your credit2 patch to 4.0-testing tip and it seems to be starting up my test workload. I'll let you know what I see (but it takes a few hours).
While someone is fixing up the new NUMA code, it would be nice if the output of the NUMA info in "xm info" would conform to the rest of the "xm info" output (e.g. all on the same line, maybe "topology: cpu=X node=Y etc"). I often write scripts that parse various Xen output, and I suspect many Xen-based products do also, so format consistency is always good.
Quoted 29 lines
-----Original Message-----
From: Keir Fraser [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 10:36 AM
To: Dulloor
Cc: George Dunlap; Dan Magenheimer; [email protected]
Subject: Re: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)Quoted 1 line · Dulloor
Quoted 1 line
No, it crashes for me too. I think it's related to the recent NUMA
patches.
Quoted 1 line
Keir, which numa patches
Nitin's interface-changing patch, and the ensuing patch to remove
sockets_per_node. Not that I'm certain, but it was around then that the crashes began, and that's what touched the Xc Python extension package (and
C extensions to Python code are usually what make Python programs crash).-- Keir
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
I have not measured cache / TLB misses with this workload yet. In the past I've instrumented the scheduler trace records in Xen to include performance counters such as instructions executed and cache / tlb misses, and then used xenalyze (http://xenbits.xensource.com/ext/xenalyze.hg) to analyze them. But the functionality for both capture and analysis was never standardized or added to mainline.
I'd be happy to help point you in the right direction if you're interested in investing in that approach. :-)
-George
Quoted 36 lines · Naresh Rapolu
Hello George,
How did you measure Cache/ TLB misses etc while using/profiling this new scheduler ? Any tool that you`ve used which works with Xen ?
Thanks,
Naresh Rapolu.
PhD Student, Computer Science,
Purdue University.Quoted 23 lines · George Dunlap
This patch series introduces the credit2 scheduler. The first two patches introduce changes necessary to allow the credit2 shared runqueue functionality to work properly; the last two implement the functionality itself.
The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 6 lines · Dulloor
As we talked before, I am interested in improving the mutiple-socket scenario and adding the load balancing functionalilty, which could provide an acceptable alternative to pinning vcpus to sockets (for my NUMA work). I am going over your patch right now, but what are your thoughts ?
That would be great -- my focus for the next several months will be setting up a testing infrastructure to automatically test performance of different workloads mixes so I can hone the algorithm and test regressions.
My idea with load balancing was to do this:
* One runqueue per L2 cache.
* Add code to calculate the load of a runqueue. Load would be the average (~integral) of (vcpus running + vcpus on runqueue). I was planning on doing accurate load calculation, rather than sample-based, and falling back to sample-based if accurate turned out to be too slow.
* Calculate the load contributed by various vcpus.
* At regular intervals, determine of some kind of balancing needs to be done by looking at the overall runqueue load and placing based on "contributory" load of each VCPU.
Does that make sense? Thoughts?
I have some old patches that calculated accurate load, I could dig them up if you wanted something to start with. (I don't think they'll apply cleanly at the moment.)
Thanks,
-George
Quoted 42 lines
-dulloor
Quoted 37 lines · George Dunlap
This patch series introduces the credit2 scheduler. The first two patches introduce changes necessary to allow the credit2 shared runqueue functionality to work properly; the last two implement the functionality itself.
The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Hello George,
I am trying to get linux "perf" tool work with Xen(Virtualize PMU to measure hardware events from inside guests). I have the following options :
1. allowing the guest kernel to see the PMU hardware features via cpuid, and then doing whatever is necessary to make them work as expected (by instruction emulation, etc), or
2. keeping them hidden, but adding a new Xen interface and the appropriate Linux-side code to detect that interface and use it
Does Xenalyze have any code relevant to this ? Can you think of any directions in this regard ?
Thanks,
Naresh Rapolu.
Quoted 43 lines · George Dunlap
I have not measured cache / TLB misses with this workload yet. In the past I've instrumented the scheduler trace records in Xen to include performance counters such as instructions executed and cache / tlb misses, and then used xenalyze (http://xenbits.xensource.com/ext/xenalyze.hg) to analyze them. But the functionality for both capture and analysis was never standardized or added to mainline.
I'd be happy to help point you in the right direction if you're interested in investing in that approach. :-)
-George
Quoted 35 lines · Naresh Rapolu
Hello George,
How did you measure Cache/ TLB misses etc while using/profiling this new scheduler ? Any tool that you`ve used which works with Xen ?
Thanks,
Naresh Rapolu.
PhD Student, Computer Science,
Purdue University.Quoted 22 lines · George Dunlap
This patch series introduces the credit2 scheduler. The first two patches introduce changes necessary to allow the credit2 shared runqueue functionality to work properly; the last two implement the functionality itself.
The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125+++++++++++++++++++++++++++++
xen/common/schedule.c| 22xen/include/public/domctl.h| 4xen/include/public/trace.h| 1xen/include/xen/sched-if.h| 28
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Naresh,
If you are interested only in profiling, you could use xenoprof too. I had ported xenoprof to pvops (attaching a patch that applies cleanly to linux pvops). I have used this with passive profiling and for profiling xen/dom0. This patch also includes an obvious fix (over oprofile branch in Jeremy's repo) for active profiling, although I didn't get a chance to test.
Please let know if you try this and if you face any issues.
thanks
dulloor
Quoted 91 lines · Naresh Rapolu
Hello George,
I am trying to get linux "perf" tool work with Xen(Virtualize PMU to measure hardware events from inside guests).
I have the following options :1. allowing the guest kernel to see the PMU hardware features via cpuid, and then doing whatever is necessary to make them work as expected (by instruction emulation, etc), or
2. keeping them hidden, but adding a new Xen interface and the appropriate Linux-side code to detect that interface and use itDoes Xenalyze have any code relevant to this ? Can you think of any directions in this regard ?
Thanks,
Naresh Rapolu.Quoted 8 lines · George Dunlap
I have not measured cache / TLB misses with this workload yet. In the past I've instrumented the scheduler trace records in Xen to include performance counters such as instructions executed and cache / tlb misses, and then used xenalyze (http://xenbits.xensource.com/ext/xenalyze.hg) to analyze them. But the functionality for both capture and analysis was never standardized or added to mainline.
Quoted 60 lines
I'd be happy to help point you in the right direction if you're interested in investing in that approach. :-)
-George
Quoted 53 lines · Naresh Rapolu
Hello George,
How did you measure Cache/ TLB misses etc while using/profiling this new scheduler ? Any tool that you`ve used which works with Xen ?
Thanks,
Naresh Rapolu.
PhD Student, Computer Science,
Purdue University.Quoted 38 lines · George Dunlap
This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-)
tools/libxc/Makefile| 1tools/libxc/xc_csched2.c| 50 +tools/libxc/xenctrl.h| 8tools/python/xen/lowlevel/xc/xc.c| 58 +tools/python/xen/xend/XendAPI.py| 3tools/python/xen/xend/XendDomain.py| 54 +tools/python/xen/xend/XendDomainInfo.py| 4tools/python/xen/xend/XendNode.py| 4tools/python/xen/xend/XendVMMetrics.py| 1tools/python/xen/xend/server/SrvDomain.py| 14tools/python/xen/xm/main.py| 82 ++xen/arch/ia64/vmx/vmmu.c| 6 xen/common/Makefile| 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125+++++++++++++++++++++++++++++
xen/common/schedule.c| 22xen/include/public/domctl.h| 4xen/include/public/trace.h| 1xen/include/xen/sched-if.h| 28
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 5 lines
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
arch/x86/include/asm/xen/hypercall.h +6 -0
@@ -450,6 +450,12 @@ HYPERVISOR_nmi_op(unsigned long op, unsigned long arg)return _hypercall2 (int, nmi_op, op, arg); } +static inline int +HYPERVISOR_xenoprof_op (unsigned int op,void *arg) +{ +return _hypercall2 (int, xenoprof_op, op, arg); +} +static inline void MULTI_fpu_taskswitch (struct multicall_entry *mcl,int set) {
arch/x86/oprofile/Makefile +6 -0
@@ -6,6 +6,12 @@ DRIVER_OBJS = $(addprefix ../../../drivers/oprofile/, \ oprofilefs.o oprofile_stats.o \ timer_int.o ) +ifdef CONFIG_XEN +XENOPROF_COMMON_OBJS = $(addprefix ../../../drivers/xen/xenoprof/, \ + xenoprofile.o) +DRIVER_OBJS := $(DRIVER_OBJS) \ + $(XENOPROF_COMMON_OBJS) xenoprof.o +endif oprofile-y := $(DRIVER_OBJS) init.o backtrace.o oprofile-$(CONFIG_X86_LOCAL_APIC) += nmi_int.o op_model_amd.o \ op_model_ppro.o op_model_p4.o
arch/x86/oprofile/xenoprof.c new file +172 -0
172 lines not on this page · raw message
arch/x86/xen/mmu.c +35 -13
92 lines not on this page · raw message
drivers/oprofile/buffer_sync.c +69 -12
150 lines not on this page · raw message
drivers/oprofile/cpu_buffer.c +59 -14
135 lines not on this page · raw message
drivers/oprofile/cpu_buffer.h +16 -4
@@ -40,7 +40,7 @@ struct op_entry;struct oprofile_cpu_buffer {unsigned long buffer_size;struct task_struct *last_task; -int last_is_kernel; +int last_cpu_mode;int tracing;unsigned long sample_received;unsigned long sample_lost_overflow;
@@ -62,7 +62,7 @@ static inline void op_cpu_buffer_reset(int cpu) {struct oprofile_cpu_buffer *cpu_buf = &per_cpu (cpu_buffer, cpu); - cpu_buf->last_is_kernel =-1 ; + cpu_buf->last_cpu_mode =-1 ; cpu_buf->last_task =NULL ; }
@@ -111,10 +111,22 @@ int op_cpu_buffer_get_data(struct op_entry *entry, unsigned long *val) return size; } +/* data flags */ +/* cpu modes */ +/* */ +#define CPU_MODE_BEGIN (0UL) +#define CPU_MODE_USER (CPU_MODE_BEGIN + 0x0) +#define CPU_MODE_KERNEL (CPU_MODE_BEGIN + 0x1) +#ifdef CONFIG_XEN +#define CPU_MODE_XEN (CPU_MODE_BEGIN + 0x2) +#endif +#define CPU_MODE_END (CPU_MODE_BEGIN + 0x3) +#define CPU_MODE_MASK 0x3 +/* extra data flags */ -#define KERNEL_CTX_SWITCH (1UL << 0) -#define IS_KERNEL (1UL << 1)#define TRACE_BEGIN (1UL << 2)#define USER_CTX_SWITCH (1UL << 3) +#define KERNEL_CTX_SWITCH (1UL << 4) +#define XEN_DOMAIN_SWITCH (1UL << 5)#endif /* OPROFILE_CPU_BUFFER_H */
drivers/oprofile/event_buffer.h +5 -0
@@ -30,6 +30,11 @@ void wake_up_buffer_waiter(void);#define INVALID_COOKIE ~0UL#define NO_COOKIE 0UL +#ifdef CONFIG_XEN +#define XEN_COORDINATOR_DOMAIN -1 +#endif + +extern const struct file_operations event_buffer_fops;/* mutex between sync_cpu_buffers() and the
drivers/oprofile/oprof.c +50 -3
77 lines not on this page · raw message
drivers/oprofile/oprof.h +5 -0
@@ -36,4 +36,9 @@ void oprofile_timer_init(struct oprofile_operations *ops);int oprofile_set_backtrace (unsigned long depth); +#ifdef CONFIG_XEN +int oprofile_xen_set_active (int active_domains[],unsigned int adomains); +int oprofile_xen_set_passive (int passive_domains[],unsigned int pdomains); +#endif +#endif /* OPROF_H */
drivers/oprofile/oprofile_files.c +210 -0
228 lines not on this page · raw message
drivers/xen/xenoprof/xenoprofile.c new file +547 -0
547 lines not on this page · raw message
include/linux/oprofile.h +21 -1
@@ -16,6 +16,9 @@#include <linux/types.h> #include <linux/spinlock.h> #include <asm/atomic.h> +#ifdef CONFIG_XEN +#include <xen/interface/xenoprof.h> +#endif /* Each escaped entry is prefixed by ESCAPE_CODE * then one of the following codes, then the
@@ -28,14 +31,18 @@#define CPU_SWITCH_CODE 2#define COOKIE_SWITCH_CODE 3#define KERNEL_ENTER_SWITCH_CODE 4 -#define KERNEL_EXIT_SWITCH_CODE 5 +#define USER_ENTER_SWITCH_CODE 5#define MODULE_LOADED_CODE 6#define CTX_TGID_CODE 7#define TRACE_BEGIN_CODE 8#define TRACE_END_CODE 9#define XEN_ENTER_SWITCH_CODE 10 +#ifndef CONFIG_XEN #define SPU_PROFILING_CODE 11#define SPU_CTX_SWITCH_CODE 12 +#else +#define XEN_DOMAIN_SWITCH_CODE 11 +#endif #define IBS_FETCH_CODE 13#define IBS_OP_CODE 14
@@ -49,6 +56,12 @@ struct oprofile_operations {/* create any necessary configuration files in the oprofile fs. * Optional. */ int (*create_files)(struct super_block * sb,struct dentry * root); +#ifdef CONFIG_XEN +/* setup active domains with Xen */ +int (*xen_set_active)(int *active_domains,unsigned int adomains); +/* setup passive domains with Xen */ +int (*xen_set_passive)(int *passive_domains,unsigned int pdomains); +#endif /* Do any necessary interrupt setup. Optional. */ int (*setup)(void );/* Do any necessary interrupt shutdown. Optional. */
@@ -104,9 +117,16 @@ void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,* backtrace. */ void oprofile_add_pc (unsigned long pc,int is_kernel,unsigned long event); +/* Record when the cpu mode switches between user/kernel/xen(hypervisor) */ +void oprofile_add_mode (int cpu_mode); +/* add a backtrace entry, to be called from the ->backtrace callback */ void oprofile_add_trace (unsigned long eip); +#ifdef CONFIG_XEN +/* add a xen domain switch entry */ +int oprofile_add_domain_switch (int32_t domain_id); +#endif /** * Create a file of the given name as a child of the given root, with
include/xen/interface/xen.h +2 -0
@@ -79,6 +79,8 @@#define VIRQ_CONSOLE 2/* (DOM0) Bytes received on emergency console. */ #define VIRQ_DOM_EXC 3/* (DOM0) Exceptional event for some domain. */ #define VIRQ_DEBUGGER 6/* (DOM0) A domain has paused for debugging. */ +#define VIRQ_XENOPROF 7/* V. XenOprofile interrupt: new sample available */ +/* Architecture-specific VIRQ definitions. */ #define VIRQ_ARCH_0 16
include/xen/interface/xenoprof.h new file +140 -0
140 lines not on this page · raw message
include/xen/xen-ops.h +3 -0
@@ -25,4 +25,7 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma, pgprot_t prot, unsigned domid); +int xen_remap_domain_kernel_mfn_range (unsigned long addr, +unsigned long mfn,int nr, +pgprot_t prot,unsigned domid);#endif /* INCLUDE_XEN_OPS_H */
include/xen/xenoprof.h new file +69 -0
69 lines not on this page · raw message
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Hello Dulloor,
Thank you so much for sharing this patch !! Jeremy and I feel that "perf" support is needed in Xen for thorough (per-process, per-vpcu) profiling, not just statistical profiling by Oprofile. Now that I have this latest Xenoprof patch, I will try to stick to its design and see how "perf" linux subsystem can use xenoprof hypercall interfaces.
Will keep updating you on this regularly.
Thanks,
Naresh Rapolu,
PhD student, Computer Science,
Purdue University.
Quoted 116 lines · Dulloor
Naresh,
If you are interested only in profiling, you could use xenoprof too. I had ported xenoprof to pvops (attaching a patch that applies cleanly to linux pvops). I have used this with passive profiling and for profiling xen/dom0. This patch also includes an obvious fix (over oprofile branch in Jeremy's repo) for active profiling, although I didn't get a chance to test.
Please let know if you try this and if you face any issues.
thanks
dulloorQuoted 100 lines · Naresh Rapolu
Hello George,
I am trying to get linux "perf" tool work with Xen(Virtualize PMU to measure hardware events from inside guests).
I have the following options :1. allowing the guest kernel to see the PMU hardware features via cpuid, and then doing whatever is necessary to make them work as expected (by instruction emulation, etc), or
2. keeping them hidden, but adding a new Xen interface and the appropriate Linux-side code to detect that interface and use itDoes Xenalyze have any code relevant to this ? Can you think of any directions in this regard ?
Thanks,
Naresh Rapolu.Quoted 7 lines · George Dunlap
I have not measured cache / TLB misses with this workload yet. In the past I've instrumented the scheduler trace records in Xen to include performance counters such as instructions executed and cache / tlb misses, and then used xenalyze (http://xenbits.xensource.com/ext/xenalyze.hg) to analyze them. But the functionality for both capture and analysis was never standardized or added to mainline.
Quoted 59 lines
I'd be happy to help point you in the right direction if you're interested in investing in that approach. :-)
-George
Quoted 52 lines · Naresh Rapolu
Hello George,
How did you measure Cache/ TLB misses etc while using/profiling this new scheduler ? Any tool that you`ve used which works with Xen ?
Thanks,
Naresh Rapolu.
PhD Student, Computer Science,
Purdue University.Quoted 38 lines · George Dunlap
This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8
xen/common/sched_credit2.c| 1125
+++++++++++++++++++++++++++++xen/common/schedule.c| 22xen/include/public/domctl.h| 4xen/include/public/trace.h| 1 xen/include/xen/sched-if.h | 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel------------------------------------------------------------------------
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Well, sadly, credit2 doesn't seem to solve the problem... and even more sadly causes worse performance on my overcommitted workload.
elapsed is wallclock seconds from the time the first VM
launches the first "make clean" until the fourth VM finishes its second make.
sumvcpu is the sum of the vcpu sec (including dom0) reported
by xm list after all VM's have finished the workload and force-crashed
dom0 is the vcpu sec reported by xm list for dom0
credit: 5 test runs
elapsed=(9447,9388,9578,9576,9412)
sumvcpu=(13665,13671,13693,13589,13598)
dom0=(559,556,555,467,483)
sedf: 6 test runs
elapsed=(10022,9418,9637,12129,13599,11875)
sumvcpu=(13539,13514,13510,14270,14447,14237)
dom0=(473,468,460,482,537,475)
credit2: 6 test runs
elapsed=(11007,9931,10051,10090,11647,10070)
sumvcpu=(14878,14615,14610,14641,14886,14594)
dom0=(510,470,471,482,536,463)
P.S. physical machine is a single socket dual core
Quoted 89 lines
-----Original Message-----
From: George Dunlap [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 8:30 AM
To: Dan Magenheimer
Cc: [email protected]
Subject: Re: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)Keir has checked the patches in, so if you wait a bit, they should show up on the public repository.
The tool patch is only necessary for adjusting the weight; if you're OK using the default weight, just adding "sched=credit2" on the xen command-line should be fine.
Don't forget that this isn't meant to perform well on multiple sockets yet. :-)
-George
Quoted 9 lines · Dan Magenheimer
Hi George --
I'm seeing some problems applying the patches (such as "malformed patch"). If you could send me a monolithic patch in an attachment and tell me what cset in http://xenbits.xensource.com/xen-unstable.hg
that it successfully applies against, I will try to give my workload a test against it to see if it has the same symptoms.Quoted 24 lines
Also, do I need to apply the tools patch if I don't intend to specify any parameters, or is the xen patch + "sched=credit2" in a boot param sufficient?
Thanks,
DanQuoted 16 lines
-----Original Message-----
From: George Dunlap [mailto:[email protected]]
Sent: Wednesday, April 14, 2010 4:26 AM
To: [email protected]
Cc: [email protected]
Subject: [Xen-devel] [PATCH 0 of 5] Add credit2 scheduler (EXPERIMENTAL)This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development;
Quoted 1 line
Quoted 1 line
George Dunlap <[email protected]> or check out the wiki
page
Quoted 31 lines
Quoted 31 lines
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas
and status updates.19 files changed, 1453 insertions(+), 21 deletions(-) tools/libxc/Makefile | 1 tools/libxc/xc_csched2.c | 50 + tools/libxc/xenctrl.h | 8 tools/python/xen/lowlevel/xc/xc.c | 58 + tools/python/xen/xend/XendAPI.py | 3 tools/python/xen/xend/XendDomain.py | 54 + tools/python/xen/xend/XendDomainInfo.py | 4 tools/python/xen/xend/XendNode.py | 4 tools/python/xen/xend/XendVMMetrics.py | 1 tools/python/xen/xend/server/SrvDomain.py | 14 tools/python/xen/xm/main.py | 82 ++ xen/arch/ia64/vmx/vmmu.c | 6 xen/common/Makefile | 1 xen/common/sched_credit.c | 8 xen/common/sched_credit2.c | 1125 +++++++++++++++++++++++++++++ xen/common/schedule.c | 22 xen/include/public/domctl.h | 4 xen/include/public/trace.h | 1 xen/include/xen/sched-if.h | 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 24 lines · George Dunlap
Quoted 7 lines · Dulloor
As we talked before, I am interested in improving the mutiple-socket scenario and adding the load balancing functionalilty, which could provide an acceptable alternative to pinning vcpus to sockets (for my NUMA work). I am going over your patch right now, but what are your thoughts ?
That would be great -- my focus for the next several months will be setting up a testing infrastructure to automatically test performance of different workloads mixes so I can hone the algorithm and test regressions.
My idea with load balancing was to do this:
* One runqueue per L2 cache.
* Add code to calculate the load of a runqueue. Load would be the average (~integral) of (vcpus running + vcpus on runqueue). I was planning on doing accurate load calculation, rather than sample-based, and falling back to sample-based if accurate turned out to be too slow.
* Calculate the load contributed by various vcpus.
* At regular intervals, determine of some kind of balancing needs to be done by looking at the overall runqueue load and placing based on "contributory" load of each VCPU.
Quoted 1 line
Does that make sense? Thoughts?
Sounds good. I can see that the runq_map for all cpus point to the same run-queue (in make_runq_map). I will start there.
Quoted 56 lines
I have some old patches that calculated accurate load, I could dig them up if you wanted something to start with. (I don't think they'll apply cleanly at the moment.)
Thanks,
-GeorgeQuoted 47 lines
-dulloor
Quoted 41 lines · George Dunlap
This patch series introduces the credit2 scheduler. The first two patches
introduce changes necessary to allow the credit2 shared runqueue functionality
to work properly; the last two implement the functionality itself.The scheduler is still in the experimental phase. There's lots of opportunity to contribute with independent lines of development; email George Dunlap <[email protected]> or check out the wiki page
http://wiki.xensource.com/xenwiki/Credit2_Scheduler_Development for ideas and status updates.19 files changed, 1453 insertions(+), 21 deletions(-)
tools/libxc/Makefile| 1tools/libxc/xc_csched2.c| 50 +tools/libxc/xenctrl.h| 8tools/python/xen/lowlevel/xc/xc.c| 58 +tools/python/xen/xend/XendAPI.py| 3tools/python/xen/xend/XendDomain.py| 54 +tools/python/xen/xend/XendDomainInfo.py| 4tools/python/xen/xend/XendNode.py| 4tools/python/xen/xend/XendVMMetrics.py| 1tools/python/xen/xend/server/SrvDomain.py| 14tools/python/xen/xm/main.py| 82 ++xen/arch/ia64/vmx/vmmu.c| 6 xen/common/Makefile | 1xen/common/sched_credit.c| 8xen/common/sched_credit2.c| 1125
+++++++++++++++++++++++++++++xen/common/schedule.c| 22xen/include/public/domctl.h| 4xen/include/public/trace.h| 1xen/include/xen/sched-if.h| 28_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel