shadow threads with prio 0 / SCHED_NORMAL
RFC 1/1 shadow threads with prio 0 / SCHED_NORMAL
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under SCHED_NORMAL when in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not need SCHED_FIFO for this.
The patch (be careful, quick-hack!) addresses the prio level 0 in the ipipe patch, the nucleus/shadow subsystem, and the native skin. A quick test with the attached demo showed the expected behaviour so far: no lock-up during busy-waiting in secondary mode, prio-boost when holding the lock (visible via /proc/xenomai/sched), no obvious side effects.
Any comments? Does this break other things in a subtle way?
Jan
kernel/sched.c +1 -1
@@ -5779,7 +5779,7 @@ int ipipe_setscheduler_root (struct taskrunqueue_t *rq;int oldprio; -if (prio <1 || prio >MAX_RT_PRIO -1 ) +if (prio <0 || prio >MAX_RT_PRIO -1 )return -EINVAL ; rq =task_rq_lock (p, &flags);
nucleus/core.h +1 -1
@@ -38,7 +38,7 @@#define XNCORE_NR_PRIO (XNCORE_MAX_PRIO - XNCORE_MIN_PRIO + 2)/* Priority sub-range used by core APIs. */ -#define XNCORE_LOW_PRIO 1 +#define XNCORE_LOW_PRIO 0#define XNCORE_HIGH_PRIO 99/* Priority of IRQ servers in user-space. */
skins/native/task.c +8 -4
@@ -56,8 +56,10 @@ static void *rt_task_trampoline (void *clong err;/* Ok, this looks like weird, but we need this. */ - param.sched_priority =sched_get_priority_max (SCHED_FIFO ); -pthread_setschedparam (pthread_self (),SCHED_FIFO ,¶m); +if (iargs->prio >0 ) { + param.sched_priority =sched_get_priority_max (SCHED_FIFO ); +pthread_setschedparam (pthread_self (),SCHED_FIFO ,¶m); + }/* rt_task_delete requires asynchronous cancellation */ pthread_setcanceltype (PTHREAD_CANCEL_ASYNCHRONOUS ,NULL );
@@ -132,8 +134,10 @@ int rt_task_create (RT_TASK *task,pthread_attr_setstacksize (&thattr,stksize);if (!(mode &T_JOINABLE ))pthread_attr_setdetachstate (&thattr,PTHREAD_CREATE_DETACHED ); -pthread_attr_setschedpolicy (&thattr,SCHED_FIFO ); - param.sched_priority =sched_get_priority_max (SCHED_FIFO ); +if (prio >0 ) { +pthread_attr_setschedpolicy (&thattr,SCHED_FIFO ); + param.sched_priority =sched_get_priority_max (SCHED_FIFO ); + }pthread_attr_setschedparam (&thattr,¶m); err =pthread_create (&thid,&thattr,&rt_task_trampoline,&iargs);
nucleus/shadow.c +8 -6
@@ -120,7 +120,7 @@ static inline void request_syscall_restastatic inline void set_linux_task_priority (struct task_struct *p,int prio) { -if (rthal_setsched_root (p,SCHED_FIFO ,prio) <0 ) +if (rthal_setsched_root (p,prio ?SCHED_FIFO :SCHED_NORMAL ,prio) <0 )printk (KERN_WARNING "Xenomai: invalid Linux priority level: %d, task=%s\n" ,prio,p->comm); }
@@ -577,7 +577,7 @@ void xnshadow_relax (int notify) xnthread_user_pid(thread));#endif /* CONFIG_XENO_OPT_DEBUG */ cprio = thread->cprio <MAX_RT_PRIO ? thread->cprio :MAX_RT_PRIO -1 ; -rthal_reenter_root (get_switch_lock_owner (),SCHED_FIFO ,cprio ?:1 ); +rthal_reenter_root (get_switch_lock_owner (),cprio ?SCHED_FIFO :SCHED_NORMAL ,cprio);xnthread_inc_ssw (thread);/* Account for secondary mode switch. */
@@ -671,11 +671,13 @@ static int xnshadow_wait_completion (xncvoid xnshadow_exit (void ) { -rthal_reenter_root (get_switch_lock_owner (),SCHED_FIFO ,current->rt_priority); +rthal_reenter_root (get_switch_lock_owner (), + current->rt_priority ?SCHED_FIFO :SCHED_NORMAL , + current->rt_priority);do_exit (0 ); } -/*! +/*! * \fn int xnshadow_map(xnthread_t *thread, xncompletion_t __user *u_completion) * @internal * \brief Create a shadow thread context.
@@ -762,7 +764,7 @@ int xnshadow_map (xnthread_t *thread,xnarch_init_shadow_tcb (xnthread_archtcb (thread),thread,xnthread_name (thread)); prio =xnthread_base_priority (thread) <MAX_RT_PRIO ?xnthread_base_priority (thread) :MAX_RT_PRIO -1 ; -set_linux_task_priority (current,prio ?:1 ); +set_linux_task_priority (current,prio);xnshadow_thrptd (current) = thread;xnpod_suspend_thread (thread,XNRELAX ,XN_INFINITE ,NULL );
@@ -919,7 +921,7 @@ void xnshadow_renice (xnthread_t *threadrange, since the core pod's priority scale is a superset of Linux's priority scale. */ int prio = thread->cprio <MAX_RT_PRIO ? thread->cprio :MAX_RT_PRIO -1 ; -schedule_linux_call (LO_RENICE_REQ ,p,prio ?:1 ); +schedule_linux_call (LO_RENICE_REQ ,p,prio); }void xnshadow_suspend (xnthread_t *thread)
#include <native/task.h>
#include <native/mutex.h>
#include <native/timer.h>
#include <stdio.h>
#include <sys/mman.h>
RT_TASK task1, task2;RT_MUTEX mutex;
void spin(long long time)
{
long long timeout = rt_timer_read() + time;
while (rt_timer_read() < timeout);
}void func1(void *arg)
{
printf("switching to secondary\n");
spin(3000000000LL);
printf("back to primary\n");
rt_mutex_lock(&mutex, TM_INFINITE);
rt_task_sleep(3000000000LL);
spin(1000000000LL);
rt_mutex_unlock(&mutex);
printf("done with primary\n");
sleep(3);
spin(3000000000LL);
}void func2(void *arg)
{
rt_task_sleep(3500000000LL);
rt_mutex_lock(&mutex, TM_INFINITE);
rt_mutex_unlock(&mutex);
}int main()
{
mlockall(MCL_CURRENT|MCL_FUTURE);
rt_mutex_create(&mutex, NULL);
spin(3000000000LL);
printf("rt_task_spawn(task1) = %d\n",
rt_task_spawn(&task1, "mytask", 0, 0, T_JOINABLE, func1, NULL));
printf("rt_task_spawn(task2) = %d\n",
rt_task_spawn(&task2, "mytask2", 0, 10, T_JOINABLE, func2, NULL));
rt_task_join(&task1);
rt_task_join(&task2);
return 0;
}Quoted 9 lines · Jan Kiszka
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under
SCHED_NORMALwhen in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not needSCHED_FIFOfor this.
Quoted 8 lines
The patch (be careful, quick-hack!) addresses the prio level 0 in the ipipe patch, the nucleus/shadow subsystem, and the native skin. A quick test with the attached demo showed the expected behaviour so far: no lock-up during busy-waiting in secondary mode, prio-boost when holding the lock (visible via
/proc/xenomai/sched), no obvious side effects.Any comments? Does this break other things in a subtle way?
Normally, the fixes that went in 2.1 regarding the internal synchronization of the thread hardening/relaxing services should allow this since we don't care anymore of the underlying Linux scheduling class. I'm queuing this patch for further testing, but I basically agree with its purpose, since it's another step toward deep integration with Linux. Thanks.
Signature · 1 line
Philippe.
Quoted 9 lines · Jan Kiszka
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under
SCHED_NORMALwhen in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not needSCHED_FIFOfor this.
Quoted 7 lines
The patch (be careful, quick-hack!) addresses the prio level 0 in the ipipe patch, the nucleus/shadow subsystem, and the native skin. A quick test with the attached demo showed the expected behaviour so far: no lock-up during busy-waiting in secondary mode, prio-boost when holding the lock (visible via
/proc/xenomai/sched), no obvious side effects.Any comments? Does this break other things in a subtle way?
An initial comment on the general usage of this extension: since the threads running in SCHED_OTHER/SCHED_NORMAL mode are expected to run non-RT workloads while still being able to use the RT infrastructure for
communicating with the rest of the RT system, I think that the best
places for creating such hybrids are in the rt_task_shadow (native skin) and pthread_setschedparam (POSIX skin) calls, which would make it clear that a regular Linux thread is involved [and as such needs to be created by a normal call to pthread_create()], which also happens to benefit from the RT infrastructure mainly for communication/sychronization purpose.
Signature · 1 line
Philippe.
Quoted 18 lines · Philippe Gerum
Quoted 9 lines · Jan Kiszka
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under
SCHED_NORMALwhen in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not needSCHED_FIFOfor this.Quoted 7 lines
The patch (be careful, quick-hack!) addresses the prio level 0 in the ipipe patch, the nucleus/shadow subsystem, and the native skin. A quick test with the attached demo showed the expected behaviour so far: no lock-up during busy-waiting in secondary mode, prio-boost when holding the lock (visible via
/proc/xenomai/sched), no obvious side effects.Any comments? Does this break other things in a subtle way?
Quoted 9 lines
An initial comment on the general usage of this extension: since the threads running in SCHED_OTHER/SCHED_NORMAL mode are expected to run non-RT workloads while still being able to use the RT infrastructure for
communicating with the rest of the RT system, I think that the best
places for creating such hybrids are in thert_task_shadow(native skin) andpthread_setschedparam(POSIX skin) calls, which would make it clear that a regular Linux thread is involved [and as such needs to be created by a normal call topthread_create()], which also happens to benefit from the RT infrastructure mainly for communication/sychronization purpose.
What about keeping SCHED_RR as the default scheduling policy and requiring users to manually select SCHED_NORMAL in thread creation attributes in order to create hybrid threads with pthread_create ?
Signature · 1 line
Gilles Chanteperdrix.
Quoted 29 lines · Gilles Chanteperdrix
Quoted 18 lines · Philippe Gerum
Quoted 9 lines · Jan Kiszka
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under
SCHED_NORMALwhen in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not needSCHED_FIFOfor this.Quoted 7 lines
The patch (be careful, quick-hack!) addresses the prio level 0 in the ipipe patch, the nucleus/shadow subsystem, and the native skin. A quick test with the attached demo showed the expected behaviour so far: no lock-up during busy-waiting in secondary mode, prio-boost when holding the lock (visible via
/proc/xenomai/sched), no obvious side effects.Any comments? Does this break other things in a subtle way?
Quoted 9 lines
An initial comment on the general usage of this extension: since the threads running in SCHED_OTHER/SCHED_NORMAL mode are expected to run non-RT workloads while still being able to use the RT infrastructure for
communicating with the rest of the RT system, I think that the best
places for creating such hybrids are in thert_task_shadow(native skin) andpthread_setschedparam(POSIX skin) calls, which would make it clear that a regular Linux thread is involved [and as such needs to be created by a normal call topthread_create()], which also happens to benefit from the RT infrastructure mainly for communication/sychronization purpose.
Quoted 4 lines
What about keeping
SCHED_RRas the default scheduling policy and requiring users to manually selectSCHED_NORMALin thread creation attributes in order to create hybrid threads withpthread_create?
No objection a priori, but what would this buy us?
Signature · 1 line
Philippe.
Quoted 6 lines · Philippe Gerum
Quoted 4 lines
What about keeping
SCHED_RRas the default scheduling policy and requiring users to manually selectSCHED_NORMALin thread creation attributes in order to create hybrid threads withpthread_create?No objection a priori, but what would this buy us?
As a user, I would expect pthread_create and pthread_setschedparam to allow the same scheduling policies and priorities.
Signature · 1 line
Gilles Chanteperdrix.
Quoted 11 lines · Gilles Chanteperdrix
Quoted 6 lines · Philippe Gerum
Quoted 4 lines
What about keeping
SCHED_RRas the default scheduling policy and requiring users to manually selectSCHED_NORMALin thread creation attributes in order to create hybrid threads withpthread_create?No objection a priori, but what would this buy us?
As a user, I would expect
pthread_createandpthread_setschedparamto allow the same scheduling policies and priorities.
Is your concern about having the POSIX skin in kernel space currently assuming that SCHED_OTHER == SCHED_RR, which would be different than Linux's perception in that case?
However, I really think that we should not rely on implicit rules when we could avoid it (e.g. SCHED_RR as default), especially when those rules are going to be most often overlooked (people are rather used to default to SCHED_NORMAL/OTHER in the Linux sense).
What I suggested was to let people create normal threads using pthread_create (likely conforming to the SCHED_OTHER policy), then use the redirected pthread_setschedparam syscall (i.e. always applied to the current thread) to promote them as Xenomai shadows, but leave them in their original scheduling class. The same goes for rt_task_shadow. This would be explicit actions that would not leave much room for "surprises".
Signature · 1 line
Philippe.
Quoted 6 lines · Philippe Gerum
What I suggested was to let people create normal threads using
pthread_create(likely conforming to theSCHED_OTHERpolicy), then use the redirectedpthread_setschedparamsyscall (i.e. always applied to the current thread) to promote them as Xenomai shadows, but leave them in their original scheduling class. The same goes forrt_task_shadow. This would be explicit actions that would not leave much room for "surprises".
If I understand correctly, you mean that one should not be able to create real-time threads with pthread_create. My question was about what to do of explicit scheduling parameters passed to pthread_create through thread creation attributes.
Signature · 1 line
Gilles Chanteperdrix.
Quoted 8 lines · Gilles Chanteperdrix
Quoted 6 lines · Philippe Gerum
What I suggested was to let people create normal threads using
pthread_create(likely conforming to theSCHED_OTHERpolicy), then use the redirectedpthread_setschedparamsyscall (i.e. always applied to the current thread) to promote them as Xenomai shadows, but leave them in their original scheduling class. The same goes forrt_task_shadow. This would be explicit actions that would not leave much room for "surprises".
Quoted 4 lines
If I understand correctly, you mean that one should not be able to create real-time threads with
pthread_create. My question was about what to do of explicit scheduling parameters passed topthread_createthrough thread creation attributes.
Nope, this is obviously not what I meant... :o>
This is how one would create hybrids, without changing anything else to the current interface:
pthread_attr_init(&attr);
...
pthread_attr_setschedpolicy(&attr,SCHED_OTHER);
...
pthread_create(...,&attr,&foo,NULL);and,
void *foo (void *cookie)
{
/* The following call maps a shadow thread to "current", but
currently only accepts SCHED_FIFO, and would be changed to
allow SCHED_OTHER/NORMAL. */
pthread_setschedparam(...,SCHED_OTHER,...);
/* OR, for the native API */
rt_task_shadow(...,pri=0,...);
}Signature · 1 line
Philippe.
Quoted 29 lines · Philippe Gerum
Quoted 9 lines · Gilles Chanteperdrix
Quoted 1 line · Philippe Gerum
What I suggested was to let people create normal threads using >
pthread_create(likely conforming to theSCHED_OTHERpolicy), then useQuoted 1 line
the redirected
pthread_setschedparamsyscall (i.e. always appliedto the > current thread) to promote them as Xenomai shadows, but leave them in > their original scheduling class. The same goes for
rt_task_shadow. This > would be explicit actions that would not leave much room for "surprises".Quoted 4 lines
If I understand correctly, you mean that one should not be able to create real-time threads with
pthread_create. My question was about what to do of explicit scheduling parameters passed topthread_createthrough thread creation attributes.Nope, this is obviously not what I meant... :o>
This is how one would create hybrids, without changing anything else to the current interface:
pthread_attr_init(&attr); ... pthread_attr_setschedpolicy(&attr,SCHED_OTHER); ... pthread_create(...,&attr,&foo,NULL);and,
void *foo (void *cookie)
Quoted 9 lines
{
/* The following call maps a shadow thread to "current", but
currently only accepts SCHED_FIFO, and would be changed to
allow SCHED_OTHER/NORMAL. */
pthread_setschedparam(...,SCHED_OTHER,...);
/* OR, for the native API */
rt_task_shadow(...,pri=0,...);
}I think it melts down to the question: which kind of SCHED_OTHER threads will be more common in applications, the hybrid or the normal ones? Remember that we also still have the __real_pthread_create interface for creating /really/ normal threads...
The advantage of making hybrids default, even for the attr=NULL case, would be that the user will be able to interact with real-time SCHED_FIFO/RR threads without further coding effort (might be interesting for porting existing apps...).
How much overhead would default-shadowing introduce when it is not used actively? Is it just the kernel pthread structure? Maybe we should consider allocating that structure from normal kernel memory to save rt-memory resources (under memory pressure, there is a risk in failing to create the Linux mate anyway). I could also imagine that SCHED_OTHER threads will get started in relaxed mode by default so that the Xenomai scheduler will not be additionally loaded.
Jan
Quoted 48 lines · Jan Kiszka
Quoted 30 lines · Philippe Gerum
Quoted 9 lines · Gilles Chanteperdrix
Quoted 1 line · Philippe Gerum
What I suggested was to let people create normal threads using >
pthread_create(likely conforming to theSCHED_OTHERpolicy), then useQuoted 1 line
the redirected
pthread_setschedparamsyscall (i.e. always appliedto the > current thread) to promote them as Xenomai shadows, but leave them in > their original scheduling class. The same goes for
rt_task_shadow. This > would be explicit actions that would not leave much room for "surprises".Quoted 4 lines
If I understand correctly, you mean that one should not be able to create real-time threads with
pthread_create. My question was about what to do of explicit scheduling parameters passed topthread_createthrough thread creation attributes.Nope, this is obviously not what I meant... :o>
This is how one would create hybrids, without changing anything else to the current interface:
pthread_attr_init(&attr); ... pthread_attr_setschedpolicy(&attr,SCHED_OTHER); ... pthread_create(...,&attr,&foo,NULL);and,
void *foo (void *cookie)
Quoted 9 lines
{ /* The following call maps a shadow thread to "current", but currently only accepts SCHED_FIFO, and would be changed to allow SCHED_OTHER/NORMAL. */ pthread_setschedparam(...,SCHED_OTHER,...); /* OR, for the native API */ rt_task_shadow(...,pri=0,...); }I think it melts down to the question: which kind of
SCHED_OTHERthreads will be more common in applications, the hybrid or the normal ones? Remember that we also still have the__real_pthread_createinterface for creating /really/ normal threads...
Quoted 5 lines
The advantage of making hybrids default, even for the attr=NULL case, would be that the user will be able to interact with real-time SCHED_FIFO/RR threads without further coding effort (might be interesting for porting existing apps...).
That's POSIX centric and seems acceptable, but the question remains open for the native API. Passing a null priority to rt_task_create would be way too confusing (i.e. a priority value should be an attribute of the scheduling class, not the other way around), hence the use of rt_task_shadow for this particular purpose.
Quoted 5 lines
How much overhead would default-shadowing introduce when it is not used actively? Is it just the kernel pthread structure? Maybe we should consider allocating that structure from normal kernel memory to save rt-memory resources (under memory pressure, there is a risk in failing to create the Linux mate anyway).
Memory is not an issue anyway, since we need a full shadow TCB to allow the underlying thread to wait for sync objects, and generally speaking, to use the RT infrastructure to its full extent. IOW, there is nothing to save here, an hybrid thread would just be a regular Xenomai shadow that _only_ happens to undergo the SCHED_OTHER policy.
I could also imagine that SCHED_OTHER
Quoted 3 lines
threads will get started in relaxed mode by default so that the Xenomai scheduler will not be additionally loaded.
Yes, we could do that. But in any case, an hybrid would be allowed to switch to primary for the purpose of pending on synchronization objects, which in turn would give hybrids a higher priority than regular Linux SCHED_FIFO threads (i.e. threads not mapped to a Xenomai shadow) in this mode.
Signature · 1 line
Philippe.
Quoted 29 lines · Gilles Chanteperdrix
Quoted 18 lines · Philippe Gerum
Quoted 9 lines · Jan Kiszka
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under
SCHED_NORMALwhen in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not needSCHED_FIFOfor this.Quoted 7 lines
The patch (be careful, quick-hack!) addresses the prio level 0 in the ipipe patch, the nucleus/shadow subsystem, and the native skin. A quick test with the attached demo showed the expected behaviour so far: no lock-up during busy-waiting in secondary mode, prio-boost when holding the lock (visible via
/proc/xenomai/sched), no obvious side effects.Any comments? Does this break other things in a subtle way?
Quoted 9 lines
An initial comment on the general usage of this extension: since the threads running in SCHED_OTHER/SCHED_NORMAL mode are expected to run non-RT workloads while still being able to use the RT infrastructure for
communicating with the rest of the RT system, I think that the best
places for creating such hybrids are in thert_task_shadow(native skin) andpthread_setschedparam(POSIX skin) calls, which would make it clear that a regular Linux thread is involved [and as such needs to be created by a normal call topthread_create()], which also happens to benefit from the RT infrastructure mainly for communication/sychronization purpose.
Quoted 4 lines
What about keeping
SCHED_RRas the default scheduling policy and requiring users to manually selectSCHED_NORMALin thread creation attributes in order to create hybrid threads withpthread_create?
Considering that Linux uses SCHED_OTHER as default, I think applying the same on the POSIX skin would follow the "least surprise" principle.
Jan
Quoted 8 lines · Jan Kiszka
Hi,
this is an experimental hack to open the non-rt priority levels of Linux to Xenomai shadow threads, i.e. allow shadows to be scheduled under
SCHED_NORMALwhen in secondary mode. The scenario are typical borderline threads between RT and non-RT: they share a critical code path with RT threads, maybe mutex protected, but they are mostly time-sharing threads which do not needSCHED_FIFOfor this.
Merged the nucleus and native API changes, thanks. The POSIX skin still needs to be updated to use those new timesharing Xenomai-enabled threads though.
Signature · 1 line
Philippe.