Bug-fixes and new features for 2.6.34-rc1
Subsystems (2)
cover Bug-fixes and new features for 2.6.34-rc1
Hi,
There are some patches which I've had in my branch for some time and I would like to get them merged by 2.6.34.
The first three are fixes to allow Linux to work better on SMP systems. The next two are improvements for the ARMv7. The last patch remove the use of domains for ARMv6k and ARMv7 processors. One of the reasons is that because of domain switching, we can get speculative fetches from I/O areas.
Catalin Marinas (6):
Global ASID allocation on SMP
Broadcast the DMA cache operations on ARMv6 SMP hardware Fix a race in the vfp_notifier() function on SMP systems
ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
ARMv7: Improved page table format with TRE and AFE
Remove the domain switching on ARMv6k/v7 CPUs
arch/arm/include/asm/assembler.h | 9 +- arch/arm/include/asm/cacheflush.h | 29 ++++++++ arch/arm/include/asm/domain.h | 31 ++++++++ arch/arm/include/asm/futex.h | 9 +- arch/arm/include/asm/memory.h | 6 +- arch/arm/include/asm/mmu.h | 1 arch/arm/include/asm/mmu_context.h | 15 ++++ arch/arm/include/asm/page.h | 8 ++ arch/arm/include/asm/pgalloc.h | 10 ++- arch/arm/include/asm/pgtable.h | 117 ++++++++++++++++++++++++++++---- arch/arm/include/asm/smp_plat.h | 9 ++ arch/arm/include/asm/uaccess.h | 16 ++-- arch/arm/kernel/entry-armv.S | 4 + arch/arm/kernel/smp.c | 133 ++++++++++++++++++++++++++++++++++++ arch/arm/kernel/traps.c | 17 +++++ arch/arm/lib/getuser.S | 13 ++-- arch/arm/lib/putuser.S | 29 ++++---- arch/arm/lib/uaccess.S | 83 +++++++++++----------- arch/arm/mm/Kconfig | 26 +++++++ arch/arm/mm/context.c | 120 +++++++++++++++++++++++++++++--- arch/arm/mm/dma-mapping.c | 20 ++++- arch/arm/mm/fault-armv.c | 2 - arch/arm/mm/fault.c | 10 +++ arch/arm/mm/flush.c | 9 +- arch/arm/mm/mmu.c | 7 +- arch/arm/mm/proc-v7.S | 58 ++++++---------- arch/arm/vfp/vfpmodule.c | 25 ++++++- 27 files changed, 647 insertions(+), 169 deletions(-)
Signature · 1 line
Catalin
PATCH 1/6 Global ASID allocation on SMP
The current ASID allocation algorithm doesn't ensure the notification of the other CPUs when the ASID rolls over. This may lead to two processes using the same ASID (but different generation) or multiple threads of the same process using different ASIDs.
This patch adds the broadcasting of the ASID rollover event to the other CPUs. To avoid a race on multiple CPUs modifying "cpu_last_asid" during the handling of the broadcast, the ASID numbering now starts at "smp_processor_id() + 1". At rollover, the cpu_last_asid will be set to NR_CPUS.
--- arch/arm/include/asm/mmu.h | 1 arch/arm/include/asm/mmu_context.h | 15 +++++ arch/arm/mm/context.c | 120 ++++++++++++++++++++++++++++++++---- 3 files changed, 122 insertions(+), 14 deletions(-)
arch/arm/include/asm/mmu.h +1 -0
@@ -6,6 +6,7 @@typedef struct {#ifdef CONFIG_CPU_HAS_ASID unsigned int id; +spinlock_t id_lock;#endif unsigned int kvm_seq; }mm_context_t ;
arch/arm/include/asm/mmu_context.h +15 -0
@@ -43,12 +43,23 @@ void __check_kvm_seq(struct mm_struct *mm);#define ASID_FIRST_VERSION (1 << ASID_BITS)extern unsigned int cpu_last_asid; +#ifdef CONFIG_SMP +DECLARE_PER_CPU(struct mm_struct *, current_mm); +#endif void __init_new_context (struct task_struct *tsk,struct mm_struct *mm);void __new_context (struct mm_struct *mm);static inline void check_context (struct mm_struct *mm) { +/* +* This code is executed with interrupts enabled. Therefore, +* mm->context.id cannot be updated to the latest ASID version +* on a different CPU (and condition below not triggered) +* without first getting an IPI to reset the context. The +* alternative is to take a read_lock on mm->context.id_lock +* (after changing its type to rwlock_t). +*/ if (unlikely ((mm->context.id ^ cpu_last_asid) >>ASID_BITS ))__new_context (mm);
@@ -108,6 +119,10 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,__flush_icache_all ();#endif if (!cpumask_test_and_set_cpu (cpu,mm_cpumask (next)) || prev != next) { +#ifdef CONFIG_SMP +struct mm_struct **crt_mm = &per_cpu (current_mm, cpu); + *crt_mm = next; +#endif check_context (next);cpu_switch_mm (next->pgd, next);if (cache_is_vivt ())
arch/arm/mm/context.c +106 -14
151 lines not on this page · raw message
PATCH 2/6 Broadcast the DMA cache operations on ARMv6 SMP hardware
In reply to Catalin MarinasThe Snoop Control Unit on the ARM11MPCore hardware does not detect the cache operations and the dma_cache_maint() function may leave stale cache entries on other CPUs. The solution is to broadcast the cache operations to the other CPUs in software. However, there is no restriction to the contexts in which dma_cache_maint() function can be called (interrupt context or IRQs disabled).
This patch implements the smp_dma_cache_op() function which performs the broadcast and it can be called with interrupts disabled or from interrupt context.
To avoid deadlocking when more than one CPU try to invoke this function, the implementation uses spin_trylock() loop if the IRQs are disabled and, if the lock cannot be acquired, it polls for an incoming IPI and executes it. In the unlikely situation of two or more CPUs calling the smp_dma_cache_op() function with interrupts disabled, there may be spurious (or delayed) IPIs after a CPU completes and enables the IRQs. These are handled by checking the corresponding "unfinished" bits in the IPI handler.
---
Just a note - the DMA cache ops broadcasting in software cannot easily use the generic IPI functionality in the kernel because of the restriction to have the interrupts enabled when invoking smp_call_function(). Another reason to do it separately is that the introduced smp_dma_cache_op() function runs the DMA cache operation locally in parallel with the other CPUs while smp_call_function() would only run it on the other CPUs in parallel but not with the current CPU.
arch/arm/include/asm/cacheflush.h | 29 ++++++++ arch/arm/kernel/smp.c | 133 +++++++++++++++++++++++++++++++++++++ arch/arm/mm/Kconfig | 5 + arch/arm/mm/dma-mapping.c | 14 ++-- 4 files changed, 174 insertions(+), 7 deletions(-)
arch/arm/include/asm/cacheflush.h +29 -0
@@ -280,6 +280,35 @@ extern void dmac_flush_range(const void *, const void *);#endif +#ifdef CONFIG_CPU_NO_CACHE_BCAST +enum smp_dma_cache_type { +SMP_DMA_CACHE_INV , +SMP_DMA_CACHE_CLEAN , +SMP_DMA_CACHE_FLUSH , +}; + +extern void smp_dma_cache_op (int type,const void *start,const void *end); + +static inline void smp_dma_inv_range (const void *start,const void *end) +{ +smp_dma_cache_op (SMP_DMA_CACHE_INV , start, end); +} + +static inline void smp_dma_clean_range (const void *start,const void *end) +{ +smp_dma_cache_op (SMP_DMA_CACHE_CLEAN , start, end); +} + +static inline void smp_dma_flush_range (const void *start,const void *end) +{ +smp_dma_cache_op (SMP_DMA_CACHE_FLUSH , start, end); +} +#else +#define smp_dma_inv_range dmac_inv_range +#define smp_dma_clean_range dmac_clean_range +#define smp_dma_flush_range dmac_flush_range +#endif +#ifdef CONFIG_OUTER_CACHE extern struct outer_cache_fns outer_cache;
arch/arm/kernel/smp.c +133 -0
154 lines not on this page · raw message
arch/arm/mm/Kconfig +5 -0
@@ -516,6 +516,11 @@ config CPU_CACHE_VIPT config CPU_CACHE_FA bool +config CPU_NO_CACHE_BCAST + bool + depends on SMP + default y if CPU_V6 + if MMU # The copy-page model config CPU_COPY_V3
arch/arm/mm/dma-mapping.c +7 -7
@@ -219,7 +219,7 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, {void *ptr =page_address (page);memset (ptr,0 , size); -dmac_flush_range (ptr, ptr + size); +smp_dma_flush_range (ptr, ptr + size);outer_flush_range (__pa (ptr),__pa (ptr) + size); }
@@ -548,15 +548,15 @@ void dma_cache_maint(const void *start, size_t size, int direction)switch (direction) {case DMA_FROM_DEVICE :/* invalidate only */ - inner_op = dmac_inv_range; + inner_op = smp_dma_inv_range; outer_op = outer_inv_range;break ;case DMA_TO_DEVICE :/* writeback only */ - inner_op = dmac_clean_range; + inner_op = smp_dma_clean_range; outer_op = outer_clean_range;break ;case DMA_BIDIRECTIONAL :/* writeback and invalidate */ - inner_op = dmac_flush_range; + inner_op = smp_dma_flush_range; outer_op = outer_flush_range;break ;default :
@@ -578,15 +578,15 @@ static void dma_cache_maint_contiguous(struct page *page, unsigned long offset,switch (direction) {case DMA_FROM_DEVICE :/* invalidate only */ - inner_op = dmac_inv_range; + inner_op = smp_dma_inv_range; outer_op = outer_inv_range;break ;case DMA_TO_DEVICE :/* writeback only */ - inner_op = dmac_clean_range; + inner_op = smp_dma_clean_range; outer_op = outer_clean_range;break ;case DMA_BIDIRECTIONAL :/* writeback and invalidate */ - inner_op = dmac_flush_range; + inner_op = smp_dma_flush_range; outer_op = outer_flush_range;break ;default :
PATCH 3/6 Fix a race in the vfp_notifier() function on SMP systems
In reply to Catalin MarinasThe vfp_notifier(THREAD_NOTIFY_RELEASE) maybe be called with thread->cpu different from the current one, causing a race condition with both the THREAD_NOTIFY_SWITCH path and vfp_support_entry().
--- arch/arm/vfp/vfpmodule.c | 25 ++++++++++++++++++++++--- 1 files changed, 22 insertions(+), 3 deletions(-)
arch/arm/vfp/vfpmodule.c +22 -3
@@ -14,6 +14,7 @@#include <linux/signal.h> #include <linux/sched.h> #include <linux/init.h> +#include <linux/rcupdate.h> #include <asm/thread_notify.h> #include <asm/vfp.h>
@@ -49,14 +50,21 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)#ifdef CONFIG_SMP /* +* RCU locking is needed in case last_VFP_context[cpu] is +* released on a different CPU. +*/ +rcu_read_lock (); + vfp = last_VFP_context[cpu]; +/* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ -if ((fpexc &FPEXC_EN ) && last_VFP_context[cpu]) { -vfp_save_state (last_VFP_context[cpu], fpexc); - last_VFP_context[cpu]->hard.cpu = cpu; +if ((fpexc &FPEXC_EN ) && vfp) { +vfp_save_state (vfp, fpexc); + vfp->hard.cpu = cpu; } +rcu_read_unlock ();/* * Thread migration, just force the reloading of the * state on the new CPU in case the VFP registers
@@ -91,8 +99,19 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) }/* flush and release case: Per-thread VFP cleanup. */ +#ifndef CONFIG_SMP if (last_VFP_context[cpu] == vfp) last_VFP_context[cpu] =NULL ; +#else +/* +* Since release_thread() may be called from a different CPU, we use +* cmpxchg() here to avoid a race with the vfp_support_entry() code +* which modifies last_VFP_context[cpu]. Note that on SMP systems, a +* STR instruction on a different CPU clears the global exclusive +* monitor state. +*/ + (void )cmpxchg (&last_VFP_context[cpu], vfp,NULL ); +#endif return NOTIFY_DONE ; }
PATCH 3/6 Fix a race in the vfp_notifier() function on SMP systems
Quoted 3 lines · Catalin Marinas
The
vfp_notifier(THREAD_NOTIFY_RELEASE)maybe be called withthread->cpudifferent from the current one, causing a race condition with both theTHREAD_NOTIFY_SWITCHpath andvfp_support_entry().
The only call where thread->cpu may not be the current CPU is in the THREAD_NOFTIFY_RELEASE case.
When called in the THREAD_NOTIFY_SWITCH case, we are switching to the specified thread, and thread->cpu better be smp_processor_id() or else we're saving our CPUs VFP state into some other CPUs currently running thread.
Not only that, but the thread we're switching away from will still be 'owned' by the CPU we're running on, and can't be scheduled onto another CPU without this function first completing, nor can it be flushed nor released.
arch/arm/vfp/vfpmodule.c static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) · line 51
Quoted 3 lines
arch/arm/vfp/vfpmodule.c +0
-0@@ -49,14 +50,21 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)#ifdef CONFIG_SMP
BUG_ON(cpu != smp_processor_id());
since it would be very bad if this was any different. Note that this is also a non-preemptible context - we're called from the scheduler, and the scheduler can't be preempted mid-thead switch.
arch/arm/vfp/vfpmodule.c static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) · line 56
Quoted 5 lines
arch/arm/vfp/vfpmodule.c +4
-0/* +* RCU locking is needed in case last_VFP_context[cpu] is +* released on a different CPU. +*/ +rcu_read_lock ();
Given that we're modifying our CPUs last_VFP_context here, I don't see what the RCU locks give us - the thread we're switching to can _not_ be being released at this time - we can't be switching to a dead task. Not only that, but this notifier is already called under the RCU lock, so this is a no-op.
Quoted 24 lines
arch/arm/vfp/vfpmodule.c +7
-3+ vfp = last_VFP_context[cpu]; +/* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ -if ((fpexc &FPEXC_EN ) && last_VFP_context[cpu]) { -vfp_save_state (last_VFP_context[cpu], fpexc); - last_VFP_context[cpu]->hard.cpu = cpu; +if ((fpexc &FPEXC_EN ) && vfp) { +vfp_save_state (vfp, fpexc); + vfp->hard.cpu = cpu; } +rcu_read_unlock ();/* * Thread migration, just force the reloading of the * state on the new CPU in case the VFP registers @@ -91,8 +99,19 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) }/* flush and release case: Per-thread VFP cleanup. */ +#ifndef CONFIG_SMP if (last_VFP_context[cpu] == vfp) last_VFP_context[cpu] =NULL ;
arch/arm/vfp/vfpmodule.c static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) · line 114
Quoted 10 lines
arch/arm/vfp/vfpmodule.c +10
-0+#else +/* +* Since release_thread() may be called from a different CPU, we use +* cmpxchg() here to avoid a race with the vfp_support_entry() code +* which modifies last_VFP_context[cpu]. Note that on SMP systems, a +* STR instruction on a different CPU clears the global exclusive +* monitor state. +*/ + (void )cmpxchg (&last_VFP_context[cpu], vfp,NULL ); +#endif
I think this hunk is the only part which makes sense.
PATCH 3/6 Fix a race in the vfp_notifier() function on SMP systems
Quoted 13 lines · Russell King - ARM Linux
Quoted 3 lines · Catalin Marinas
The
vfp_notifier(THREAD_NOTIFY_RELEASE)maybe be called withthread->cpudifferent from the current one, causing a race condition with both theTHREAD_NOTIFY_SWITCHpath andvfp_support_entry().The only call where
thread->cpumay not be the current CPU is in theTHREAD_NOFTIFY_RELEASEcase.When called in the
THREAD_NOTIFY_SWITCHcase, we are switching to the specified thread, andthread->cpubetter besmp_processor_id()or else we're saving our CPUs VFP state into some other CPUs currently running thread.
Quoted 4 lines
Not only that, but the thread we're switching away from will still be 'owned' by the CPU we're running on, and can't be scheduled onto another CPU without this function first completing, nor can it be flushed nor released.
Here's a patch which adds this documentation, and fixes the THREAD_NOTIFY_FLUSH case - since that could be preempted.
arch/arm/vfp/vfpmodule.c +62 -21
@@ -38,16 +38,72 @@ union vfp_state *last_VFP_context[NR_CPUS];*/ unsigned int VFP_arch; +/* +* Per-thread VFP initialization. +*/ +static void vfp_thread_flush (struct thread_info *thread) +{ +union vfp_state *vfp = &thread->vfpstate; +unsigned int cpu; + +memset (vfp,0 ,sizeof (union vfp_state )); + + vfp->hard.fpexc =FPEXC_EN ; + vfp->hard.fpscr =FPSCR_ROUND_NEAREST ; + +/* +* Disable VFP to ensure we initialize it first. We must ensure +* that the modification of last_VFP_context[] and hardware disable +* are done for the same CPU and without preemption. +*/ + cpu =get_cpu (); +if (last_VFP_context[cpu] == vfp) + last_VFP_context[cpu] =NULL ; +fmxr (FPEXC ,fmrx (FPEXC ) & ~FPEXC_EN ); +put_cpu (); +} + +static void vfp_thread_release (struct thread_info *thread) +{ +/* release case: Per-thread VFP cleanup. */ +union vfp_state *vfp = &thread->vfpstate; +unsigned int cpu = thread->cpu; + +if (last_VFP_context[cpu] == vfp) + last_VFP_context[cpu] =NULL ; +} + +/* +* When this function is called with the following 'cmd's, the following +* is true while this function is being run: +* THREAD_NOFTIFY_SWTICH: +* - the previously running thread will not be scheduled onto another CPU. +* - the next thread to be run (v) will not be running on another CPU. +* - thread->cpu is the local CPU number +* - not preemptible as we're called in the middle of a thread switch +* THREAD_NOTIFY_FLUSH: +* - the thread (v) will be running on the local CPU, so +* v === current_thread_info() +* - thread->cpu is the local CPU number at the time it is accessed, +* but may change at any time. +* - we could be preempted if tree preempt rcu is enabled, so +* it is unsafe to use thread->cpu. +* THREAD_NOTIFY_RELEASE: +* - the thread (v) will not be running on any CPU; it is a dead thread. +* - thread->cpu will be the last CPU the thread ran on, which may not +* be the current CPU. +* - we could be preempted if tree preempt rcu is enabled. +*/ static int vfp_notifier (struct notifier_block *self,unsigned long cmd,void *v) {struct thread_info *thread = v; -union vfp_state *vfp; -__u32 cpu = thread->cpu;if (likely (cmd ==THREAD_NOTIFY_SWITCH )) {u32 fpexc =fmrx (FPEXC );#ifdef CONFIG_SMP +unsigned int cpu = thread->cpu; +/* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The
@@ -74,25 +130,10 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)return NOTIFY_DONE ; } - vfp = &thread->vfpstate; -if (cmd ==THREAD_NOTIFY_FLUSH ) { -/* -* Per-thread VFP initialisation. -*/ -memset (vfp,0 ,sizeof (union vfp_state )); - - vfp->hard.fpexc =FPEXC_EN ; - vfp->hard.fpscr =FPSCR_ROUND_NEAREST ; - -/* -* Disable VFP to ensure we initialise it first. -*/ -fmxr (FPEXC ,fmrx (FPEXC ) & ~FPEXC_EN ); - } - -/* flush and release case: Per-thread VFP cleanup. */ -if (last_VFP_context[cpu] == vfp) - last_VFP_context[cpu] =NULL ; +if (cmd ==THREAD_NOTIFY_FLUSH ) +vfp_thread_flush (thread); +else +vfp_thread_release (thread);return NOTIFY_DONE ; }
PATCH 3/6 Fix a race in the vfp_notifier() function on SMP systems
Quoted 21 lines · Russell King - ARM Linux
Quoted 13 lines · Russell King - ARM Linux
Quoted 3 lines · Catalin Marinas
The
vfp_notifier(THREAD_NOTIFY_RELEASE)maybe be called withthread->cpudifferent from the current one, causing a race condition with both theTHREAD_NOTIFY_SWITCHpath andvfp_support_entry().The only call where
thread->cpumay not be the current CPU is in theTHREAD_NOFTIFY_RELEASEcase.When called in the
THREAD_NOTIFY_SWITCHcase, we are switching to the specified thread, andthread->cpubetter besmp_processor_id()or else we're saving our CPUs VFP state into some other CPUs currently running thread.Quoted 4 lines
Not only that, but the thread we're switching away from will still be 'owned' by the CPU we're running on, and can't be scheduled onto another CPU without this function first completing, nor can it be flushed nor released.
Here's a patch which adds this documentation, and fixes the
THREAD_NOTIFY_FLUSHcase - since that could be preempted.
[...]
Quoted 2 lines
+ *
THREAD_NOFTIFY_SWTICH:
+ * - the previously running thread will not be scheduled onto another CPU.
While this comment is certainly true, I don't think it is relevant since we aren't always switching the VFP context from the thread being switched out but it may be a thread that ran much earlier.
Otherwise the patch is fine.
Signature · 1 line
Catalin
PATCH 3/6 Fix a race in the vfp_notifier() function on SMP systems
In reply to Russell King - ARM LinuxQuoted 7 lines · Russell King - ARM Linux
Quoted 3 lines · Catalin Marinas
The
vfp_notifier(THREAD_NOTIFY_RELEASE)maybe be called withthread->cpudifferent from the current one, causing a race condition with both theTHREAD_NOTIFY_SWITCHpath andvfp_support_entry().The only call where
thread->cpumay not be the current CPU is in theTHREAD_NOFTIFY_RELEASEcase.
Correct.
Quoted 4 lines
When called in the
THREAD_NOTIFY_SWITCHcase, we are switching to the specified thread, andthread->cpubetter besmp_processor_id()or else we're saving our CPUs VFP state into some other CPUs currently running thread.
Also correct.
Quoted 4 lines
Not only that, but the thread we're switching away from will still be 'owned' by the CPU we're running on, and can't be scheduled onto another CPU without this function first completing, nor can it be flushed nor released.
Correct but see below.
Quoted 11 lines
arch/arm/vfp/vfpmodule.c static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) · line 56
Quoted 5 lines
arch/arm/vfp/vfpmodule.c +4
-0/* +* RCU locking is needed in case last_VFP_context[cpu] is +* released on a different CPU. +*/ +rcu_read_lock ();Given that we're modifying our CPUs
last_VFP_contexthere, I don't see what the RCU locks give us - the thread we're switching to can _not_ be being released at this time - we can't be switching to a dead task. Not only that, but this notifier is already called under the RCU lock, so this is a no-op.
With the current implementation, the last_CPU_context is set when the CPU takes an undef for a VFP instruction and not during every context switch, so last_VFP_context may *not* point to the task we are switching away from. The RCU locking was added to prevent the task structure (and thread_info) pointed to by last_VFP_context from being freed while executing the switch between two tasks other than the one released.
If the region is RCU locked anyway, we can simply add a comment but unless we change how last_CPU_context is set, we still need this check.
Signature · 1 line
Catalin
PATCH 3/6 Fix a race in the vfp_notifier() function on SMPsystems
Quoted 7 lines · Catalin Marinas
Quoted 6 lines · Russell King - ARM Linux
Quoted 5 lines · Catalin Marinas
/*(quoted diff) +4
-0+ * RCU locking is needed in case last_VFP_context[cpu] is + * released on a different CPU. + */ + rcu_read_lock();
[...]
Quoted 2 lines
Quoted 2 lines
Not only that, but this notifier is already called under the RCU lock, so this is a no-op.
[...]
Quoted 2 lines
If the region is RCU locked anyway, we can simply add a comment but unless we change how
last_CPU_contextis set, we still need this check.
I updated patch below which doesn't take the RCU lock but adds a comment.
Apart from the last hunk with which you are OK, the patch makes sure that last_VFP_context[cpu] is only read once and stored to a local variable otherwise you may have the surprise that it becomes NULL if the thread that was using it is released on a different CPU (that's actually the failure we were getting under stress testing).
Fix a race in the vfp_notifier() function on SMP systems
From: Catalin Marinas <[email protected]>
The vfp_notifier(THREAD_NOTIFY_RELEASE) maybe be called with thread->cpu different from the current one, causing a race condition with both the THREAD_NOTIFY_SWITCH path and vfp_support_entry().
--- arch/arm/vfp/vfpmodule.c | 26 +++++++++++++++++++++++--- 1 files changed, 23 insertions(+), 3 deletions(-)
arch/arm/vfp/vfpmodule.c +23 -3
@@ -14,6 +14,7 @@#include <linux/signal.h> #include <linux/sched.h> #include <linux/init.h> +#include <linux/rcupdate.h> #include <asm/thread_notify.h> #include <asm/vfp.h>
@@ -49,14 +50,22 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)#ifdef CONFIG_SMP /* +* The vfpstate structure pointed to by last_VFP_context[cpu] +* may be released via call_rcu(delayed_put_task_struct) but +* atomic_notifier_call_chain() already holds the RCU lock. +*/ + vfp = last_VFP_context[cpu]; + +/* * On SMP, if VFP is enabled, save the old state in * case the thread migrates to a different CPU. The * restoring is done lazily. */ -if ((fpexc &FPEXC_EN ) && last_VFP_context[cpu]) { -vfp_save_state (last_VFP_context[cpu], fpexc); - last_VFP_context[cpu]->hard.cpu = cpu; +if ((fpexc &FPEXC_EN ) && vfp) { +vfp_save_state (vfp, fpexc); + vfp->hard.cpu = cpu; } +/* * Thread migration, just force the reloading of the * state on the new CPU in case the VFP registers
@@ -91,8 +100,19 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) }/* flush and release case: Per-thread VFP cleanup. */ +#ifndef CONFIG_SMP if (last_VFP_context[cpu] == vfp) last_VFP_context[cpu] =NULL ; +#else +/* +* Since release_thread() may be called from a different CPU, we use +* cmpxchg() here to avoid a race with the vfp_support_entry() code +* which modifies last_VFP_context[cpu]. Note that on SMP systems, a +* STR instruction on a different CPU clears the global exclusive +* monitor state. +*/ + (void )cmpxchg (&last_VFP_context[cpu], vfp,NULL ); +#endif return NOTIFY_DONE ; }
Signature · 1 line
Catalin
PATCH 4/6 ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
In reply to Catalin MarinasARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
--- arch/arm/include/asm/smp_plat.h | 9 +++++++++ arch/arm/mm/fault-armv.c | 2 -- arch/arm/mm/flush.c | 9 ++++----- 3 files changed, 13 insertions(+), 7 deletions(-)
arch/arm/include/asm/smp_plat.h +9 -0
@@ -13,4 +13,13 @@ static inline int tlb_ops_need_broadcast(void) return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; } +#ifndef CONFIG_SMP +#define cache_ops_need_broadcast () 0 +#else +static inline int cache_ops_need_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) <1 ; +} +#endif +#endif
arch/arm/mm/fault-armv.c +0 -2
@@ -153,10 +153,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) page =pfn_to_page (pfn); mapping =page_mapping (page); -#ifndef CONFIG_SMP if (test_and_clear_bit (PG_dcache_dirty, &page->flags))__flush_dcache_page (mapping, page); -#endif if (mapping) {if (cache_is_vivt ())make_coherent (mapping, vma, addr, pfn);
arch/arm/mm/flush.c +4 -5
@@ -15,6 +15,7 @@#include <asm/cachetype.h> #include <asm/system.h> #include <asm/tlbflush.h> +#include <asm/smp_plat.h> #include "mm.h"
@@ -198,12 +199,10 @@ void flush_dcache_page(struct page *page) {struct address_space *mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (!PageHighMem (page) && mapping && !mapping_mapped (mapping)) +if (!cache_ops_need_broadcast () && + !PageHighMem (page) && mapping && !mapping_mapped (mapping))set_bit (PG_dcache_dirty, &page->flags); - else -#endif - { +else {__flush_dcache_page (mapping, page);if (mapping &&cache_is_vivt ())__flush_dcache_aliases (mapping, page);
PATCH 4/6 ARMv7: Use lazy cache flushing if hardware broadcastscache operations
Hi Russell,
Quoted 4 lines · Catalin Marinas
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
It looks like I haven't got a final ok from you on this patch (I had the impression that it's in the patch system already but rebased my patches and found that it's not in mainline).
Are you ok with it (quoting it below)?
Quoted 70 lines
Signed-off-by Catalin Marinas <[email protected]>--- arch/arm/include/asm/smp_plat.h | 9 +++++++++ arch/arm/mm/fault-armv.c | 2 -- arch/arm/mm/flush.c | 9 ++++----- 3 files changed, 13 insertions(+), 7 deletions(-)arch/arm/include/asm/smp_plat.h +9
-0@@ -13,4 +13,13 @@ static inline int tlb_ops_need_broadcast(void) return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; } +#ifndef CONFIG_SMP +#define cache_ops_need_broadcast () 0 +#else +static inline int cache_ops_need_broadcast (void ) +{ +return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) <1 ; +} +#endif +#endif arch/arm/mm/fault-armv.c +0
-2@@ -153,10 +153,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte) page =pfn_to_page (pfn); mapping =page_mapping (page); -#ifndef CONFIG_SMP if (test_and_clear_bit (PG_dcache_dirty, &page->flags))__flush_dcache_page (mapping, page); -#endif if (mapping) {if (cache_is_vivt ())make_coherent(mapping, vma, addr, pfn);arch/arm/mm/flush.c +4
-5@@ -15,6 +15,7 @@#include <asm/cachetype.h> #include <asm/system.h> #include <asm/tlbflush.h> +#include <asm/smp_plat.h> #include "mm.h" @@ -198,12 +199,10 @@ void flush_dcache_page(struct page *page) {struct address_space *mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (!PageHighMem (page) && mapping && !mapping_mapped (mapping)) +if (!cache_ops_need_broadcast () && + !PageHighMem (page) && mapping && !mapping_mapped (mapping))set_bit (PG_dcache_dirty, &page->flags); - else -#endif - { +else {__flush_dcache_page (mapping, page);if (mapping &&cache_is_vivt ())__flush_dcache_aliases (mapping, page);
Signature · 1 line
Catalin
PATCH 4/6 ARMv7: Use lazy cache flushing if hardware broadcastscache operations
Quoted 11 lines · Catalin Marinas
Hi Russell,
Quoted 4 lines · Catalin Marinas
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
It looks like I haven't got a final ok from you on this patch (I had the impression that it's in the patch system already but rebased my patches and found that it's not in mainline).
It needs to be updated - we have a cache_ops_need_broadcast() in smp_plat.h now for the ptrace issues.
PATCH 4/6 ARMv7: Use lazy cache flushing if hardwarebroadcastscache operations
Quoted 9 lines · Russell King - ARM Linux
Quoted 8 lines · Catalin Marinas
Hi Russell,
Quoted 4 lines · Catalin Marinas
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
Quoted 6 lines
Quoted 3 lines
It looks like I haven't got a final ok from you on this patch (I had the impression that it's in the patch system already but rebased my patches and found that it's not in mainline).
It needs to be updated - we have a
cache_ops_need_broadcast()insmp_plat.hnow for the ptrace issues.
I noticed that when rebasing. Here's the updated patch:
ARMv7: Use lazy cache flushing if hardware broadcasts cache operations
From: Catalin Marinas <[email protected]>
ARMv7 processors like Cortex-A9 broadcast the cache maintenance operations in hardware. The patch adds the CPU ID checks for such feature and allows the flush_dcache_page/update_mmu_cache pair to work in lazy flushing mode similar to the UP case.
--- arch/arm/include/asm/smp_plat.h | 4 ++++ arch/arm/mm/fault-armv.c | 2 -- arch/arm/mm/flush.c | 9 ++++----- 3 files changed, 8 insertions(+), 7 deletions(-)
arch/arm/include/asm/smp_plat.h +4 -0
@@ -13,9 +13,13 @@ static inline int tlb_ops_need_broadcast(void) return ((read_cpuid_ext(CPUID_EXT_MMFR3) >> 12) & 0xf) < 2; } +#if !defined(CONFIG_SMP ) || __LINUX_ARM_ARCH__ >=7 +#define cache_ops_need_broadcast () 0 +#else static inline int cache_ops_need_broadcast (void ) {return ((read_cpuid_ext (CPUID_EXT_MMFR3 ) >>12 ) &0xf ) <1 ; } +#endif #endif
arch/arm/mm/fault-armv.c +0 -2
@@ -169,10 +169,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr,return ; mapping =page_mapping (page); -#ifndef CONFIG_SMP if (test_and_clear_bit (PG_dcache_dirty, &page->flags))__flush_dcache_page (mapping, page); -#endif if (mapping) {if (cache_is_vivt ())make_coherent (mapping, vma, addr, ptep, pfn);
arch/arm/mm/flush.c +4 -5
@@ -16,6 +16,7 @@#include <asm/smp_plat.h> #include <asm/system.h> #include <asm/tlbflush.h> +#include <asm/smp_plat.h> #include "mm.h"
@@ -241,12 +242,10 @@ void flush_dcache_page(struct page *page) mapping =page_mapping (page); -#ifndef CONFIG_SMP -if (!PageHighMem (page) && mapping && !mapping_mapped (mapping)) +if (!cache_ops_need_broadcast () && + !PageHighMem (page) && mapping && !mapping_mapped (mapping))set_bit (PG_dcache_dirty, &page->flags); - else -#endif - { +else {__flush_dcache_page (mapping, page);if (mapping &&cache_is_vivt ())__flush_dcache_aliases (mapping, page);
Signature · 1 line
Catalin
PATCH 5/6 ARMv7: Improved page table format with TRE and AFE
In reply to Catalin MarinasThis patch enables the Access Flag in SCTLR and, together with the TEX remapping, allows the use of the spare bits in the page table entry thus removing the Linux specific PTEs. The simplified permission model is used which means that "kernel read/write, user read-only" is no longer available. This was used for the vectors page but with a dedicated TLS register it is no longer necessary.
With this feature, the following bits were changed to overlap with the hardware bits:
L_PTE_NOEXEC -> XNL_PTE_PRESENT -> bit 1L_PTE_YOUNG -> AP[0] (access flag)L_PTE_USER -> AP[1] (simplified permission model)L_PTE_NOWRITE -> AP[2] (simplified permission model)L_PTE_DIRTY -> TEX[1] (spare bit)
The TEX[2] spare bit is available for future use.
Since !L_PTE_PRESENT requires bit 0 to be unset (otherwise it is a Large Page Table entry), L_PTE_FILE occupies bit 2. This requires some changes to the __swp_* and pte_to_pgoff/pgoff_to_pte macros to avoid overriding this bit. PTE_FILE_MAXBITS becomes 29 if AFE is enabled.
There are no changes required to the PMD_SECT_* macros because the current usage is compatible with the simplified permission model.
If hardware management of the access flag is available and SCTLR.HA is set, the L_PTE_YOUNG bit is automatically set when a page is accessed. With software management of the access flag, an "access flag" fault is generated which is handled by the do_page_fault() function.
--- arch/arm/include/asm/memory.h | 6 ++ arch/arm/include/asm/page.h | 8 +++ arch/arm/include/asm/pgalloc.h | 10 ++- arch/arm/include/asm/pgtable.h | 117 +++++++++++++++++++++++++++++++++++----- arch/arm/mm/Kconfig | 12 ++++ arch/arm/mm/dma-mapping.c | 6 ++ arch/arm/mm/fault.c | 10 +++ arch/arm/mm/mmu.c | 7 +- arch/arm/mm/proc-v7.S | 56 ++++++++----------- 9 files changed, 177 insertions(+), 55 deletions(-)
arch/arm/include/asm/memory.h +5 -1
@@ -113,11 +113,15 @@#endif /* !CONFIG_MMU */ /* -* Size of DMA-consistent memory region. Must be multiple of 2M, +* Size of DMA-consistent memory region. Must be multiple of 2M (4MB if AFE), * between 2MB and 14MB inclusive. */ #ifndef CONSISTENT_DMA_SIZE +#ifndef CONFIG_CPU_AFE #define CONSISTENT_DMA_SIZE SZ_2M +#else +#define CONSISTENT_DMA_SIZE SZ_4M +#endif #endif /*
arch/arm/include/asm/page.h +8 -0
@@ -158,7 +158,11 @@ extern void copy_page(void *to, const void *from);*/ typedef struct {unsigned long pte; }pte_t ;typedef struct {unsigned long pmd; }pmd_t ; +#ifndef CONFIG_CPU_AFE typedef struct {unsigned long pgd[2 ]; }pgd_t ; +#else +typedef struct {unsigned long pgd[4 ]; }pgd_t ; +#endif typedef struct {unsigned long pgprot; }pgprot_t ;#define pte_val (x) ((x).pte)
@@ -176,7 +180,11 @@ typedef struct { unsigned long pgprot; } pgprot_t;*/ typedef unsigned long pte_t ;typedef unsigned long pmd_t ; +#ifndef CONFIG_CPU_AFE typedef unsigned long pgd_t [2 ]; +#else +typedef unsigned long pgd_t [4 ]; +#endif typedef unsigned long pgprot_t ;#define pte_val (x) (x)
arch/arm/include/asm/pgalloc.h +7 -3
@@ -62,7 +62,7 @@ pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) pte = (pte_t *)__get_free_page (PGALLOC_GFP );if (pte) {clean_dcache_area (pte,sizeof (pte_t) *PTRS_PER_PTE ); - pte +=PTRS_PER_PTE ; + pte +=LINUX_PTE_OFFSET ; }return pte;
@@ -95,7 +95,7 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)static inline void pte_free_kernel (struct mm_struct *mm,pte_t *pte) {if (pte) { - pte -=PTRS_PER_PTE ; + pte -=LINUX_PTE_OFFSET ;free_page ((unsigned long )pte); } }
@@ -110,6 +110,10 @@ static inline void __pmd_populate(pmd_t *pmdp, unsigned long pmdval) { pmdp[0 ] =__pmd (pmdval); pmdp[1 ] =__pmd (pmdval +256 *sizeof (pte_t)); +#ifdef CONFIG_CPU_AFE + pmdp[2 ] =__pmd (pmdval +512 *sizeof (pte_t)); + pmdp[3 ] =__pmd (pmdval +768 *sizeof (pte_t)); +#endif flush_pmd_entry (pmdp); }
@@ -128,7 +132,7 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)* The pmd must be loaded with the physical * address of the PTE table */ - pte_ptr -=PTRS_PER_PTE *sizeof (void *); + pte_ptr -=LINUX_PTE_OFFSET *sizeof (void *);__pmd_populate (pmdp,__pa (pte_ptr) | _PAGE_KERNEL_TABLE); }
arch/arm/include/asm/pgtable.h +103 -14
210 lines not on this page · raw message
arch/arm/mm/Kconfig +12 -0
@@ -454,6 +454,18 @@ config CPU_32v6 config CPU_32v7 bool +# Page table format +config CPU_AFE + bool + depends on MMU + default y if CPU_V7 + help + This option sets the Access Flag Enable bit forcing the simplified + permission model and automatic management of the access bit (if + supported by the hardware). With this option enabled and TEX + remapping, Linux no longer keeps a separate page table entry for + storing additional bits. + # The abort model config CPU_ABRT_NOMMU bool
arch/arm/mm/dma-mapping.c +6 -0
@@ -25,9 +25,15 @@#include <asm/sizes.h> /* Sanity check size */ +#ifndef CONFIG_CPU_AFE #if (CONSISTENT_DMA_SIZE %SZ_2M )#error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"#endif +#else +#if (CONSISTENT_DMA_SIZE %SZ_4M ) +#error "CONSISTENT_DMA_SIZE must be multiple of 4MiB" +#endif +#endif #define CONSISTENT_END (0xffe00000)#define CONSISTENT_BASE (CONSISTENT_END - CONSISTENT_DMA_SIZE)
arch/arm/mm/fault.c +10 -0
@@ -107,7 +107,9 @@ void show_pte(struct mm_struct *mm, unsigned long addr) pte =pte_offset_map (pmd, addr);printk (", *pte=%08lx" ,pte_val (*pte)); +#ifndef CONFIG_CPU_AFE printk (", *ppte=%08lx" ,pte_val (pte[-PTRS_PER_PTE ])); +#endif pte_unmap (pte); }while (0 );
@@ -458,7 +460,11 @@ static struct fsr_info { { do_bad, SIGILL, BUS_ADRALN, "alignment exception" }, { do_bad, SIGBUS, 0, "external abort on linefetch" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" }, +#ifndef CONFIG_CPU_AFE { do_bad, SIGBUS, 0, "external abort on linefetch" }, +#else + { do_page_fault, SIGSEGV, SEGV_MAPERR, "access flag fault" }, +#endif { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" }, { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
@@ -532,7 +538,11 @@ static struct fsr_info ifsr_info[] = { { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" }, { do_bad, SIGBUS, 0, "unknown 4" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" }, +#ifndef CONFIG_CPU_AFE { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" }, +#else + { do_page_fault, SIGSEGV, SEGV_MAPERR, "access flag fault" }, +#endif { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" }, { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
arch/arm/mm/mmu.c +4 -3
@@ -190,7 +190,7 @@ void adjust_cr(unsigned long mask, unsigned long set) }#endif -#define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_WRITE +#define PROT_PTE_DEVICE L_PTE_PRESENT|L_PTE_YOUNG|L_PTE_DIRTY|L_PTE_WRITE|L_PTE_NOEXEC#define PROT_SECT_DEVICE PMD_TYPE_SECT|PMD_SECT_AP_WRITEstatic struct mem_type mem_types[] = {
@@ -241,7 +241,7 @@ static struct mem_type mem_types[] = { }, [MT_HIGH_VECTORS] = { .prot_pte =L_PTE_PRESENT |L_PTE_YOUNG |L_PTE_DIRTY | - L_PTE_USER | L_PTE_EXEC, + L_PTE_USER | L_PTE_EXEC | L_PTE_NOWRITE, .prot_l1 = PMD_TYPE_TABLE, .domain = DOMAIN_USER, },
@@ -491,7 +491,8 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,pte_t *pte;if (pmd_none (*pmd)) { - pte =alloc_bootmem_low_pages (2 *PTRS_PER_PTE *sizeof (pte_t)); + pte =alloc_bootmem_low_pages ((LINUX_PTE_OFFSET + +PTRS_PER_PTE ) *sizeof (pte_t));__pmd_populate (pmd,__pa (pte) | type->prot_l1); }
arch/arm/mm/proc-v7.S +22 -34
74 lines not on this page · raw message
PATCH 5/6 ARMv7: Improved page table format with TRE and AFE
Quoted 6 lines · Catalin Marinas
This patch enables the Access Flag in SCTLR and, together with the TEX remapping, allows the use of the spare bits in the page table entry thus removing the Linux specific PTEs. The simplified permission model is used which means that "kernel read/write, user read-only" is no longer available. This was used for the vectors page but with a dedicated TLS register it is no longer necessary.
I really do not want to go here without an explaination of how situations such as:
- Kernel reads PTE and modifies it
- Hardware accesses page
- TLB reads PTE, updates, and writes new back
- Kernel writes PTE back
- Kernel cleans cache line
are handled. What about SMP, where CPU0 may access and modify the active page tables on CPU1 (eg, clearing PTEs)?
Are TLB accesses with AFE enabled guaranteed to read from the L1 cache? If not, we need to clean _and_ invalidate PTE updates.
PATCH 5/6 ARMv7: Improved page table format with TRE and AFE
Quoted 10 lines · Russell King - ARM Linux
Quoted 6 lines · Catalin Marinas
This patch enables the Access Flag in SCTLR and, together with the TEX remapping, allows the use of the spare bits in the page table entry thus removing the Linux specific PTEs. The simplified permission model is used which means that "kernel read/write, user read-only" is no longer available. This was used for the vectors page but with a dedicated TLS register it is no longer necessary.
I really do not want to go here without an explaination of how situations such as:
I think we discussed some of these when I first posted the patch some time ago but I'm happy do it again here.
BTW, all the hardware implementations I'm aware of only raise an access flag fault when this bit is cleared rather than doing it in hardware.
But even if they do it in hardware, it can still work (you also have the option of disabling the hardware management via the SCTLR.HA bit).
Quoted 1 line
- Kernel reads PTE and modifies it
B3.3.5 in the ARM ARM describes the requirements for the Hardware management of the access flag:
Any implementation of hardware management of the access flag must ensure that any software changes to the translation table are not lost. The architecture does not require software that performs translation table changes to use interlocked operations. The hardware management mechanisms for the access flag must prevent any loss of data written to translation table entries that might occur when, for example, a write by another processor occurs between the read and write phases of a translation table walk that updates the
access flag.
At the hardware level, it could be implemented similar to a LDREX/STREX block.
Quoted 3 lines
- Hardware accesses page
- TLB reads PTE, updates, and writes new back
- Kernel writes PTE back
Addressed above. The hardware write should fail if there was an STR from the current or different CPU.
Quoted 1 line
- Kernel cleans cache line
A hardware implementation of the AF would probably require the PTWs via L1 (Cortex-A9 has such PTWs) otherwise it breaks the requirements. It is mandatory to have the same TTBR cacheability settings as the mapping of the page tables (i.e. Normal cached in the Linux case) so we don't need further kernel modifications.
Signature · 1 line
Catalin
PATCH 5/6 ARMv7: Improved page table format with TRE and AFE
Quoted 37 lines · Catalin Marinas
Quoted 10 lines · Russell King - ARM Linux
Quoted 6 lines · Catalin Marinas
This patch enables the Access Flag in SCTLR and, together with the TEX remapping, allows the use of the spare bits in the page table entry thus removing the Linux specific PTEs. The simplified permission model is used which means that "kernel read/write, user read-only" is no longer available. This was used for the vectors page but with a dedicated TLS register it is no longer necessary.
I really do not want to go here without an explaination of how situations such as:
I think we discussed some of these when I first posted the patch some time ago but I'm happy do it again here.
BTW, all the hardware implementations I'm aware of only raise an access flag fault when this bit is cleared rather than doing it in hardware.
But even if they do it in hardware, it can still work (you also have the option of disabling the hardware management via the SCTLR.HA bit).
Quoted 1 line
- Kernel reads PTE and modifies it
B3.3.5 in the ARM ARM describes the requirements for the Hardware management of the access flag:
Any implementation of hardware management of the access flag must ensure that any software changes to the translation table are not lost. The architecture does not require software that performs translation table changes to use interlocked operations. The hardware management mechanisms for the access flag must prevent any loss of data written to translation table entries that might occur when, for example, a write by another processor occurs between the read and write phases of a translation table walk that updates the
access flag.
Quoted 2 lines
At the hardware level, it could be implemented similar to a LDREX/STREX block.
Just avoid a question on this - it is possible for the kernel to read a PTE with AP[0] cleared, the hardware could set AP[0] to 1 as a result of an access then the kernel clears it again when storing the modified PTE.
The above cannot be prevented since the PTE modifications are not atomic but it doesn't actually matter. In the worst case, the kernel would think that a page wasn't accessed for a longer time and it may decide to swap it out. I doubt this would be a performance hit since trapping the access faults takes much more time.
If precise access timing is required (not sure why), you can always disable SCTLR.HA and handle the accesses in software.
Signature · 1 line
Catalin
PATCH 5/6 ARMv7: Improved page table format with TRE and AFE
In reply to Catalin MarinasQuoted 19 lines · Catalin Marinas
Quoted 1 line
- Kernel reads PTE and modifies it
B3.3.5 in the ARM ARM describes the requirements for the Hardware management of the access flag:
Any implementation of hardware management of the access flag must ensure that any software changes to the translation table are not lost. The architecture does not require software that performs translation table changes to use interlocked operations. The hardware management mechanisms for the access flag must prevent any loss of data written to translation table entries that might occur when, for example, a write by another processor occurs between the read and write phases of a translation table walk that updates the
access flag.At the hardware level, it could be implemented similar to a LDREX/STREX block.
Quoted 6 lines
Quoted 3 lines
- Hardware accesses page
- TLB reads PTE, updates, and writes new back
- Kernel writes PTE backAddressed above. The hardware write should fail if there was an STR from the current or different CPU.
I don't think it is - the paragraph you quote talks about the following situation:
- Hardware reads PTE
- Kernel writes PTE
- Hardware (tries to) write PTE
What it says is that the hardware write in this case must fail.
The case I was talking about is:
- Kernel reads PTE
- Hardware reads PTE
- Hardware writes PTE
- Kernel writes PTE
Since there is no STR between the hardware reading and writing the PTE, the hardware can not know that its update has been lost.
Whether it matters or not is a different kettle of fish.
PATCH 5/6 ARMv7: Improved page table format with TRE and AFE
Quoted 26 lines · Russell King - ARM Linux
Quoted 25 lines · Catalin Marinas
Quoted 1 line
- Kernel reads PTE and modifies it
B3.3.5 in the ARM ARM describes the requirements for the Hardware management of the access flag:
Any implementation of hardware management of the access flag must ensure that any software changes to the translation table are not lost. The architecture does not require software that performs translation table changes to use interlocked operations. The hardware management mechanisms for the access flag must prevent any loss of data written to translation table entries that might occur when, for example, a write by another processor occurs between the read and write phases of a translation table walk that updates the
access flag.At the hardware level, it could be implemented similar to a LDREX/STREX block.
Quoted 3 lines
- Hardware accesses page
- TLB reads PTE, updates, and writes new back
- Kernel writes PTE backAddressed above. The hardware write should fail if there was an STR from the current or different CPU.
[...]
Quoted 11 lines
The case I was talking about is:
- Kernel reads PTE
- Hardware reads PTE
- Hardware writes PTE
- Kernel writes PTESince there is no STR between the hardware reading and writing the PTE, the hardware can not know that its update has been lost.
Whether it matters or not is a different kettle of fish.
I was expected this follow-up :-), so I already replied to my post. I don't think it matters.
Signature · 1 line
Catalin
PATCH 6/6 Remove the domain switching on ARMv6k/v7 CPUs
In reply to Catalin MarinasThis patch removes the domain switching functionality via the set_fs and __switch_to functions on cores that have a TLS register.
Currently, the ioremap and vmalloc areas share the same level 1 page tables and therefore have the same domain (DOMAIN_KERNEL). When the kernel domain is modified from Client to Manager (via the __set_fs or in the __switch_to function), the XN (eXecute Never) bit is overridden and newer CPUs can speculatively prefetch the ioremap'ed memory.
Linux performs the kernel domain switching to allow user-specific functions (copy_to/from_user, get/put_user etc.) to access kernel memory. In order for these functions to work with the kernel domain set to Client, the patch modifies the LDRT/STRT and related instructions to the LDR/STR ones.
The user pages access rights are also modified for kernel read-only access rather than read/write so that the copy-on-write mechanism still works. CPU_USE_DOMAINS gets disabled only if HAS_TLS_REG is defined since writing the TLS value to the high vectors page isn't possible.
The user addresses passed to the kernel are checked by the access_ok() function so that they do not point to the kernel space.
---
An additional note - prior to ARMv6 we cannot set the user R/O, kernel R/O permission on a page hence we have to use STRT variant to write such page from the kernel. Because of this, the T macro had to be introduced to differentiate between the STRT and STR usages and build time.
A better name could be used instead of "T".
arch/arm/include/asm/assembler.h | 9 ++-- arch/arm/include/asm/domain.h | 31 +++++++++++++- arch/arm/include/asm/futex.h | 9 ++-- arch/arm/include/asm/uaccess.h | 16 ++++--- arch/arm/kernel/entry-armv.S | 4 +- arch/arm/kernel/traps.c | 17 ++++++++ arch/arm/lib/getuser.S | 13 +++--- arch/arm/lib/putuser.S | 29 +++++++------ arch/arm/lib/uaccess.S | 83 +++++++++++++++++++------------------- arch/arm/mm/Kconfig | 9 ++++ arch/arm/mm/proc-v7.S | 2 - 11 files changed, 139 insertions(+), 83 deletions(-)
arch/arm/include/asm/assembler.h +5 -4
@@ -18,6 +18,7 @@#endif #include <asm/ptrace.h> +#include <asm/domain.h> /* * Endian independent macros for shifting bytes within registers.
@@ -186,9 +187,9 @@ .macro usraccoff, instr, reg, ptr, inc, off, cond, abort 9999: .if \inc == 1 - \instr\cond\()bt \reg, [\ptr, #\off] + T(\instr\cond\()b) \reg, [\ptr, #\off] .elseif \inc == 4 - \instr\cond\()t \reg, [\ptr, #\off] + T(\instr\cond\()) \reg, [\ptr, #\off] .else .error "Unsupported inc macro argument" .endif
@@ -227,9 +228,9 @@ .rept \rept 9999: .if \inc == 1 - \instr\cond\()bt \reg, [\ptr], #\inc + T(\instr\cond\()b) \reg, [\ptr], #\inc .elseif \inc == 4 - \instr\cond\()t \reg, [\ptr], #\inc + T(\instr\cond\()) \reg, [\ptr], #\inc .else .error "Unsupported inc macro argument" .endif
arch/arm/include/asm/domain.h +29 -2
@@ -45,13 +45,17 @@*/ #define DOMAIN_NOACCESS 0#define DOMAIN_CLIENT 1 +#ifdef CONFIG_CPU_USE_DOMAINS #define DOMAIN_MANAGER 3 +#else +#define DOMAIN_MANAGER 1 +#endif #define domain_val (dom,type) ((type) << (2*(dom)))#ifndef __ASSEMBLY__ -#ifdef CONFIG_MMU +#ifdef CONFIG_CPU_USE_DOMAINS #define set_domain (x) \ do { \ __asm__ __volatile__( \
@@ -74,5 +78,28 @@#define modify_domain (dom,type) do { } while (0)#endif +/* +* Generate the T (user) versions of the LDR/STR and related +* instructions (inline assembly) +*/ +#ifdef CONFIG_CPU_USE_DOMAINS +#define T (instr) #instr "t" +#else +#define T (instr) #instr#endif -#endif /* !__ASSEMBLY__ */ + +#else /* __ASSEMBLY__ */ + +/* +* Generate the T (user) versions of the LDR/STR and related +* instructions +*/ +#ifdef CONFIG_CPU_USE_DOMAINS +#define T (instr) instr ## t +#else +#define T (instr) instr +#endif + +#endif /* __ASSEMBLY__ */ + +#endif /* !__ASM_PROC_DOMAIN_H */
arch/arm/include/asm/futex.h +5 -4
@@ -13,12 +13,13 @@#include <linux/preempt.h> #include <linux/uaccess.h> #include <asm/errno.h> +#include <asm/domain.h> #define __futex_atomic_op (insn, ret, oldval, uaddr, oparg) \ __asm__ __volatile__( \ - "1: ldrt %1, [%2]\n" \ + "1: " T(ldr) " %1, [%2]\n" \ " " insn "\n" \ - "2: strt %0, [%2]\n" \ + "2: " T(str) " %0, [%2]\n" \ " mov %0, #0\n" \ "3:\n" \ " .section __ex_table,\"a\"\n" \
@@ -97,10 +98,10 @@ futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval)pagefault_disable ();/* implies preempt_disable() */ __asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n" -"1: ldrt %0, [%3]\n" + "1: " T(ldr) " %0, [%3]\n"" teq %0, %1\n" " it eq @ explicit IT needed for the 2b label\n" -"2: streqt %2, [%3]\n" + "2: " T(streq) " %2, [%3]\n""3:\n" " .section __ex_table,\"a\"\n" " .align 3\n"
arch/arm/include/asm/uaccess.h +8 -8
@@ -227,7 +227,7 @@ do { \#define __get_user_asm_byte (x,addr,err) \ __asm__ __volatile__( \ - "1: ldrbt %1,[%2]\n" \ + "1: " T(ldrb) " %1,[%2],#0\n" \ "2:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \
@@ -263,7 +263,7 @@ do { \#define __get_user_asm_word (x,addr,err) \ __asm__ __volatile__( \ - "1: ldrt %1,[%2]\n" \ + "1: " T(ldr) " %1,[%2],#0\n" \ "2:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \
@@ -308,7 +308,7 @@ do { \#define __put_user_asm_byte (x,__pu_addr,err) \ __asm__ __volatile__( \ - "1: strbt %1,[%2]\n" \ + "1: " T(strb) " %1,[%2],#0\n" \ "2:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \
@@ -341,7 +341,7 @@ do { \#define __put_user_asm_word (x,__pu_addr,err) \ __asm__ __volatile__( \ - "1: strt %1,[%2]\n" \ + "1: " T(str) " %1,[%2],#0\n" \ "2:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \
@@ -366,10 +366,10 @@ do { \#define __put_user_asm_dword (x,__pu_addr,err) \ __asm__ __volatile__( \ - ARM( "1: strt " __reg_oper1 ", [%1], #4\n" ) \ - ARM( "2: strt " __reg_oper0 ", [%1]\n" ) \ - THUMB( "1: strt " __reg_oper1 ", [%1]\n" ) \ - THUMB( "2: strt " __reg_oper0 ", [%1, #4]\n" ) \ + ARM( "1: " T(str) " " __reg_oper1 ", [%1], #4\n" ) \ + ARM( "2: " T(str) " " __reg_oper0 ", [%1]\n" ) \ + THUMB( "1: " T(str) " " __reg_oper1 ", [%1]\n" ) \ + THUMB( "2: " T(str) " " __reg_oper0 ", [%1, #4]\n" ) \ "3:\n" \ " .section .fixup,\"ax\"\n" \ " .align 2\n" \
arch/arm/kernel/entry-armv.S +2 -2
@@ -736,7 +736,7 @@ ENTRY(__switch_to) THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack THUMB( str sp, [ip], #4 ) THUMB( str lr, [ip], #4 ) -#ifdef CONFIG_MMU +#ifdef CONFIG_CPU_USE_DOMAINS ldr r6, [r2, #TI_CPU_DOMAIN] #endif #if defined(CONFIG_HAS_TLS_REG)
@@ -745,7 +745,7 @@ ENTRY(__switch_to) mov r4, #0xffff0fff str r3, [r4, #-15] @ TLS val at 0xffff0ff0 #endif -#ifdef CONFIG_MMU +#ifdef CONFIG_CPU_USE_DOMAINS mcr p15, 0, r6, c3, c0, 0 @ Set domain register #endif mov r5, r0
arch/arm/kernel/traps.c +17 -0
@@ -28,6 +28,7 @@#include <asm/unistd.h> #include <asm/traps.h> #include <asm/unwind.h> +#include <asm/tlbflush.h> #include "ptrace.h" #include "signal.h"
@@ -735,6 +736,16 @@ void __init early_trap_init(void)extern char __vectors_start[], __vectors_end[];extern char __kuser_helper_start[], __kuser_helper_end[];int kuser_sz = __kuser_helper_end - __kuser_helper_start; +#ifndef CONFIG_CPU_USE_DOMAINS +pgd_t *pgd =pgd_offset_k (vectors); +pmd_t *pmd =pmd_offset (pgd, vectors); +pte_t *pte =pte_offset_kernel (pmd, vectors); +pte_t entry = *pte; + +/* allow writing to the vectors page */ +set_pte_ext (pte,pte_mkwrite (entry),0 ); +local_flush_tlb_kernel_page (vectors); +#endif /* * Copy the vectors, stubs and kuser helpers (in entry-armv.S)
@@ -754,6 +765,12 @@ void __init early_trap_init(void)memcpy ((void *)KERN_RESTART_CODE , syscall_restart_code,sizeof (syscall_restart_code)); +#ifndef CONFIG_CPU_USE_DOMAINS +/* restore the vectors page permissions */ +set_pte_ext (pte, entry,0 ); +local_flush_tlb_kernel_page (vectors); +#endif +flush_icache_range (vectors, vectors +PAGE_SIZE );modify_domain (DOMAIN_USER ,DOMAIN_CLIENT ); }
arch/arm/lib/getuser.S +7 -6
@@ -28,20 +28,21 @@ */ #include <linux/linkage.h> #include <asm/errno.h> +#include <asm/domain.h> ENTRY(__get_user_1) -1: ldrbt r2, [r0] +1: T(ldrb) r2, [r0] mov r0, #0 mov pc, lr ENDPROC(__get_user_1) ENTRY(__get_user_2) #ifdef CONFIG_THUMB2_KERNEL -2: ldrbt r2, [r0] -3: ldrbt r3, [r0, #1] +2: T(ldrb) r2, [r0] +3: T(ldrb) r3, [r0, #1] #else -2: ldrbt r2, [r0], #1 -3: ldrbt r3, [r0] +2: T(ldrb) r2, [r0], #1 +3: T(ldrb) r3, [r0] #endif #ifndef __ARMEB__ orr r2, r2, r3, lsl #8
@@ -53,7 +54,7 @@ ENTRY(__get_user_2) ENDPROC(__get_user_2) ENTRY(__get_user_4) -4: ldrt r2, [r0] +4: T(ldr) r2, [r0] mov r0, #0 mov pc, lr ENDPROC(__get_user_4)
arch/arm/lib/putuser.S +15 -14
@@ -28,9 +28,10 @@ */ #include <linux/linkage.h> #include <asm/errno.h> +#include <asm/domain.h> ENTRY(__put_user_1) -1: strbt r2, [r0] +1: T(strb) r2, [r0] mov r0, #0 mov pc, lr ENDPROC(__put_user_1)
@@ -39,19 +40,19 @@ ENTRY(__put_user_2) mov ip, r2, lsr #8 #ifdef CONFIG_THUMB2_KERNEL #ifndef __ARMEB__ -2: strbt r2, [r0] -3: strbt ip, [r0, #1] +2: T(strb) r2, [r0] +3: T(strb) ip, [r0, #1] #else -2: strbt ip, [r0] -3: strbt r2, [r0, #1] +2: T(strb) ip, [r0] +3: T(strb) r2, [r0, #1] #endif #else /* !CONFIG_THUMB2_KERNEL */ #ifndef __ARMEB__ -2: strbt r2, [r0], #1 -3: strbt ip, [r0] +2: T(strb) r2, [r0], #1 +3: T(strb) ip, [r0] #else -2: strbt ip, [r0], #1 -3: strbt r2, [r0] +2: T(strb) ip, [r0], #1 +3: T(strb) r2, [r0] #endif #endif /* CONFIG_THUMB2_KERNEL */ mov r0, #0
@@ -59,18 +60,18 @@ ENTRY(__put_user_2) ENDPROC(__put_user_2) ENTRY(__put_user_4) -4: strt r2, [r0] +4: T(str) r2, [r0] mov r0, #0 mov pc, lr ENDPROC(__put_user_4) ENTRY(__put_user_8) #ifdef CONFIG_THUMB2_KERNEL -5: strt r2, [r0] -6: strt r3, [r0, #4] +5: T(str) r2, [r0] +6: T(str) r3, [r0, #4] #else -5: strt r2, [r0], #4 -6: strt r3, [r0] +5: T(str) r2, [r0], #4 +6: T(str) r3, [r0] #endif mov r0, #0 mov pc, lr
arch/arm/lib/uaccess.S +42 -41
260 lines not on this page · raw message
arch/arm/mm/Kconfig +9 -0
@@ -618,6 +618,15 @@ config CPU_CP15_MPU help Processor has the CP15 register, which has MPU related registers. +config CPU_USE_DOMAINS + bool + depends on MMU + default n if HAS_TLS_REG + default y + help + This option enables or disable the use of domain switching + via the set_fs() function. + # # CPU supports 36-bit I/O #
arch/arm/mm/proc-v7.S +0 -2
@@ -223,8 +223,6 @@ __v7_setup: mcr p15, 0, r10, c2, c0, 2 @ TTB control register orr r4, r4, #TTB_FLAGS mcr p15, 0, r4, c2, c0, 1 @ load TTB1 - mov r10, #0x1f @ domains 0, 1 = manager - mcr p15, 0, r10, c3, c0, 0 @ load domain access register /* * Memory region attributes with SCTLR.TRE=1 *