KVM: arm64: Fixes for parallel faults series
v4 cover KVM: arm64: Fixes for parallel faults series
Small set of fixes for the parallel faults series. Most importantly, stop taking the RCU read lock for walking hyp stage-1. For the sake of consistency, take a pointer to kvm_pgtable_walker in kvm_dereference_pteref() as well.
Tested on an Ampere Altra system with kvm-arm.mode={nvhe,protected} and lockdep. Applies on top of the parallel faults series picked up last week.
v3: https://lore.kernel.org/kvmarm/[email protected]/
v3 -> v4:
- Return an error instead of WARN() in hyp for shared walks (Will)
Oliver Upton (3):
KVM: arm64: Take a pointer to walker data in kvm_dereference_pteref()
KVM: arm64: Don't acquire RCU read lock for exclusive table walks
KVM: arm64: Reject shared table walks in the hyp code
arch/arm64/include/asm/kvm_pgtable.h | 159 +++++++++++++++------------ arch/arm64/kvm/hyp/pgtable.c | 13 ++- 2 files changed, 96 insertions(+), 76 deletions(-)
Signature · 1 line
2.38.1.584.g0f3c55d4c2-goog
PATCH v4 1/3 KVM: arm64: Take a pointer to walker data in kvm_dereference_pteref()
Rather than passing through the state of the KVM_PGTABLE_WALK_SHARED flag, just take a pointer to the whole walker structure instead. Move around struct kvm_pgtable and the RCU indirection such that the associated ifdeffery remains in one place while ensuring the walker + flags definitions precede their use.
No functional change intended.
--- arch/arm64/include/asm/kvm_pgtable.h | 144 ++++++++++++++------------- arch/arm64/kvm/hyp/pgtable.c | 6 +- 2 files changed, 76 insertions(+), 74 deletions(-)
arch/arm64/include/asm/kvm_pgtable.h +73 -71
@@ -37,54 +37,6 @@ static inline u64 kvm_get_parange(u64 mmfr0)typedef u64 kvm_pte_t ; -/* -* RCU cannot be used in a non-kernel context such as the hyp. As such, page -* table walkers used in hyp do not call into RCU and instead use other -* synchronization mechanisms (such as a spinlock). -*/ -#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) - -typedef kvm_pte_t *kvm_pteref_t ; - -static inline kvm_pte_t *kvm_dereference_pteref (kvm_pteref_t pteref,bool shared) -{ -return pteref; -} - -static inline void kvm_pgtable_walk_begin (void ) {} -static inline void kvm_pgtable_walk_end (void ) {} - -static inline bool kvm_pgtable_walk_lock_held (void ) -{ -return true; -} - -#else - -typedef kvm_pte_t __rcu *kvm_pteref_t ; - -static inline kvm_pte_t *kvm_dereference_pteref (kvm_pteref_t pteref,bool shared) -{ -return rcu_dereference_check (pteref, !shared); -} - -static inline void kvm_pgtable_walk_begin (void ) -{ -rcu_read_lock (); -} - -static inline void kvm_pgtable_walk_end (void ) -{ -rcu_read_unlock (); -} - -static inline bool kvm_pgtable_walk_lock_held (void ) -{ -return rcu_read_lock_held (); -} - -#endif -#define KVM_PTE_VALID BIT(0)#define KVM_PTE_ADDR_MASK GENMASK(47, PAGE_SHIFT)
@@ -212,29 +164,6 @@ enum kvm_pgtable_prot {typedef bool (*kvm_pgtable_force_pte_cb_t )(u64 addr,u64 end,enum kvm_pgtable_prot prot); -/** -* struct kvm_pgtable - KVM page-table. -* @ia_bits: Maximum input address size, in bits. -* @start_level: Level at which the page-table walk starts. -* @pgd: Pointer to the first top-level entry of the page-table. -* @mm_ops: Memory management callbacks. -* @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables. -* @flags: Stage-2 page-table flags. -* @force_pte_cb: Function that returns true if page level mappings must -* be used instead of block mappings. -*/ -struct kvm_pgtable { -u32 ia_bits; -u32 start_level; -kvm_pteref_t pgd; -struct kvm_pgtable_mm_ops *mm_ops; - -/* Stage-2 only */ -struct kvm_s2_mmu *mmu; -enum kvm_pgtable_stage2_flags flags; -kvm_pgtable_force_pte_cb_t force_pte_cb; -}; -/** * enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk. * @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid
@@ -285,6 +214,79 @@ struct kvm_pgtable_walker {const enum kvm_pgtable_walk_flags flags; }; +/* +* RCU cannot be used in a non-kernel context such as the hyp. As such, page +* table walkers used in hyp do not call into RCU and instead use other +* synchronization mechanisms (such as a spinlock). +*/ +#if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) + +typedef kvm_pte_t *kvm_pteref_t ; + +static inline kvm_pte_t *kvm_dereference_pteref (struct kvm_pgtable_walker *walker, +kvm_pteref_t pteref) +{ +return pteref; +} + +static inline void kvm_pgtable_walk_begin (void ) {} +static inline void kvm_pgtable_walk_end (void ) {} + +static inline bool kvm_pgtable_walk_lock_held (void ) +{ +return true; +} + +#else + +typedef kvm_pte_t __rcu *kvm_pteref_t ; + +static inline kvm_pte_t *kvm_dereference_pteref (struct kvm_pgtable_walker *walker, +kvm_pteref_t pteref) +{ +return rcu_dereference_check (pteref, !(walker->flags &KVM_PGTABLE_WALK_SHARED )); +} + +static inline void kvm_pgtable_walk_begin (void ) +{ +rcu_read_lock (); +} + +static inline void kvm_pgtable_walk_end (void ) +{ +rcu_read_unlock (); +} + +static inline bool kvm_pgtable_walk_lock_held (void ) +{ +return rcu_read_lock_held (); +} + +#endif + +/** +* struct kvm_pgtable - KVM page-table. +* @ia_bits: Maximum input address size, in bits. +* @start_level: Level at which the page-table walk starts. +* @pgd: Pointer to the first top-level entry of the page-table. +* @mm_ops: Memory management callbacks. +* @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables. +* @flags: Stage-2 page-table flags. +* @force_pte_cb: Function that returns true if page level mappings must +* be used instead of block mappings. +*/ +struct kvm_pgtable { +u32 ia_bits; +u32 start_level; +kvm_pteref_t pgd; +struct kvm_pgtable_mm_ops *mm_ops; + +/* Stage-2 only */ +struct kvm_s2_mmu *mmu; +enum kvm_pgtable_stage2_flags flags; +kvm_pgtable_force_pte_cb_t force_pte_cb; +}; +/** * kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table. * @pgt: Uninitialised page-table structure to initialise.
arch/arm64/kvm/hyp/pgtable.c +3 -3
@@ -188,7 +188,7 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, kvm_pteref_t pteref, u32 level) {enum kvm_pgtable_walk_flags flags = data->walker->flags; -kvm_pte_t *ptep =kvm_dereference_pteref (pteref, flags &KVM_PGTABLE_WALK_SHARED ); +kvm_pte_t *ptep =kvm_dereference_pteref (data->walker, pteref);struct kvm_pgtable_visit_ctx ctx = { .ptep = ptep, .old =READ_ONCE (*ptep),
@@ -558,7 +558,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt) };WARN_ON (kvm_pgtable_walk (pgt,0 ,BIT (pgt->ia_bits), &walker)); - pgt->mm_ops->put_page (kvm_dereference_pteref (pgt->pgd, false)); + pgt->mm_ops->put_page (kvm_dereference_pteref (&walker, pgt->pgd)); pgt->pgd =NULL ; }
@@ -1241,7 +1241,7 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)WARN_ON (kvm_pgtable_walk (pgt,0 ,BIT (pgt->ia_bits), &walker)); pgd_sz =kvm_pgd_pages (pgt->ia_bits, pgt->start_level) *PAGE_SIZE ; - pgt->mm_ops->free_pages_exact (kvm_dereference_pteref (pgt->pgd, false), pgd_sz); + pgt->mm_ops->free_pages_exact (kvm_dereference_pteref (&walker, pgt->pgd), pgd_sz); pgt->pgd =NULL ; }
Signature · 1 line
2.38.1.584.g0f3c55d4c2-goog
PATCH v4 2/3 KVM: arm64: Don't acquire RCU read lock for exclusive table walks
In reply to Oliver UptonMarek reported a BUG resulting from the recent parallel faults changes, as the hyp stage-1 map walker attempted to allocate table memory while holding the RCU read lock:
BUG: sleeping function called from invalid context at
include/linux/sched/mm.h:274
in_atomic(): 0, irqs_disabled(): 0, non_block: 0, pid: 1, name: swapper/0
preempt_count: 0, expected: 0
RCU nest depth: 1, expected: 0
2 locks held by swapper/0/1:
#0: ffff80000a8a44d0 (kvm_hyp_pgd_mutex){+.+.}-{3:3}, at:
__create_hyp_mappings+0x80/0xc4
#1: ffff80000a927720 (rcu_read_lock){....}-{1:2}, at:
kvm_pgtable_walk+0x0/0x1f4
CPU: 2 PID: 1 Comm: swapper/0 Not tainted 6.1.0-rc3+ #5918
Hardware name: Raspberry Pi 3 Model B (DT)
Call trace:
dump_backtrace.part.0+0xe4/0xf0
show_stack+0x18/0x40
dump_stack_lvl+0x8c/0xb8
dump_stack+0x18/0x34
__might_resched+0x178/0x220
__might_sleep+0x48/0xa0
prepare_alloc_pages+0x178/0x1a0
__alloc_pages+0x9c/0x109c
alloc_page_interleave+0x1c/0xc4
alloc_pages+0xec/0x160
get_zeroed_page+0x1c/0x44
kvm_hyp_zalloc_page+0x14/0x20
hyp_map_walker+0xd4/0x134
kvm_pgtable_visitor_cb.isra.0+0x38/0x5c
__kvm_pgtable_walk+0x1a4/0x220
kvm_pgtable_walk+0x104/0x1f4
kvm_pgtable_hyp_map+0x80/0xc4
__create_hyp_mappings+0x9c/0xc4
kvm_mmu_init+0x144/0x1cc
kvm_arch_init+0xe4/0xef4
kvm_init+0x3c/0x3d0
arm_init+0x20/0x30
do_one_initcall+0x74/0x400
kernel_init_freeable+0x2e0/0x350
kernel_init+0x24/0x130
ret_from_fork+0x10/0x20Since the hyp stage-1 table walkers are serialized by kvm_hyp_pgd_mutex, RCU protection really doesn't add anything. Don't acquire the RCU read lock for an exclusive walk.
--- arch/arm64/include/asm/kvm_pgtable.h | 14 ++++++++------ arch/arm64/kvm/hyp/pgtable.c | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-)
arch/arm64/include/asm/kvm_pgtable.h +8 -6
@@ -229,8 +229,8 @@ static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walkereturn pteref; } -static inline void kvm_pgtable_walk_begin (void ) {} -static inline void kvm_pgtable_walk_end (void ) {} +static inline void kvm_pgtable_walk_begin (struct kvm_pgtable_walker *walker) {} +static inline void kvm_pgtable_walk_end (struct kvm_pgtable_walker *walker) {}static inline bool kvm_pgtable_walk_lock_held (void ) {
@@ -247,14 +247,16 @@ static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walkereturn rcu_dereference_check (pteref, !(walker->flags &KVM_PGTABLE_WALK_SHARED )); } -static inline void kvm_pgtable_walk_begin (void ) +static inline void kvm_pgtable_walk_begin (struct kvm_pgtable_walker *walker) { -rcu_read_lock (); +if (walker->flags &KVM_PGTABLE_WALK_SHARED ) +rcu_read_lock (); } -static inline void kvm_pgtable_walk_end (void ) +static inline void kvm_pgtable_walk_end (struct kvm_pgtable_walker *walker) { -rcu_read_unlock (); +if (walker->flags &KVM_PGTABLE_WALK_SHARED ) +rcu_read_unlock (); }static inline bool kvm_pgtable_walk_lock_held (void )
arch/arm64/kvm/hyp/pgtable.c +2 -2
@@ -289,9 +289,9 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, };int r; -kvm_pgtable_walk_begin (); +kvm_pgtable_walk_begin (walker); r =_kvm_pgtable_walk (pgt, &walk_data); -kvm_pgtable_walk_end (); +kvm_pgtable_walk_end (walker);return r; }
Signature · 1 line
2.38.1.584.g0f3c55d4c2-goog
PATCH v4 3/3 KVM: arm64: Reject shared table walks in the hyp code
In reply to Oliver UptonExclusive table walks are the only supported table walk in the hyp, as there is no construct like RCU available in the hypervisor code. Reject any attempt to do a shared table walk by returning an error and allowing the caller to clean up the mess.
--- arch/arm64/include/asm/kvm_pgtable.h | 17 +++++++++++++++-- arch/arm64/kvm/hyp/pgtable.c | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-)
arch/arm64/include/asm/kvm_pgtable.h +15 -2
@@ -229,7 +229,18 @@ static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walkereturn pteref; } -static inline void kvm_pgtable_walk_begin (struct kvm_pgtable_walker *walker) {} +static inline int kvm_pgtable_walk_begin (struct kvm_pgtable_walker *walker) +{ +/* +* Due to the lack of RCU (or a similar protection scheme), only +* non-shared table walkers are allowed in the hypervisor. +*/ +if (walker->flags &KVM_PGTABLE_WALK_SHARED ) +return -EPERM ; + +return 0 ; +} +static inline void kvm_pgtable_walk_end (struct kvm_pgtable_walker *walker) {}static inline bool kvm_pgtable_walk_lock_held (void )
@@ -247,10 +258,12 @@ static inline kvm_pte_t *kvm_dereference_pteref(struct kvm_pgtable_walker *walkereturn rcu_dereference_check (pteref, !(walker->flags &KVM_PGTABLE_WALK_SHARED )); } -static inline void kvm_pgtable_walk_begin (struct kvm_pgtable_walker *walker) +static inline int kvm_pgtable_walk_begin (struct kvm_pgtable_walker *walker) {if (walker->flags &KVM_PGTABLE_WALK_SHARED )rcu_read_lock (); + +return 0 ; }static inline void kvm_pgtable_walk_end (struct kvm_pgtable_walker *walker)
arch/arm64/kvm/hyp/pgtable.c +4 -1
@@ -289,7 +289,10 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, };int r; -kvm_pgtable_walk_begin (walker); + r =kvm_pgtable_walk_begin (walker); +if (r) +return r; + r =_kvm_pgtable_walk (pgt, &walk_data);kvm_pgtable_walk_end (walker);
Signature · 1 line
2.38.1.584.g0f3c55d4c2-goog
Quoted 5 lines · Oliver Upton
Small set of fixes for the parallel faults series. Most importantly, stop taking the RCU read lock for walking hyp stage-1. For the sake of consistency, take a pointer to
kvm_pgtable_walkerinkvm_dereference_pteref()as well.
Quoted 8 lines
Tested on an Ampere Altra system with kvm-arm.mode={nvhe,protected} and lockdep. Applies on top of the parallel faults series picked up last week.
v3: https://lore.kernel.org/kvmarm/[email protected]/
v3 -> v4:
- Return an error instead ofWARN()in hyp for shared walks (Will)
For the series:
Thanks!
Will
Quoted 10 lines · Oliver Upton
Small set of fixes for the parallel faults series. Most importantly, stop taking the RCU read lock for walking hyp stage-1. For the sake of consistency, take a pointer to
kvm_pgtable_walkerinkvm_dereference_pteref()as well.Tested on an Ampere Altra system with kvm-arm.mode={nvhe,protected} and lockdep. Applies on top of the parallel faults series picked up last week.
[...]
Applied to next, thanks!
[1/3] KVM: arm64: Take a pointer to walker data in kvm_dereference_pteref()
commit: 3a5154c723ba5ceb9ce374a7307e03263c03fd29
[2/3] KVM: arm64: Don't acquire RCU read lock for exclusive table walks
commit: b7833bf202e3068abb77c642a0843f696e9c8d38
[3/3] KVM: arm64: Reject shared table walks in the hyp code
commit: 5e806c5812e8012a83496cf96bdba266b3aec428Cheers,
M.
Signature · 1 line
Without deviation from the norm, progress is not possible.