KVM: x86: (64-bit x86_64 machine) : fix -frame-larger-than warnings/errors.
Subsystems (4)
PATCH 1/1 KVM: x86: (64-bit x86_64 machine) : fix -frame-larger-than warnings/errors.
From: ajay <[email protected]>
Issue :
=======
In "kvm_hv_flush_tlb" and "kvm_hv_send_ipi" methods, defining "u64 sparse_banks[64]" inside the methods (on the stack), causes the stack-segment-memory-allocation to go beyond 1024 bytes, thus raising the warning/error which breaks the build.
Fix :
=====
Instead of defining "u64 sparse_banks [64]" inside the methods, we instead define this array in the (only) client method "kvm_hv_hypercall", and then pass the array (and its size) as additional arguments to the two methods.
Doing this, we do not exceed the 1024 bytes stack-segment-memory-allocation, on any stack-segment of any method.
--- arch/x86/kvm/hyperv.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-)
arch/x86/kvm/hyperv.c +24 -10
@@ -1750,7 +1750,8 @@ struct kvm_hv_hcall {sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS ]; }; -static u64 kvm_hv_flush_tlb (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc,bool ex) +static u64 kvm_hv_flush_tlb (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc, +bool ex,u64 *sparse_banks,u32 num_sparse_banks) {int i;gpa_t gpa;
@@ -1762,10 +1763,11 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolDECLARE_BITMAP (vcpu_bitmap,KVM_MAX_VCPUS );unsigned long *vcpu_mask;u64 valid_bank_mask; -u64 sparse_banks[64 ];int sparse_banks_len;bool all_cpus; +memset (sparse_banks,0 ,sizeof (u64) * num_sparse_banks); +if (!ex) {if (hc->fast) { flush.address_space = hc->ingpa;
@@ -1875,7 +1877,8 @@ static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector, } } -static u64 kvm_hv_send_ipi (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc,bool ex) +static u64 kvm_hv_send_ipi (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc, +bool ex,u64 *sparse_banks,u32 num_sparse_banks) {struct kvm *kvm = vcpu->kvm;struct hv_send_ipi_ex send_ipi_ex;
@@ -1884,11 +1887,12 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolDECLARE_BITMAP (vcpu_bitmap,KVM_MAX_VCPUS );unsigned long *vcpu_mask;unsigned long valid_bank_mask; -u64 sparse_banks[64 ];int sparse_banks_len;u32 vector;bool all_cpus; +memset (sparse_banks,0 ,sizeof (u64) * num_sparse_banks); +if (!ex) {if (!hc->fast) { if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi,
@@ -2162,6 +2166,10 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu)struct kvm_hv_hcall hc;u64 ret =HV_STATUS_SUCCESS ; +#define NUM_SPARSE_BANKS 64 + +u64 sparse_banks[NUM_SPARSE_BANKS ]; +/* * hypercall generates UD from non zero cpl and real mode * per HYPER-V spec
@@ -2248,42 +2256,48 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) ret =HV_STATUS_INVALID_HYPERCALL_INPUT ;break ; } - ret =kvm_hv_flush_tlb (vcpu, &hc, false); + ret =kvm_hv_flush_tlb (vcpu, &hc, false, + sparse_banks,NUM_SPARSE_BANKS );break ;case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE :if (unlikely (hc.rep)) { ret =HV_STATUS_INVALID_HYPERCALL_INPUT ;break ; } - ret =kvm_hv_flush_tlb (vcpu, &hc, false); + ret =kvm_hv_flush_tlb (vcpu, &hc, false, + sparse_banks,NUM_SPARSE_BANKS );break ;case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX :if (unlikely (!hc.rep_cnt || hc.rep_idx)) { ret =HV_STATUS_INVALID_HYPERCALL_INPUT ;break ; } - ret =kvm_hv_flush_tlb (vcpu, &hc, true); + ret =kvm_hv_flush_tlb (vcpu, &hc, true, + sparse_banks,NUM_SPARSE_BANKS );break ;case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX :if (unlikely (hc.rep)) { ret =HV_STATUS_INVALID_HYPERCALL_INPUT ;break ; } - ret =kvm_hv_flush_tlb (vcpu, &hc, true); + ret =kvm_hv_flush_tlb (vcpu, &hc, true, + sparse_banks,NUM_SPARSE_BANKS );break ;case HVCALL_SEND_IPI :if (unlikely (hc.rep)) { ret =HV_STATUS_INVALID_HYPERCALL_INPUT ;break ; } - ret =kvm_hv_send_ipi (vcpu, &hc, false); + ret =kvm_hv_send_ipi (vcpu, &hc, false, + sparse_banks,NUM_SPARSE_BANKS );break ;case HVCALL_SEND_IPI_EX :if (unlikely (hc.fast || hc.rep)) { ret =HV_STATUS_INVALID_HYPERCALL_INPUT ;break ; } - ret =kvm_hv_send_ipi (vcpu, &hc, true); + ret =kvm_hv_send_ipi (vcpu, &hc, true, + sparse_banks,NUM_SPARSE_BANKS );break ;case HVCALL_POST_DEBUG_DATA :case HVCALL_RETRIEVE_DEBUG_DATA :
Signature · 1 line
2.30.2
Sean Christopherson
maintainerQuoted 13 lines · Ajay Garg
From: ajay <[email protected]>
Issue :
=======In "
kvm_hv_flush_tlb" and "kvm_hv_send_ipi" methods, defining "u64 sparse_banks[64]" inside the methods (on the stack), causes the stack-segment-memory-allocation to go beyond 1024 bytes, thus raising the warning/error which breaks the build.Fix :
=====
Quoted 6 lines
Instead of defining "u64
sparse_banks[64]" inside the methods, we instead define this array in the (only) client method "kvm_hv_hypercall", and then pass the array (and its size) as additional arguments to the two methods.Doing this, we do not exceed the 1024 bytes stack-segment-memory-allocation, on any stack-segment of any method.
This is a hack, and it's not guaranteed to work, e.g. if the compiler decided to inline the helpers, then presumably this problem would rear its head again.
However, I don't think this is a problem any more. gcc-10 and clang-11 are both comfortably under 1024, even if I force both helpers to be inlined. Neither function has variables that would scale with NR_CPUS (and I verified high number of NR_CPUS for giggles). Can you try reproducing the behavior on the latest kvm/queue? I swear I've seen this in the past, but I couldn't find a commit that "fixed" any such warning.
If it does repro, can you provide your .config and compiler version? Maybe your compiler is doing somethign funky?
Quoted 1 line
Signed-off-by ajay <[email protected]>
The SoB needs your full name.
Quoted 21 lines
--- arch/x86/kvm/hyperv.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-)arch/x86/kvm/hyperv.c +2
-1@@ -1750,7 +1750,8 @@ struct kvm_hv_hcall {sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS ]; }; -static u64 kvm_hv_flush_tlb (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc,bool ex) +static u64 kvm_hv_flush_tlb (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc, +bool ex,u64 *sparse_banks,u32 num_sparse_banks) {arch/x86/kvm/hyperv.c +0
-0int i;gpa_t gpa;
arch/x86/kvm/hyperv.c static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool · line 1770
Quoted 9 lines
arch/x86/kvm/hyperv.c +2
-1@@ -1762,10 +1763,11 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolDECLARE_BITMAP (vcpu_bitmap,KVM_MAX_VCPUS );unsigned long *vcpu_mask;u64 valid_bank_mask; -u64 sparse_banks[64 ];int sparse_banks_len;bool all_cpus; +memset (sparse_banks,0 ,sizeof (u64) * num_sparse_banks); +
FWIW, the array size needs to be validated, there is other code in this function that assumes it's at least 64 entries.
Hi Sean,
Thanks for your time.
I have cloned the kernel many times since the time the patch was posted, and have not seen the issue again.
One thing I distinctly remember that when the build was breaking, it was with staging-drivers disabled. Since then, I have disabled the staging-drivers. Today, I again enabled staging-drivers, and build went fine.
So, I am ok with archiving this patch. We can revisit if someone else reports this/similar issue.
Thanks and Regards,
Ajay
Quoted 72 lines · Sean Christopherson
Quoted 13 lines · Ajay Garg
From: ajay <[email protected]>
Issue :
=======In "
kvm_hv_flush_tlb" and "kvm_hv_send_ipi" methods, defining "u64 sparse_banks[64]" inside the methods (on the stack), causes the stack-segment-memory-allocation to go beyond 1024 bytes, thus raising the warning/error which breaks the build.Fix :
=====Quoted 6 lines
Instead of defining "u64
sparse_banks[64]" inside the methods, we instead define this array in the (only) client method "kvm_hv_hypercall", and then pass the array (and its size) as additional arguments to the two methods.Doing this, we do not exceed the 1024 bytes stack-segment-memory-allocation, on any stack-segment of any method.
This is a hack, and it's not guaranteed to work, e.g. if the compiler decided to inline the helpers, then presumably this problem would rear its head again.
However, I don't think this is a problem any more. gcc-10 and clang-11 are both comfortably under 1024, even if I force both helpers to be inlined. Neither function has variables that would scale with
NR_CPUS(and I verified high number ofNR_CPUSfor giggles). Can you try reproducing the behavior on the latest kvm/queue? I swear I've seen this in the past, but I couldn't find a commit that "fixed" any such warning.If it does repro, can you provide your .config and compiler version? Maybe your compiler is doing somethign funky?
Quoted 1 line
Signed-off-by ajay <[email protected]>The SoB needs your full name.
Quoted 21 lines
--- arch/x86/kvm/hyperv.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-)arch/x86/kvm/hyperv.c +2
-1@@ -1750,7 +1750,8 @@ struct kvm_hv_hcall {sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS ]; }; -static u64 kvm_hv_flush_tlb (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc,bool ex) +static u64 kvm_hv_flush_tlb (struct kvm_vcpu *vcpu,struct kvm_hv_hcall *hc, +bool ex,u64 *sparse_banks,u32 num_sparse_banks) {int i; gpa_t gpa;arch/x86/kvm/hyperv.c static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool · line 1770
Quoted 9 lines
arch/x86/kvm/hyperv.c +2
-1@@ -1762,10 +1763,11 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolDECLARE_BITMAP (vcpu_bitmap,KVM_MAX_VCPUS );unsigned long *vcpu_mask;u64 valid_bank_mask; -u64 sparse_banks[64 ];int sparse_banks_len;bool all_cpus; +memset (sparse_banks,0 ,sizeof (u64) * num_sparse_banks); +FWIW, the array size needs to be validated, there is other code in this function that assumes it's at least 64 entries.
A message that is not in the archive (sent off-list or not yet received): CAHP4M8VzjPgzBmfn2DAdGD0P9OBF7_cafPP8nrjvTampGLoxyA@mail.gmail.com
Sean Christopherson
maintainerQuoted 6 lines · Ajay Garg
Hi Sean.
Today I enabled a debug-build, and the compilation broke again.
1.
Please find attached the .config file.
Aha! The problem is CONFIG_KASAN_STACK=y, which is selected (and can't be unselected) by CONFIG_KASAN=y when compiling with gcc (clang/LLVM is a stack hog in some cases so it's opt-in for clang). KASAN_STACK adds a redzone around every stack variable, which pushes the Hyper-V functions over the limit.
Quoted 1 line
Please let know if/when I should float a v2 patch.
I still don't love hoisting sparse_banks up a level, that info really shouldn't bleed into the caller. My alternative solution is to push vp_bitmap down into sparse_set_to_vcpu_mask(). That doesn't "free" up as much stack, but it's enough to get under the 1024 byte default. It's also nice in that it hides the VP index mismatch logic in sparse_set_to_vcpu_mask().
I'll post a proper patch tomorrow (completely untested):
--- arch/x86/kvm/hyperv.c | 55 ++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 27 deletions(-)
arch/x86/kvm/hyperv.c +28 -27
@@ -1710,31 +1710,36 @@ int kvm_hv_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, bool host)return kvm_hv_get_msr (vcpu, msr, pdata, host); } -static __always_inline unsigned long *sparse_set_to_vcpu_mask ( -struct kvm *kvm,u64 *sparse_banks,u64 valid_bank_mask, -u64 *vp_bitmap,unsigned long *vcpu_bitmap) +static void sparse_set_to_vcpu_mask (struct kvm *kvm,u64 *sparse_banks, +u64 valid_bank_mask,unsigned long *vcpu_mask) {struct kvm_hv *hv =to_kvm_hv (kvm); +bool has_mismatch =atomic_read (&hv->num_mismatched_vp_indexes); +u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS ];struct kvm_vcpu *vcpu;int i, bank, sbank =0 ; +u64 *bitmap; -memset (vp_bitmap,0 , -KVM_HV_MAX_SPARSE_VCPU_SET_BITS *sizeof (*vp_bitmap)); +BUILD_BUG_ON (sizeof (vp_bitmap) > +sizeof (*vcpu_mask) *BITS_TO_LONGS (KVM_MAX_VCPUS )); + +/* If vp_index == vcpu_idx for all vCPUs, fill vcpu_mask directly. */ +if (likely (!has_mismatch)) + bitmap = (u64 *)vcpu_mask; + +memset (bitmap,0 ,sizeof (vp_bitmap));for_each_set_bit (bank, (unsigned long *)&valid_bank_mask,KVM_HV_MAX_SPARSE_VCPU_SET_BITS ) - vp_bitmap[bank] = sparse_banks[sbank++]; + bitmap[bank] = sparse_banks[sbank++]; -if (likely (!atomic_read (&hv->num_mismatched_vp_indexes))) { -/* for all vcpus vp_index == vcpu_idx */ -return (unsigned long *)vp_bitmap; - } +if (likely (!has_mismatch)) +return ; -bitmap_zero (vcpu_bitmap,KVM_MAX_VCPUS ); +bitmap_zero (vcpu_mask,KVM_MAX_VCPUS );kvm_for_each_vcpu (i, vcpu, kvm) {if (test_bit (kvm_hv_get_vpindex (vcpu), (unsigned long *)vp_bitmap)) -__set_bit (i, vcpu_bitmap); +__set_bit (i, vcpu_mask); } -return vcpu_bitmap; }struct kvm_hv_hcall {
@@ -1756,9 +1761,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolstruct kvm *kvm = vcpu->kvm;struct hv_tlb_flush_ex flush_ex;struct hv_tlb_flush flush; -u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS ]; -DECLARE_BITMAP (vcpu_bitmap,KVM_MAX_VCPUS ); -unsigned long *vcpu_mask; +DECLARE_BITMAP (vcpu_mask,KVM_MAX_VCPUS );u64 valid_bank_mask;u64 sparse_banks[64 ];int sparse_banks_len;
@@ -1842,11 +1845,9 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolif (all_cpus) {kvm_make_all_cpus_request (kvm,KVM_REQ_TLB_FLUSH_GUEST ); }else { - vcpu_mask =sparse_set_to_vcpu_mask (kvm, sparse_banks, valid_bank_mask, - vp_bitmap, vcpu_bitmap); +sparse_set_to_vcpu_mask (kvm, sparse_banks, valid_bank_mask, vcpu_mask); -kvm_make_vcpus_request_mask (kvm,KVM_REQ_TLB_FLUSH_GUEST , - vcpu_mask); +kvm_make_vcpus_request_mask (kvm,KVM_REQ_TLB_FLUSH_GUEST , vcpu_mask); } ret_success:
@@ -1879,9 +1880,7 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolstruct kvm *kvm = vcpu->kvm;struct hv_send_ipi_ex send_ipi_ex;struct hv_send_ipi send_ipi; -u64 vp_bitmap[KVM_HV_MAX_SPARSE_VCPU_SET_BITS ]; -DECLARE_BITMAP (vcpu_bitmap,KVM_MAX_VCPUS ); -unsigned long *vcpu_mask; +DECLARE_BITMAP (vcpu_mask,KVM_MAX_VCPUS );unsigned long valid_bank_mask;u64 sparse_banks[64 ];int sparse_banks_len;
@@ -1937,11 +1936,13 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, boolif ((vector <HV_IPI_LOW_VECTOR ) || (vector >HV_IPI_HIGH_VECTOR ))return HV_STATUS_INVALID_HYPERCALL_INPUT ; - vcpu_mask = all_cpus ?NULL : -sparse_set_to_vcpu_mask (kvm, sparse_banks, valid_bank_mask, - vp_bitmap, vcpu_bitmap); +if (all_cpus) { +kvm_send_ipi_to_many (kvm, vector,NULL ); + }else { +sparse_set_to_vcpu_mask (kvm, sparse_banks, valid_bank_mask, vcpu_mask); -kvm_send_ipi_to_many (kvm, vector, vcpu_mask); +kvm_send_ipi_to_many (kvm, vector, vcpu_mask); + } ret_success:return HV_STATUS_SUCCESS ;
--
2.33.0.1079.g6e70778dc9-goog