Skip to contentSkip to conversation

KVM: SVM: Add support for 4096 vcpus with x2AVIC

PATCH Superseded Naveen N Rao (AMD) 3 participants 8 messages Feb 3, 2025 – Feb 6, 2025

Subsystems (3)

v2 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v2 cover KVM: SVM: Add support for 4096 vcpus with x2AVIC

Patch 1 adds support for up to 4096 vcpus with x2AVIC, and is unchanged since v1:
http://lkml.kernel.org/r/[email protected]

Patch 2 is new and limits the value that is programmed into AVIC_PHYSICAL_MAX_INDEX in the VMCB based on the max APIC ID indicated by the VMM.

- Naveen

Naveen N Rao (AMD) (1):
KVM: SVM: Limit AVIC physical max index based on configured
max_vcpu_ids

Suravee Suthikulpanit (1):
KVM: SVM: Increase X2AVIC limit to 4096 vcpus

 arch/x86/include/asm/svm.h |  4 ++
 arch/x86/kvm/svm/avic.c    | 82 ++++++++++++++++++++++++++++----------
 arch/x86/kvm/svm/svm.c     |  6 +++
 arch/x86/kvm/svm/svm.h     |  1 +
 4 files changed, 73 insertions(+), 20 deletions(-)

base-commit: eb723766b1030a23c38adf2348b7c3d1409d11f0

Signature · 1 line
2.48.1
#1 lore raw

PATCH v2 1/2 KVM: SVM: Increase X2AVIC limit to 4096 vcpus

From: Suravee Suthikulpanit <[email protected]>

Newer AMD platforms enhance x2AVIC feature to support up to 4096 vcpus. This capatility is detected via CPUID_Fn8000000A_ECX[x2AVIC_EXT].

Modify the SVM driver to check the capability. If detected, extend bitmask for guest max physical APIC ID to 0xFFF, increase maximum vcpu index to 4095, and increase the size of the Phyical APIC ID table from 4K to 32K in order to accommodate up to 4096 entries.

---
 arch/x86/include/asm/svm.h |  4 ++++
 arch/x86/kvm/svm/avic.c    | 42 ++++++++++++++++++++++++++------------
 2 files changed, 33 insertions(+), 13 deletions(-)
arch/x86/include/asm/svm.h +4 -0
index 2b59b9951c90..2e9728cec242 100644
@@ -268,6 +268,7 @@ enum avic_ipi_failure_cause {
 };
 
 #define AVIC_PHYSICAL_MAX_INDEX_MASK	GENMASK_ULL(8, 0)
+#define AVIC_PHYSICAL_MAX_INDEX_4K_MASK	GENMASK_ULL(11, 0)
 
 /*
  * For AVIC, the max index allowed for physical APIC ID table is 0xfe (254), as
@@ -277,11 +278,14 @@ enum avic_ipi_failure_cause {
 
 /*
  * For x2AVIC, the max index allowed for physical APIC ID table is 0x1ff (511).
+ * For extended x2AVIC, the max index allowed for physical APIC ID table is 0xfff (4095).
  */
 #define X2AVIC_MAX_PHYSICAL_ID		0x1FFUL
+#define X2AVIC_MAX_PHYSICAL_ID_4K	0xFFFUL
 
 static_assert((AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == AVIC_MAX_PHYSICAL_ID);
 static_assert((X2AVIC_MAX_PHYSICAL_ID & AVIC_PHYSICAL_MAX_INDEX_MASK) == X2AVIC_MAX_PHYSICAL_ID);
+static_assert((X2AVIC_MAX_PHYSICAL_ID_4K & AVIC_PHYSICAL_MAX_INDEX_4K_MASK) == X2AVIC_MAX_PHYSICAL_ID_4K);
 
 #define AVIC_HPA_MASK	~((0xFFFULL << 52) | 0xFFF)
 
arch/x86/kvm/svm/avic.c +29 -13

107 lines not on this page · raw message

Signature · 1 line
2.48.1
#2 lore raw

PATCH v2 2/2 KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids

In reply to Naveen N Rao (AMD)

KVM allows VMMs to specify the maximum possible APIC ID for a virtual machine through KVM_CAP_MAX_VCPU_ID capability so as to limit data structures related to APIC/x2APIC. Utilize the same to set the AVIC physical max index in the VMCB, similar to VMX. This helps hardware limit the number of entries to be scanned in the physical APIC ID table speeding up IPI broadcasts for virtual machines with smaller number of vcpus.

The minimum allocation required for the Physical APIC ID table is one 4k page supporting up to 512 entries. With AVIC support for 4096 vcpus though, it is sufficient to only allocate memory to accommodate the AVIC physical max index that will be programmed into the VMCB. Limit memory allocated for the Physical APIC ID table accordingly.

---
 arch/x86/kvm/svm/avic.c | 54 ++++++++++++++++++++++++++++++-----------
 arch/x86/kvm/svm/svm.c  |  6 +++++
 arch/x86/kvm/svm/svm.h  |  1 +
 3 files changed, 47 insertions(+), 14 deletions(-)
arch/x86/kvm/svm/avic.c +40 -14
index 4c940f4fd34d..e6ec3bcb1e37 100644
@@ -85,6 +85,17 @@ struct amd_svm_iommu_ir {
 	void *data;		/* Storing pointer to struct amd_ir_data */
 };
 
+static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
+{
+	u32 avic_max_physical_id = x2apic_mode ? x2avic_max_physical_id : AVIC_MAX_PHYSICAL_ID;
+
+	/*
+	 * Assume vcpu_id is the same as APIC ID. Per KVM_CAP_MAX_VCPU_ID, max_vcpu_ids
+	 * represents the max APIC ID for this vm, rather than the max vcpus.
+	 */
+	return min(kvm->arch.max_vcpu_ids - 1, avic_max_physical_id);
+}
+
 static void avic_activate_vmcb(struct vcpu_svm *svm)
 {
 	struct vmcb *vmcb = svm->vmcb01.ptr;
@@ -103,7 +114,7 @@ static void avic_activate_vmcb(struct vcpu_svm *svm)
 	 */
 	if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) {
 		vmcb->control.int_ctl |= X2APIC_MODE_MASK;
-		vmcb->control.avic_physical_id |= x2avic_max_physical_id;
+		vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, true);
 		/* Disabling MSR intercept for x2APIC registers */
 		svm_set_x2apic_msr_interception(svm, false);
 	} else {
@@ -114,7 +125,7 @@ static void avic_activate_vmcb(struct vcpu_svm *svm)
 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
 
 		/* For xAVIC and hybrid-xAVIC modes */
-		vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID;
+		vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, false);
 		/* Enabling MSR intercept for x2APIC registers */
 		svm_set_x2apic_msr_interception(svm, true);
 	}
@@ -174,6 +185,12 @@ int avic_ga_log_notifier(u32 ga_tag)
 	return 0;
 }
 
+static inline int avic_get_physical_id_table_order(struct kvm *kvm)
+{
+	/* Limit to the maximum physical ID supported in x2avic mode */
+	return get_order((avic_get_max_physical_id(kvm, true) + 1) * sizeof(u64));
+}
+
 void avic_vm_destroy(struct kvm *kvm)
 {
 	unsigned long flags;
@@ -185,7 +202,8 @@ void avic_vm_destroy(struct kvm *kvm)
 	if (kvm_svm->avic_logical_id_table_page)
 		__free_page(kvm_svm->avic_logical_id_table_page);
 	if (kvm_svm->avic_physical_id_table_page)
-		__free_page(kvm_svm->avic_physical_id_table_page);
+		__free_pages(kvm_svm->avic_physical_id_table_page,
+			     avic_get_physical_id_table_order(kvm));
 
 	spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 	hash_del(&kvm_svm->hnode);
@@ -198,22 +216,12 @@ int avic_vm_init(struct kvm *kvm)
 	int err = -ENOMEM;
 	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 	struct kvm_svm *k2;
-	struct page *p_page;
 	struct page *l_page;
-	u32 vm_id, entries;
+	u32 vm_id;
 
 	if (!enable_apicv)
 		return 0;
 
-	/* Allocating physical APIC ID table */
-	entries = x2avic_max_physical_id + 1;
-	p_page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO,
-			     get_order(sizeof(u64) * entries));
-	if (!p_page)
-		goto free_avic;
-
-	kvm_svm->avic_physical_id_table_page = p_page;
-
 	/* Allocating logical APIC ID table (4KB) */
 	l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 	if (!l_page)
@@ -264,6 +272,24 @@ void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb)
 		avic_deactivate_vmcb(svm);
 }
 
+int avic_alloc_physical_id_table(struct kvm *kvm)
+{
+	struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
+	struct page *p_page;
+
+	if (kvm_svm->avic_physical_id_table_page || !enable_apicv || !irqchip_in_kernel(kvm))
+		return 0;
+
+	p_page = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO,
+			     avic_get_physical_id_table_order(kvm));
+	if (!p_page)
+		return -ENOMEM;
+
+	kvm_svm->avic_physical_id_table_page = p_page;
+
+	return 0;
+}
+
 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
 				       unsigned int index)
 {
arch/x86/kvm/svm/svm.c +6 -0
index 7640a84e554a..19b9ebea4773 100644
@@ -1423,6 +1423,11 @@ void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb)
 	svm->vmcb = target_vmcb->ptr;
 }
 
+static int svm_vcpu_precreate(struct kvm *kvm)
+{
+	return avic_alloc_physical_id_table(kvm);
+}
+
 static int svm_vcpu_create(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm;
@@ -5009,6 +5014,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 	.emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu,
 	.has_emulated_msr = svm_has_emulated_msr,
 
+	.vcpu_precreate = svm_vcpu_precreate,
 	.vcpu_create = svm_vcpu_create,
 	.vcpu_free = svm_vcpu_free,
 	.vcpu_reset = svm_vcpu_reset,
arch/x86/kvm/svm/svm.h +1 -0
index 9d7cdb8fbf87..68687c3bcce7 100644
@@ -696,6 +696,7 @@ bool avic_hardware_setup(void);
 int avic_ga_log_notifier(u32 ga_tag);
 void avic_vm_destroy(struct kvm *kvm);
 int avic_vm_init(struct kvm *kvm);
+int avic_alloc_physical_id_table(struct kvm *kvm);
 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb);
 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu);
 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu);
Signature · 1 line
2.48.1
#3 lore raw

Re: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids

kernel test robot noticed the following build errors:

Hi Naveen,

kernel test robot noticed the following build errors:

[auto build test ERROR on eb723766b1030a23c38adf2348b7c3d1409d11f0]

url:    https://github.com/intel-lab-lkp/linux/commits/Naveen-N-Rao-AMD/KVM-SVM-Increase-X2AVIC-limit-to-4096-vcpus/20250203-144127
base:   eb723766b1030a23c38adf2348b7c3d1409d11f0
patch link:    https://lore.kernel.org/r/b79610c60de53048f3fda942fd45973c4ab1de97.1738563890.git.naveen%40kernel.org
patch subject: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids
config: x86_64-buildonly-randconfig-003-20250203 (https://download.01.org/0day-ci/archive/20250203/[email protected]/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250203/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from arch/x86/include/asm/kvm_host.h:31,

                    from include/linux/kvm_host.h:45,
                    from arch/x86/kvm/svm/avic.c:20:
>> arch/x86/include/asm/apic.h:258:34: error: expected ')' before numeric constant
     258 | #define x2apic_mode             (0)
         |                                  ^
   arch/x86/kvm/svm/avic.c:88:66: note: in expansion of macro 'x2apic_mode'
      88 | static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
         |                                                                  ^~~~~~~~~~~
   arch/x86/kvm/svm/avic.c: In function 'avic_activate_vmcb':
>> arch/x86/kvm/svm/avic.c:117:51: error: implicit declaration of function 'avic_get_max_physical_id' [-Werror=implicit-function-declaration]
     117 |                 vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, true);
         |                                                   ^~~~~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors

vim +/avic_get_max_physical_id +117 arch/x86/kvm/svm/avic.c

98
99 static void avic_activate_vmcb(struct vcpu_svm *svm)
100 {
101 struct vmcb *vmcb = svm->vmcb01.ptr;
102
103 vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK);
104 vmcb->control.avic_physical_id &= ~avic_physical_max_index_mask;
105
106 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
107
108 /*
109 * Note: KVM supports hybrid-AVIC mode, where KVM emulates x2APIC MSR
110 * accesses, while interrupt injection to a running vCPU can be
111 * achieved using AVIC doorbell. KVM disables the APIC access page
112 * (deletes the memslot) if any vCPU has x2APIC enabled, thus enabling
113 * AVIC in hybrid mode activates only the doorbell mechanism.
114 */
115 if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) {
116 vmcb->control.int_ctl |= X2APIC_MODE_MASK;

 > 117			vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, true);

118 /* Disabling MSR intercept for x2APIC registers */
119 svm_set_x2apic_msr_interception(svm, false);
120 } else {
121 /*
122 * Flush the TLB, the guest may have inserted a non-APIC
123 * mapping into the TLB while AVIC was disabled.
124 */
125 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
126
127 /* For xAVIC and hybrid-xAVIC modes */
128 vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, false);
129 /* Enabling MSR intercept for x2APIC registers */
130 svm_set_x2apic_msr_interception(svm, true);
131 }
132 }
133

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#4 lore raw

Re: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids

In reply to Naveen N Rao (AMD)
kernel test robot noticed the following build warnings:

Hi Naveen,

kernel test robot noticed the following build warnings:

[auto build test WARNING on eb723766b1030a23c38adf2348b7c3d1409d11f0]

url:    https://github.com/intel-lab-lkp/linux/commits/Naveen-N-Rao-AMD/KVM-SVM-Increase-X2AVIC-limit-to-4096-vcpus/20250203-144127
base:   eb723766b1030a23c38adf2348b7c3d1409d11f0
patch link:    https://lore.kernel.org/r/b79610c60de53048f3fda942fd45973c4ab1de97.1738563890.git.naveen%40kernel.org
patch subject: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids
config: x86_64-randconfig-008-20250203 (https://download.01.org/0day-ci/archive/20250204/[email protected]/config)
compiler: clang version 19.1.3 (https://github.com/llvm/llvm-project ab51eccf88f5321e7c60591c5546b254b6afab99)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250204/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All warnings (new ones prefixed by >>):

In file included from arch/x86/kvm/svm/avic.c:20:
In file included from include/linux/kvm_host.h:16:
In file included from include/linux/mm.h:2223:
include/linux/vmstat.h:518:36: warning: arithmetic between different enumeration types ('enum node_stat_item' and 'enum lru_list') [-Wenum-enum-conversion]

     518 |         return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
         |                               ~~~~~~~~~~~ ^ ~~~
   arch/x86/kvm/svm/avic.c:88:66: error: expected ')'
      88 | static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
         |                                                                  ^
   arch/x86/include/asm/apic.h:258:23: note: expanded from macro 'x2apic_mode'
     258 | #define x2apic_mode             (0)
         |                                  ^
   arch/x86/kvm/svm/avic.c:88:66: note: to match this '('
   arch/x86/include/asm/apic.h:258:22: note: expanded from macro 'x2apic_mode'
     258 | #define x2apic_mode             (0)
         |                                 ^
>> arch/x86/kvm/svm/avic.c:88:66: warning: omitting the parameter name in a function definition is a C23 extension [-Wc23-extensions]

88 | static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
| ^
arch/x86/include/asm/apic.h:258:23: note: expanded from macro 'x2apic_mode'
258 | #define x2apic_mode (0)
| ^
2 warnings and 1 error generated.

vim +88 arch/x86/kvm/svm/avic.c

    87	
  > 88	static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
    89	{
    90		u32 avic_max_physical_id = x2apic_mode ? x2avic_max_physical_id : AVIC_MAX_PHYSICAL_ID;
    91	
    92		/*
    93		 * Assume vcpu_id is the same as APIC ID. Per KVM_CAP_MAX_VCPU_ID, max_vcpu_ids
    94		 * represents the max APIC ID for this vm, rather than the max vcpus.
    95		 */
    96		return min(kvm->arch.max_vcpu_ids - 1, avic_max_physical_id);
    97	}
    98	
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#5 lore raw

Re: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids

In reply to Naveen N Rao (AMD)
kernel test robot noticed the following build errors:

Hi Naveen,

kernel test robot noticed the following build errors:

[auto build test ERROR on eb723766b1030a23c38adf2348b7c3d1409d11f0]

url:    https://github.com/intel-lab-lkp/linux/commits/Naveen-N-Rao-AMD/KVM-SVM-Increase-X2AVIC-limit-to-4096-vcpus/20250203-144127
base:   eb723766b1030a23c38adf2348b7c3d1409d11f0
patch link:    https://lore.kernel.org/r/b79610c60de53048f3fda942fd45973c4ab1de97.1738563890.git.naveen%40kernel.org
patch subject: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids
config: x86_64-randconfig-008-20250203 (https://download.01.org/0day-ci/archive/20250204/[email protected]/config)
compiler: clang version 19.1.3 (https://github.com/llvm/llvm-project ab51eccf88f5321e7c60591c5546b254b6afab99)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20250204/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

In file included from arch/x86/kvm/svm/avic.c:20:
In file included from include/linux/kvm_host.h:16:
In file included from include/linux/mm.h:2223:
include/linux/vmstat.h:518:36: warning: arithmetic between different enumeration types ('enum node_stat_item' and 'enum lru_list') [-Wenum-enum-conversion]

     518 |         return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
         |                               ~~~~~~~~~~~ ^ ~~~
>> arch/x86/kvm/svm/avic.c:88:66: error: expected ')'
      88 | static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
         |                                                                  ^
   arch/x86/include/asm/apic.h:258:23: note: expanded from macro 'x2apic_mode'
     258 | #define x2apic_mode             (0)
         |                                  ^
   arch/x86/kvm/svm/avic.c:88:66: note: to match this '('
   arch/x86/include/asm/apic.h:258:22: note: expanded from macro 'x2apic_mode'
     258 | #define x2apic_mode             (0)
         |                                 ^

arch/x86/kvm/svm/avic.c:88:66: warning: omitting the parameter name in a function definition is a C23 extension [-Wc23-extensions]
88 | static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
| ^
arch/x86/include/asm/apic.h:258:23: note: expanded from macro 'x2apic_mode'
258 | #define x2apic_mode (0)
| ^
2 warnings and 1 error generated.

vim +88 arch/x86/kvm/svm/avic.c

    87	
  > 88	static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
    89	{
    90		u32 avic_max_physical_id = x2apic_mode ? x2avic_max_physical_id : AVIC_MAX_PHYSICAL_ID;
    91	
    92		/*
    93		 * Assume vcpu_id is the same as APIC ID. Per KVM_CAP_MAX_VCPU_ID, max_vcpu_ids
    94		 * represents the max APIC ID for this vm, rather than the max vcpus.
    95		 */
    96		return min(kvm->arch.max_vcpu_ids - 1, avic_max_physical_id);
    97	}
    98	
Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#6 lore raw

Re: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids

In reply to Naveen N Rao (AMD)

Hi Naveen,

Quoted 75 lines · Naveen N Rao (AMD)

KVM allows VMMs to specify the maximum possible APIC ID for a virtual machine through KVM_CAP_MAX_VCPU_ID capability so as to limit data structures related to APIC/x2APIC. Utilize the same to set the AVIC physical max index in the VMCB, similar to VMX. This helps hardware limit the number of entries to be scanned in the physical APIC ID table speeding up IPI broadcasts for virtual machines with smaller number of vcpus.

The minimum allocation required for the Physical APIC ID table is one 4k page supporting up to 512 entries. With AVIC support for 4096 vcpus though, it is sufficient to only allocate memory to accommodate the AVIC physical max index that will be programmed into the VMCB. Limit memory allocated for the Physical APIC ID table accordingly.

---
 arch/x86/kvm/svm/avic.c | 54 ++++++++++++++++++++++++++++++-----------
 arch/x86/kvm/svm/svm.c  |  6 +++++
 arch/x86/kvm/svm/svm.h  |  1 +
 3 files changed, 47 insertions(+), 14 deletions(-)
arch/x86/kvm/svm/avic.c +19 -2
index 4c940f4fd34d..e6ec3bcb1e37 100644
@@ -85,6 +85,17 @@ struct amd_svm_iommu_ir {
 	void *data;		/* Storing pointer to struct amd_ir_data */
 };
 
+static inline u32 avic_get_max_physical_id(struct kvm *kvm, bool x2apic_mode)
+{
+	u32 avic_max_physical_id = x2apic_mode ? x2avic_max_physical_id : AVIC_MAX_PHYSICAL_ID;
+
+	/*
+	 * Assume vcpu_id is the same as APIC ID. Per KVM_CAP_MAX_VCPU_ID, max_vcpu_ids
+	 * represents the max APIC ID for this vm, rather than the max vcpus.
+	 */
+	return min(kvm->arch.max_vcpu_ids - 1, avic_max_physical_id);
+}
+
 static void avic_activate_vmcb(struct vcpu_svm *svm)
 {
 	struct vmcb *vmcb = svm->vmcb01.ptr;
@@ -103,7 +114,7 @@ static void avic_activate_vmcb(struct vcpu_svm *svm)
 	 */
 	if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) {
 		vmcb->control.int_ctl |= X2APIC_MODE_MASK;
-		vmcb->control.avic_physical_id |= x2avic_max_physical_id;
+		vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, true);
 		/* Disabling MSR intercept for x2APIC registers */
 		svm_set_x2apic_msr_interception(svm, false);
 	} else {
@@ -114,7 +125,7 @@ static void avic_activate_vmcb(struct vcpu_svm *svm)
 		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
 
 		/* For xAVIC and hybrid-xAVIC modes */
-		vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID;
+		vmcb->control.avic_physical_id |= avic_get_max_physical_id(svm->vcpu.kvm, false);
 		/* Enabling MSR intercept for x2APIC registers */
 		svm_set_x2apic_msr_interception(svm, true);
 	}
@@ -174,6 +185,12 @@ int avic_ga_log_notifier(u32 ga_tag)
 	return 0;
 }
 
+static inline int avic_get_physical_id_table_order(struct kvm *kvm)
+{
+	/* Limit to the maximum physical ID supported in x2avic mode */
+	return get_order((avic_get_max_physical_id(kvm, true) + 1) * sizeof(u64));
+}
+
 void avic_vm_destroy(struct kvm *kvm)
 {
 	unsigned long flags;

arch/x86/kvm/svm/avic.c void avic_vm_destroy(struct kvm *kvm) · line 206

Quoted 6 lines
arch/x86/kvm/svm/avic.c +2 -1
@@ -185,7 +202,8 @@ void avic_vm_destroy(struct kvm *kvm)
 	if (kvm_svm->avic_logical_id_table_page)
 		__free_page(kvm_svm->avic_logical_id_table_page);
 	if (kvm_svm->avic_physical_id_table_page)
-		__free_page(kvm_svm->avic_physical_id_table_page);
+		__free_pages(kvm_svm->avic_physical_id_table_page,
+			     avic_get_physical_id_table_order(kvm));

Move this hunk to previous patch (1/2) ?

Rest looks good.

-Vasant

#7 lore raw

Re: [PATCH v2 2/2] KVM: SVM: Limit AVIC physical max index based on configured max_vcpu_ids

Hi Vasant,

Quoted 4 lines · Vasant Hegde

Hi Naveen,

On 2/3/2025 12:07 PM, Naveen N Rao (AMD) wrote:

<snip>

Quoted 13 lines

arch/x86/kvm/svm/avic.c void avic_vm_destroy(struct kvm *kvm) · line 206

Quoted 10 lines

void avic_vm_destroy(struct kvm *kvm)
{

arch/x86/kvm/svm/avic.c +2 -1
 	unsigned long flags;
@@ -185,7 +202,8 @@ void avic_vm_destroy(struct kvm *kvm)
 	if (kvm_svm->avic_logical_id_table_page)
 		__free_page(kvm_svm->avic_logical_id_table_page);
 	if (kvm_svm->avic_physical_id_table_page)
-		__free_page(kvm_svm->avic_physical_id_table_page);
+		__free_pages(kvm_svm->avic_physical_id_table_page,
+			     avic_get_physical_id_table_order(kvm));

Move this hunk to previous patch (1/2) ?

Quoted 1 line

Rest looks good.

Sure, thanks for the review!

- Naveen