HVM/ASID: enable flush-by-ASID for HVM

diff -r f531ed84b066 -r cf89d82ba40b xen/arch/x86/hvm/asid.c
@@ -48,12 +48,15 @@
 
 /* Per-CPU ASID management. */
 struct hvm_asid_data {
-   uint64_t core_asid_generation;
-   uint32_t next_asid;
-   uint32_t max_asid;
-   bool_t disabled;
+    uint64_t core_asid_generation;
+    uint32_t core_id;
+    uint32_t next_asid;
+    uint32_t max_asid;
+    bool_t disabled;
 };
 
+uint32_t hvm_asid_features;
+
 static DEFINE_PER_CPU(struct hvm_asid_data, hvm_asid_data);
 
 void hvm_asid_init(int nasids)

@@ -66,11 +69,25 @@
 
     if ( g_disabled != data->disabled )
     {
-        printk("HVM: ASIDs %sabled.\n", data->disabled ? "dis" : "en");
+        printk("HVM: ASIDs %sabled", data->disabled ? "dis" : "en");
+        
+        if ( data->disabled )
+            printk("\n");
+        else
+            printk(" (# of ASIDs = 0x%x, flush by ASID = %s)\n", nasids,
+                   (hvm_asid_features & HVM_ASID_FEATURE_FLUSH_ITSELF)
+                   ? "true" : "false");
+
         if ( g_disabled < 0 )
             g_disabled = data->disabled;
     }
 
+    /* Initialize core_id to physical CPU ID. Note that the core_id is bumped 
+     * up by 1. This prevents a corner case when a new vcpu with core_id=0 
+     * (i.e. just initialized) is executed on CPU core 0. Without this trick,
+     * the vcpu will miss the opportunity of increasing ASID. */
+    data->core_id = smp_processor_id() + 1;
+
     /* Zero indicates 'invalid generation', so we start the count at one. */
     data->core_asid_generation = 1;
 

@@ -108,18 +125,28 @@
     data->disabled = 1;
 }
 
-bool_t hvm_asid_handle_vmenter(struct hvm_vcpu_asid *asid)
+hvm_asid_action_t hvm_asid_handle_vmenter(struct hvm_vcpu_asid *asid)
 {
     struct hvm_asid_data *data = &this_cpu(hvm_asid_data);
 
-    /* On erratum #170 systems we must flush the TLB. 
-     * Generation overruns are taken here, too. */
+    /* Generation overruns are taken here. */
     if ( data->disabled )
         goto disabled;
 
     /* Test if VCPU has valid ASID. */
-    if ( asid->generation == data->core_asid_generation )
-        return 0;
+    if ( asid->core_id == data->core_id && 
+         asid->generation == data->core_asid_generation )
+        return HVM_ASID_FLUSH_NONE;
+
+    /* If (1) VCPU wasn't migrated from other CPU, (2) VCPU just requested 
+     * TLB flush, and (3) CPU supports flush by ASID, then we skip ASID 
+     * incremental and return directly from here. */
+    if ( asid->core_id == data->core_id && asid->generation == 0 &&
+         (hvm_asid_features & HVM_ASID_FEATURE_FLUSH_ITSELF) )
+    {
+        asid->generation = data->core_asid_generation;
+        return HVM_ASID_FLUSH_ITSELF;
+    }
 
     /* If there are no free ASIDs, need to go to a new generation */
     if ( unlikely(data->next_asid > data->max_asid) )

@@ -133,16 +160,18 @@
     /* Now guaranteed to be a free ASID. */
     asid->asid = data->next_asid++;
     asid->generation = data->core_asid_generation;
+    asid->core_id = data->core_id;
 
     /*
      * When we assign ASID 1, flush all TLB entries as we are starting a new
-     * generation, and all old ASID allocations are now stale. 
+     * generation, and all old ASID allocations are now stale. Otherwise, 
+     * nothing needs to be flushed.
      */
-    return (asid->asid == 1);
+    return (asid->asid == 1) ? HVM_ASID_FLUSH_ALL : HVM_ASID_FLUSH_NONE;
 
  disabled:
     asid->asid = 0;
-    return 0;
+    return HVM_ASID_FLUSH_NONE;
 }
 
 /*

diff -r f531ed84b066 -r cf89d82ba40b xen/arch/x86/hvm/svm/asid.c
@@ -32,6 +32,10 @@
     if ( !cpu_has_amd_erratum(c, AMD_ERRATUM_170) )
         nasids = cpuid_ebx(0x8000000A);
 
+    hvm_asid_features = 0;
+    if ( cpu_has_svm_flushasid )
+        hvm_asid_features |= HVM_ASID_FEATURE_FLUSH_ITSELF;
+
     hvm_asid_init(nasids);
 }
 

@@ -43,10 +47,17 @@
 {
     struct vcpu *curr = current;
     struct vmcb_struct *vmcb = curr->arch.hvm_svm.vmcb;
-    struct hvm_vcpu_asid *p_asid =
-        nestedhvm_vcpu_in_guestmode(curr)
-        ? &vcpu_nestedhvm(curr).nv_n2asid : &curr->arch.hvm_vcpu.n1asid;
-    bool_t need_flush = hvm_asid_handle_vmenter(p_asid);
+    struct hvm_vcpu_asid *p_asid;
+    hvm_asid_action_t asid_action;
+    bool_t guest_mode;
+
+    guest_mode = nestedhvm_enabled(curr->domain) && 
+        nestedhvm_vcpu_in_guestmode(curr);
+
+    p_asid = guest_mode ? 
+        &vcpu_nestedhvm(curr).nv_n2asid : &curr->arch.hvm_vcpu.n1asid;
+
+    asid_action = hvm_asid_handle_vmenter(p_asid);
 
     /* ASID 0 indicates that ASIDs are disabled. */
     if ( p_asid->asid == 0 )

@@ -57,7 +68,10 @@
     }
 
     vmcb_set_guest_asid(vmcb, p_asid->asid);
-    vmcb->tlb_control = need_flush;
+
+    /* NOTE: please make sure asid_action value returned from 
+     * hvm_asid_handle_vmenter() matches SVM's definition here. */
+    vmcb->tlb_control = asid_action;
 }
 
 /*

diff -r f531ed84b066 -r cf89d82ba40b xen/arch/x86/hvm/svm/svm.c
@@ -1044,6 +1044,9 @@
     if ( !test_bit(X86_FEATURE_SVM, &boot_cpu_data.x86_capability) )
         return NULL;
 
+    svm_feature_flags = ((cpuid_eax(0x80000000) >= 0x8000000A) ?
+                         cpuid_edx(0x8000000A) : 0);
+
     if ( svm_cpu_up() )
     {
         printk("SVM: failed to initialise.\n");

@@ -1052,9 +1055,6 @@
 
     setup_vmcb_dump();
 
-    svm_feature_flags = ((cpuid_eax(0x80000000) >= 0x8000000A) ?
-                         cpuid_edx(0x8000000A) : 0);
-
     printk("SVM: Supported advanced features:\n");
 
     /* DecodeAssists fast paths assume nextrip is valid for fast rIP update. */

diff -r f531ed84b066 -r cf89d82ba40b xen/arch/x86/hvm/vmx/vmcs.c
@@ -523,6 +523,7 @@
         BUG();
     }
 
+    hvm_asid_features = 0;
     hvm_asid_init(cpu_has_vmx_vpid ? (1u << VMCS_VPID_WIDTH) : 0);
 
     if ( cpu_has_vmx_ept )

diff -r f531ed84b066 -r cf89d82ba40b xen/arch/x86/hvm/vmx/vmx.c
@@ -2535,7 +2535,7 @@
 
     p_asid = &curr->arch.hvm_vcpu.n1asid;
     old_asid = p_asid->asid;
-    need_flush = hvm_asid_handle_vmenter(p_asid);
+    need_flush = !!hvm_asid_handle_vmenter(p_asid);
     new_asid = p_asid->asid;
 
     if ( unlikely(new_asid != old_asid) )

diff -r f531ed84b066 -r cf89d82ba40b xen/include/asm-x86/hvm/asid.h
@@ -25,6 +25,18 @@
 struct vcpu;
 struct hvm_vcpu_asid;
 
+/* NOTE: the values below have special meanings. Please check AMD ASID spec 
+ * before any modification. */
+typedef enum {
+    HVM_ASID_FLUSH_NONE   = 0x0, /* do not need to flush any TLB's */
+    HVM_ASID_FLUSH_ALL    = 0x1, /* flush TLBs with all ASIDs */
+    HVM_ASID_FLUSH_NEW    = 0x2, /* flush TLB by taking a new ASID */
+    HVM_ASID_FLUSH_ITSELF = 0x3, /* flush ASIDs own TLB only */
+} hvm_asid_action_t;
+
+extern uint32_t hvm_asid_features;
+#define HVM_ASID_FEATURE_FLUSH_ITSELF  0x1
+
 /* Initialise ASID management for the current physical CPU. */
 void hvm_asid_init(int nasids);
 

@@ -39,7 +51,7 @@
 
 /* Called before entry to guest context. Checks ASID allocation, returns a
  * boolean indicating whether all ASIDs must be flushed. */
-bool_t hvm_asid_handle_vmenter(struct hvm_vcpu_asid *asid);
+hvm_asid_action_t hvm_asid_handle_vmenter(struct hvm_vcpu_asid *asid);
 
 #endif /* __ASM_X86_HVM_ASID_H__ */
 

diff -r f531ed84b066 -r cf89d82ba40b xen/include/asm-x86/hvm/svm/svm.h
@@ -90,6 +90,7 @@
 #define cpu_has_svm_svml      cpu_has_svm_feature(SVM_FEATURE_SVML)
 #define cpu_has_svm_nrips     cpu_has_svm_feature(SVM_FEATURE_NRIPS)
 #define cpu_has_svm_cleanbits cpu_has_svm_feature(SVM_FEATURE_VMCBCLEAN)
+#define cpu_has_svm_flushasid cpu_has_svm_feature(SVM_FEATURE_FLUSHBYASID)
 #define cpu_has_svm_decode    cpu_has_svm_feature(SVM_FEATURE_DECODEASSISTS)
 #define cpu_has_pause_filter  cpu_has_svm_feature(SVM_FEATURE_PAUSEFILTER)
 

diff -r f531ed84b066 -r cf89d82ba40b xen/include/asm-x86/hvm/vcpu.h
@@ -38,6 +38,7 @@
 };
 
 struct hvm_vcpu_asid {
+    uint32_t core_id;
     uint64_t generation;
     uint32_t asid;
 };