Intel Code and Data Prioritization (CDP) feature enabling

docs/misc/xen-command-line.markdown +3 -0

@@ -1197,6 +1197,10 @@ The following resources are available:
   the cache allocation.
   * `cat` instructs Xen to enable/disable Cache Allocation Technology.
   * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+  regarding the code cache and the data cache allocation. CDP is based on CAT.
+  * `cdp` instructs Xen to enable/disable Code and Data Prioritization.

xen/arch/x86/psr.c +0 -2

         set_bit(socket, cat_socket_enable);
-        printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, cbm_len:%u\n",
-               socket, info->cos_max, info->cbm_len);

xen/arch/x86/psr.c +13 -0

+
+        if ( (ecx & PSR_CAT_CDP_CAPABILITY) && (opt_psr & PSR_CDP) )
+        {
+            if ( test_bit(socket, cdp_socket_enable) )
+                return;
+
+            rdmsrl(MSR_IA32_PSR_L3_QOS_CFG, val);
+            wrmsrl(MSR_IA32_PSR_L3_QOS_CFG, val | 1 << PSR_L3_QOS_CDP_ENABLE_BIT);
+
+            info->cos_to_cbm[0].u.cdp.code = (1ull << info->cbm_len) - 1;
+            info->cos_to_cbm[0].u.cdp.data = (1ull << info->cbm_len) - 1;
+
+            /* We only write mask1 since mask0 is always all ones by default */

xen/arch/x86/psr.c +3 -0

+            wrmsrl(MSR_IA32_PSR_L3_MASK(1), (1ull << info->cbm_len) - 1);
+
+            /* Cut half of cos_max when CDP enabled */

xen/arch/x86/psr.c +2 -0

@@ -508,6 +557,8 @@ static void cat_cpu_fini(unsigned int cpu)
     {
         struct psr_cat_socket_info *info = cat_socket_info + socket;
 
+        clear_bit(socket, cdp_socket_enable);
+

domctl cmds are introdunced to support set/get CDP CBM. Some CAT functions to operation CBMs are extended to support CDP.

Signed-off-by He Chen <[email protected]>

xen/arch/x86/psr.c +6 -0

+
+    /* If old cos is referred only by the domain, then use it. */
+    if ( map[old_cos].ref == 1 )
+        return old_cos;
+
+    /* Find an unused one other than cos0 */

xen/arch/x86/psr.c +2 -0

@@ -381,53 +449,79 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
     if ( !psr_check_cbm(info->cbm_len, cbm) )
         return -EINVAL;
 
+    if ( !cdp_enabled && (type == PSR_CBM_TYPE_L3_CODE ||
+        type == PSR_CBM_TYPE_L3_DATA) )

xen/arch/x86/psr.c +4 -2

+        return -EINVAL;
+
+    cos_max = info->cos_max;
     old_cos = d->arch.psr_cos_ids[socket];
     map = info->cos_to_cbm;
 
-    spin_lock(&info->cbm_lock);
 
-    for ( cos = 0; cos <= info->cos_max; cos++ )
+    switch( type )

xen/arch/x86/psr.c +9 -0

+    {
+        cos = pick_avail_cos(map, cos_max, old_cos);
+        if ( cos < 0 )
+        {
+            spin_unlock(&info->cbm_lock);
+            return cos;
+        }
+
+        /* We try to avoid writing MSR */

tools/libxl/libxl_psr.c +8 -0

     }
 
+    rc = libxl_psr_cat_get_l3_info(ctx, &info, &nr_sockets);
+    if (rc) {
+        LOGE(ERROR, "Failed to get cat info");
+        goto out;
+    }
+
+    if (!info->cdp_enabled &&
+       (type == LIBXL_PSR_CBM_TYPE_L3_CODE ||

docs/man/xl.pod.1 +2 -0

@@ -1530,6 +1530,12 @@ applications. In the Xen implementation, CAT is used to control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocation, which support specify code or data cache for

docs/man/xl.pod.1 +3 -0

@@ -1545,6 +1551,14 @@ B<OPTIONS>
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM with CDP enabled.

docs/man/xl.pod.1 +4 -0

+
+=item B<-d>, B<--data>
+
+Set data CBM with CDP enabled.

docs/misc/xl-psr.markdown +10 -0

+Code/Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cache capacity to the characteristics of the workload. CDP extends
+Cache Allocation Technology (CAT) by providing separate code and data masks
+per Class of Service (COS).
+
+CDP is disabled on the processor by default. If the CAT MSRs are used without
+enabling CDP, the processor operates in a traditional CAT-only mode.

docs/misc/xl-psr.markdown +13 -0

+
+When CDP is enabled,
+
+ * the CAT mask MSRs are re-mapped into interleaved pairs of mask MSRs for
+   data or code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows OS or Hypervisor to partition cache allocation more fine-grained,
+code cache and data cache can be specified respectively. With CDP enabled,
+one COS corresponds to two CBMs (code CBM & data CBM), which means the number
+of available COS will reduce to half when CDP is on.

docs/misc/xl-psr.markdown +7 -0

@@ -119,6 +144,13 @@ A cbm is valid only when:
 In a multi-socket system, the same cbm will be set on each socket by default.
 Per socket cbm can be specified with the `--socket SOCKET` option.
 
+To specify code CBM for the domain, `-c` or `--code` option is needed.
+
+To specify data CBM for the domain, `-d` or `--data` option is needed.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same code CBM
+and data CBM will be set for the domain.
+

xen/arch/x86/psr.c +7 -1

 
 struct psr_cat_cbm {
-    uint64_t cbm;
+    union {
+        uint64_t cbm;
+        struct {
+            uint64_t code;
+            uint64_t data;
+        } cdp;
+    } u;

xen/arch/x86/psr.c +1 -3

@@ -490,13 +519,33 @@ static void cat_cpu_init(void)
         info->cos_to_cbm = temp_cos_to_cbm;
         temp_cos_to_cbm = NULL;
         /* cos=0 is reserved as default cbm(all ones). */
-        info->cos_to_cbm[0].cbm = (1ull << info->cbm_len) - 1;
+        info->cos_to_cbm[0].u.cbm = (1ull << info->cbm_len) - 1;
 
         spin_lock_init(&info->cbm_lock);
 
         set_bit(socket, cat_socket_enable);
-        printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, cbm_len:%u\n",
-               socket, info->cos_max, info->cbm_len);

xen/arch/x86/psr.c +4 -0

+        if ( (ecx & PSR_CAT_CDP_CAPABILITY) && (opt_psr & PSR_CDP) )
+        {
+            if ( test_bit(socket, cdp_socket_enable) )
+                return;

xen/arch/x86/psr.c +2 -0

+            rdmsrl(MSR_IA32_PSR_L3_QOS_CFG, val);
+            wrmsrl(MSR_IA32_PSR_L3_QOS_CFG, val | 1 << PSR_L3_QOS_CDP_ENABLE_BIT);

xen/arch/x86/psr.c +8 -0

+            info->cos_to_cbm[0].u.cdp.code = (1ull << info->cbm_len) - 1;
+            info->cos_to_cbm[0].u.cdp.data = (1ull << info->cbm_len) - 1;
+
+            /* We only write mask1 since mask0 is always all ones by default */
+            wrmsrl(MSR_IA32_PSR_L3_MASK(1), (1ull << info->cbm_len) - 1);
+
+            /* Cut half of cos_max when CDP enabled */
+            info->cos_max = info->cos_max / 2;

xen/arch/x86/psr.c +2 -1

@@ -535,8 +588,9 @@ static void __init init_psr_cat(void)
 
     cat_socket_enable = xzalloc_array(unsigned long, BITS_TO_LONGS(nr_sockets));
     cat_socket_info = xzalloc_array(struct psr_cat_socket_info, nr_sockets);
+    cdp_socket_enable = xzalloc_array(unsigned long, BITS_TO_LONGS(nr_sockets));
 
-    if ( !cat_socket_enable || !cat_socket_info )
+    if ( !cat_socket_enable || !cat_socket_info || !cdp_socket_enable )
         psr_cat_free();

xen/include/asm-x86/msr-index.h +3 -0

@@ -328,7 +328,10 @@
 #define MSR_IA32_CMT_EVTSEL		0x00000c8d
 #define MSR_IA32_CMT_CTR		0x00000c8e
 #define MSR_IA32_PSR_ASSOC		0x00000c8f
+#define MSR_IA32_PSR_L3_QOS_CFG	0x00000c81
 #define MSR_IA32_PSR_L3_MASK(n)	(0x00000c90 + (n))
+#define MSR_IA32_PSR_L3_MASK_CODE(n)	(0x00000c90 + (n * 2 + 1))
+#define MSR_IA32_PSR_L3_MASK_DATA(n)	(0x00000c90 + (n * 2))

xen/include/public/sysctl.h +2 -0

@@ -704,6 +704,8 @@ struct xen_sysctl_psr_cat_op {
         struct {
             uint32_t cbm_len;   /* OUT: CBM length */
             uint32_t cos_max;   /* OUT: Maximum COS */
+            #define XEN_SYSCTL_PSR_CAT_L3_CDP       (1u << 0)
+            uint32_t flags;     /* OUT: CAT flags */
         } l3_info;

xen/arch/x86/psr.c +6 -1

     if ( IS_ERR(info) )
         return PTR_ERR(info);
 
-    *cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+    if ( type == PSR_CBM_TYPE_L3 && cdp_enabled )
+        return -EXDEV;
+
+    if ( (type == PSR_CBM_TYPE_L3_CODE || type == PSR_CBM_TYPE_L3_DATA)
+        && !cdp_enabled )
+        return -EXDEV;

xen/arch/x86/psr.c +10 -0

+        case PSR_CBM_TYPE_L3_CODE:
+            *cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.code;
+            break;
+
+        case PSR_CBM_TYPE_L3_DATA:
+            *cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.data;
+            break;
+
+        default:
+            return -EINVAL;

xen/arch/x86/psr.c +10 -0

+            if ( map[cos].ref && map[cos].u.cbm == cbm_code )
+                return cos;
+    }
+    else
+    {
+        for ( cos = 0; cos <= cos_max; cos++ )
+            if ( map[cos].ref && map[cos].u.cdp.code == cbm_code &&
+                 map[cos].u.cdp.data == cbm_data )
+                return cos;
+    }

xen/arch/x86/psr.c +3 -0

+static int pick_avail_cos(struct psr_cat_cbm *map, int cos_max, int old_cos)
+{
+    int cos;

xen/arch/x86/psr.c +5 -0

+int psr_set_l3_cbm(struct domain *d, unsigned int socket,
+                   uint64_t cbm, enum cbm_type type)
+{
+    unsigned int old_cos, cos_max;
+    int cos, ret;

xen/arch/x86/psr.c +3 -7

-            if ( unlikely(cos == old_cos) )
-            {
-                ASSERT(cos == 0 || map[cos].ref != 0);
-                spin_unlock(&info->cbm_lock);
-                return 0;
-            }
-            found = map + cos;
+        case PSR_CBM_TYPE_L3:
+            cbm_code = cbm;
+            cbm_data = cbm;

xen/arch/x86/psr.c +4 -5

-    cos = found - map;
-    if ( found->u.cbm != cbm )
+    spin_lock(&info->cbm_lock);
+    cos = find_cos(map, cos_max, cbm_code, cbm_data, cdp_enabled);
+    if ( cos >= 0 )
     {
-        int ret = write_l3_cbm(socket, cos, cbm, 0, 0);
-
-        if ( ret )
+        if ( unlikely(cos == old_cos) )

xen/arch/x86/psr.c +10 -0

+
+        /* We try to avoid writing MSR */
+        if ( cdp_enabled )
+        {
+            if ( map[cos].u.cdp.code == cbm_code &&
+                 map[cos].u.cdp.data == cbm_data )
+                need_write = 0;
+        }
+        else
+            need_write = map[cos].u.cbm == cbm_code ? 0 : 1;

xen/include/asm-x86/psr.h +4 -0

@@ -49,6 +49,12 @@ struct psr_cmt {
     struct psr_cmt_l3 l3;
 };
 
+enum cbm_type {
+    PSR_CBM_TYPE_L3         = 1,
+    PSR_CBM_TYPE_L3_CODE    = 2,
+    PSR_CBM_TYPE_L3_DATA    = 3,