x86/intel_rdt: Intel Cache Allocation Technology

Add support for cache bit mask manipulation. The change adds a file to the RDT cgroup which represents the CBM(cache bit mask) for the cgroup.

The RDT cgroup follows cgroup hierarchy ,mkdir and adding tasks to the cgroup never fails. When a child cgroup is created it inherits the CLOSid and the CBM from its parent. When a user changes the default CBM for a cgroup, a new CLOSid may be allocated if the CBM was not used before. If the new CBM is the one that is already used, the count for that CLOSid<->CBM is incremented. The changing of 'cbm' may fail with -ENOSPC once the kernel runs out of maximum CLOSids it can support.
User can create as many cgroups as he wants but having different CBMs at the same time is restricted by the maximum number of CLOSids (multiple cgroups can have the same CBM).
Kernel maintains a CLOSid<->cbm mapping which keeps count of cgroups using a CLOSid.

The tasks in the CAT cgroup would get to fill the LLC cache represented by the cgroup's 'cbm' file.

Reuse of CLOSids for cgroups with same bitmask also has following advantages:
- This helps to use the scant CLOSids optimally.
- This also implies that during context switch, write to PQR-MSR is done only when a task with a different bitmask is scheduled in.

Signed-off-by Vikas Shivappa <[email protected]>

---
 arch/x86/include/asm/intel_rdt.h |   3 +
 arch/x86/kernel/cpu/intel_rdt.c  | 205 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 208 insertions(+)

arch/x86/include/asm/intel_rdt.h +3 -0

index 87af1a5..0ed28d9 100644
@@ -4,6 +4,9 @@
 #ifdef CONFIG_CGROUP_RDT
 
 #include <linux/cgroup.h>
+#define MAX_CBM_LENGTH			32
+#define IA32_L3_CBM_BASE		0xc90
+#define CBM_FROM_INDEX(x)		(IA32_L3_CBM_BASE + x)
 
 struct rdt_subsys_info {
 	/* Clos Bitmap to keep track of available CLOSids.*/

arch/x86/kernel/cpu/intel_rdt.c +62 -0

index 3726f41..495497a 100644
@@ -33,6 +33,9 @@ static struct rdt_subsys_info rdtss_info;
 static DEFINE_MUTEX(rdt_group_mutex);
 struct intel_rdt rdt_root_group;
 
+#define rdt_for_each_child(pos_css, parent_ir)		\
+	css_for_each_child((pos_css), &(parent_ir)->css)
+
 static inline bool cat_supported(struct cpuinfo_x86 *c)
 {
 	if (cpu_has(c, X86_FEATURE_CAT_L3))

@@ -83,6 +86,31 @@ static int __init rdt_late_init(void)
 late_initcall(rdt_late_init);
 
 /*
+ * Allocates a new closid from unused closids.
+ * Called with the rdt_group_mutex held.
+ */
+
+static int rdt_alloc_closid(struct intel_rdt *ir)
+{
+	unsigned int id;
+	unsigned int maxid;
+
+	lockdep_assert_held(&rdt_group_mutex);
+
+	maxid = boot_cpu_data.x86_cat_closs;
+	id = find_next_zero_bit(rdtss_info.closmap, maxid, 0);
+	if (id == maxid)
+		return -ENOSPC;
+
+	set_bit(id, rdtss_info.closmap);
+	WARN_ON(ccmap[id].cgrp_count);
+	ccmap[id].cgrp_count++;
+	ir->clos = id;
+
+	return 0;
+}
+
+/*
 * Called with the rdt_group_mutex held.
 */
 static int rdt_free_closid(struct intel_rdt *ir)

@@ -133,8 +161,185 @@ static void rdt_css_free(struct cgroup_subsys_state *css)
 	mutex_unlock(&rdt_group_mutex);
 }
 
+/*
+ * Tests if atleast two contiguous bits are set.
+ */
+
+static inline bool cbm_is_contiguous(unsigned long var)
+{
+	unsigned long first_bit, zero_bit;
+	unsigned long maxcbm = MAX_CBM_LENGTH;
+
+	if (bitmap_weight(&var, maxcbm) < 2)
+		return false;
+
+	first_bit = find_next_bit(&var, maxcbm, 0);
+	zero_bit = find_next_zero_bit(&var, maxcbm, first_bit);
+
+	if (find_next_bit(&var, maxcbm, zero_bit) < maxcbm)
+		return false;
+
+	return true;
+}
+
+static int cat_cbm_read(struct seq_file *m, void *v)
+{
+	struct intel_rdt *ir = css_rdt(seq_css(m));
+
+	seq_printf(m, "%08lx\n", ccmap[ir->clos].cbm);
+	return 0;
+}
+
+static int validate_cbm(struct intel_rdt *ir, unsigned long cbmvalue)
+{
+	struct intel_rdt *par, *c;
+	struct cgroup_subsys_state *css;
+	unsigned long *cbm_tmp;

arch/x86/kernel/cpu/intel_rdt.c +48 -0

+	rcu_read_lock();
+	rdt_for_each_child(css, ir) {
+		c = css_rdt(css);
+		cbm_tmp = &ccmap[c->clos].cbm;
+		if (!bitmap_subset(cbm_tmp, &cbmvalue, MAX_CBM_LENGTH)) {
+			pr_info("Children's mask not a subset\n");
+			rcu_read_unlock();
+			return -EINVAL;
+		}
+	}
+
+	rcu_read_unlock();
+	return 0;
+}
+
+static bool cbm_search(unsigned long cbm, int *closid)
+{
+	int maxid = boot_cpu_data.x86_cat_closs;
+	unsigned int i;
+
+	for (i = 0; i < maxid; i++)
+		if (bitmap_equal(&cbm, &ccmap[i].cbm, MAX_CBM_LENGTH)) {
+			*closid = i;
+			return true;
+		}
+
+	return false;
+}
+
+static void cbmmap_dump(void)
+{
+	int i;
+
+	pr_debug("CBMMAP\n");
+	for (i = 0; i < boot_cpu_data.x86_cat_closs; i++)
+		pr_debug("cbm: 0x%x,cgrp_count: %u\n",
+		 (unsigned int)ccmap[i].cbm, ccmap[i].cgrp_count);
+}
+
+static void cpu_cbm_update(void *info)
+{
+	unsigned int closid = *((unsigned int *)info);
+
+	wrmsrl(CBM_FROM_INDEX(closid), ccmap[closid].cbm);
+}
+
+static inline void cbm_update(unsigned int closid)
+{

Quoted 134 lines · Vikas Shivappa

Add support for cache bit mask manipulation. The change adds a file to the RDT cgroup which represents the CBM(cache bit mask) for the cgroup.

The RDT cgroup follows cgroup hierarchy ,mkdir and adding tasks to the cgroup never fails. When a child cgroup is created it inherits the CLOSid and the CBM from its parent. When a user changes the default CBM for a cgroup, a new CLOSid may be allocated if the CBM was not used before. If the new CBM is the one that is already used, the count for that CLOSid<->CBM is incremented. The changing of 'cbm' may fail with -ENOSPC once the kernel runs out of maximum CLOSids it can support.
User can create as many cgroups as he wants but having different CBMs at the same time is restricted by the maximum number of CLOSids (multiple cgroups can have the same CBM).
Kernel maintains a CLOSid<->cbm mapping which keeps count of cgroups using a CLOSid.

The tasks in the CAT cgroup would get to fill the LLC cache represented by the cgroup's 'cbm' file.

Reuse of CLOSids for cgroups with same bitmask also has following advantages:
- This helps to use the scant CLOSids optimally.
- This also implies that during context switch, write to PQR-MSR is done only when a task with a different bitmask is scheduled in.

Signed-off-by Vikas Shivappa <[email protected]>

---
 arch/x86/include/asm/intel_rdt.h |   3 +
 arch/x86/kernel/cpu/intel_rdt.c  | 205 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 208 insertions(+)

arch/x86/include/asm/intel_rdt.h +3 -0

index 87af1a5..0ed28d9 100644
@@ -4,6 +4,9 @@
 #ifdef CONFIG_CGROUP_RDT
 
 #include <linux/cgroup.h>
+#define MAX_CBM_LENGTH			32
+#define IA32_L3_CBM_BASE		0xc90
+#define CBM_FROM_INDEX(x)		(IA32_L3_CBM_BASE + x)
 
 struct rdt_subsys_info {
 	/* Clos Bitmap to keep track of available CLOSids.*/

arch/x86/kernel/cpu/intel_rdt.c +66 -0

index 3726f41..495497a 100644
@@ -33,6 +33,9 @@ static struct rdt_subsys_info rdtss_info;
 static DEFINE_MUTEX(rdt_group_mutex);
 struct intel_rdt rdt_root_group;
 
+#define rdt_for_each_child(pos_css, parent_ir)		\
+	css_for_each_child((pos_css), &(parent_ir)->css)
+
 static inline bool cat_supported(struct cpuinfo_x86 *c)
 {
 	if (cpu_has(c, X86_FEATURE_CAT_L3))

@@ -83,6 +86,31 @@ static int __init rdt_late_init(void)
 late_initcall(rdt_late_init);
 
 /*
+ * Allocates a new closid from unused closids.
+ * Called with the rdt_group_mutex held.
+ */
+
+static int rdt_alloc_closid(struct intel_rdt *ir)
+{
+	unsigned int id;
+	unsigned int maxid;
+
+	lockdep_assert_held(&rdt_group_mutex);
+
+	maxid = boot_cpu_data.x86_cat_closs;
+	id = find_next_zero_bit(rdtss_info.closmap, maxid, 0);
+	if (id == maxid)
+		return -ENOSPC;
+
+	set_bit(id, rdtss_info.closmap);
+	WARN_ON(ccmap[id].cgrp_count);
+	ccmap[id].cgrp_count++;
+	ir->clos = id;
+
+	return 0;
+}
+
+/*
 * Called with the rdt_group_mutex held.
 */
 static int rdt_free_closid(struct intel_rdt *ir)

@@ -133,8 +161,185 @@ static void rdt_css_free(struct cgroup_subsys_state *css)
 	mutex_unlock(&rdt_group_mutex);
 }
 
+/*
+ * Tests if atleast two contiguous bits are set.
+ */
+
+static inline bool cbm_is_contiguous(unsigned long var)
+{
+	unsigned long first_bit, zero_bit;
+	unsigned long maxcbm = MAX_CBM_LENGTH;
+
+	if (bitmap_weight(&var, maxcbm) < 2)
+		return false;
+
+	first_bit = find_next_bit(&var, maxcbm, 0);
+	zero_bit = find_next_zero_bit(&var, maxcbm, first_bit);
+
+	if (find_next_bit(&var, maxcbm, zero_bit) < maxcbm)
+		return false;
+
+	return true;
+}
+
+static int cat_cbm_read(struct seq_file *m, void *v)
+{
+	struct intel_rdt *ir = css_rdt(seq_css(m));
+
+	seq_printf(m, "%08lx\n", ccmap[ir->clos].cbm);
+	return 0;
+}
+
+static int validate_cbm(struct intel_rdt *ir, unsigned long cbmvalue)
+{
+	struct intel_rdt *par, *c;
+	struct cgroup_subsys_state *css;
+	unsigned long *cbm_tmp;
+
+	if (!cbm_is_contiguous(cbmvalue)) {
+		pr_info("cbm should have >= 2 bits and be contiguous\n");
+		return -EINVAL;

arch/x86/kernel/cpu/intel_rdt.c static void rdt_css_free(struct cgroup_subsys_state *css) · line 207

Quoted 6 lines

arch/x86/kernel/cpu/intel_rdt.c +6 -0

+	}
+
+	par = parent_rdt(ir);
+	cbm_tmp = &ccmap[par->clos].cbm;
+	if (!bitmap_subset(&cbmvalue, cbm_tmp, MAX_CBM_LENGTH))
+		return -EINVAL;

Can you have different errors for the different cases?

Quoted 49 lines

arch/x86/kernel/cpu/intel_rdt.c +48 -0

+	rcu_read_lock();
+	rdt_for_each_child(css, ir) {
+		c = css_rdt(css);
+		cbm_tmp = &ccmap[c->clos].cbm;
+		if (!bitmap_subset(cbm_tmp, &cbmvalue, MAX_CBM_LENGTH)) {
+			pr_info("Children's mask not a subset\n");
+			rcu_read_unlock();
+			return -EINVAL;
+		}
+	}
+
+	rcu_read_unlock();
+	return 0;
+}
+
+static bool cbm_search(unsigned long cbm, int *closid)
+{
+	int maxid = boot_cpu_data.x86_cat_closs;
+	unsigned int i;
+
+	for (i = 0; i < maxid; i++)
+		if (bitmap_equal(&cbm, &ccmap[i].cbm, MAX_CBM_LENGTH)) {
+			*closid = i;
+			return true;
+		}
+
+	return false;
+}
+
+static void cbmmap_dump(void)
+{
+	int i;
+
+	pr_debug("CBMMAP\n");
+	for (i = 0; i < boot_cpu_data.x86_cat_closs; i++)
+		pr_debug("cbm: 0x%x,cgrp_count: %u\n",
+		 (unsigned int)ccmap[i].cbm, ccmap[i].cgrp_count);
+}
+
+static void cpu_cbm_update(void *info)
+{
+	unsigned int closid = *((unsigned int *)info);
+
+	wrmsrl(CBM_FROM_INDEX(closid), ccmap[closid].cbm);
+}
+
+static inline void cbm_update(unsigned int closid)
+{

arch/x86/kernel/cpu/intel_rdt.c static void rdt_css_free(struct cgroup_subsys_state *css) · line 268

Quoted 10 lines

arch/x86/kernel/cpu/intel_rdt.c +10 -0

+	int pkg_id = -1;
+	int cpu;
+
+	for_each_online_cpu(cpu) {
+		if (pkg_id == topology_physical_package_id(cpu))
+			continue;
+		smp_call_function_single(cpu, cpu_cbm_update, &closid, 1);
+		pkg_id = topology_physical_package_id(cpu);
+
+

Can use smp_call_function_many, once, more efficient.

Can this race with CPU hotplug? BTW, on CPU hotplug, where are the IA32_L3_MASK_n initialized for the new CPU ?

This patch adds a description of Cache allocation technology, overview of kernel implementation and usage of CAT cgroup interface.

Signed-off-by Vikas Shivappa <[email protected]>

---
 Documentation/cgroups/rdt.txt | 183 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 183 insertions(+)
 create mode 100644 Documentation/cgroups/rdt.txt

Documentation/cgroups/rdt.txt new file +99 -0

new file mode 100644
index 0000000..98eb4b8
@@ -0,0 +1,183 @@
+        RDT
+        ---
+
+Copyright (C) 2014 Intel Corporation
+Written by [email protected]
+(based on contents and format from cpusets.txt)
+
+CONTENTS:
+=========
+
+1. Cache Allocation Technology
+  1.1 What is RDT and CAT ?
+  1.2 Why is CAT needed ?
+  1.3 CAT implementation overview
+  1.4 Assignment of CBM and CLOS
+  1.5 Scheduling and Context Switch
+2. Usage Examples and Syntax
+
+1. Cache Allocation Technology(CAT)
+===================================
+
+1.1 What is RDT and CAT
+-----------------------
+
+CAT is a part of Resource Director Technology(RDT) or Platform Shared
+resource control which provides support to control Platform shared
+resources like cache. Currently Cache is the only resource that is
+supported in RDT.
+More information can be found in the Intel SDM section 17.15.
+
+Cache Allocation Technology provides a way for the Software (OS/VMM)
+to restrict cache allocation to a defined 'subset' of cache which may
+be overlapping with other 'subsets'.  This feature is used when
+allocating a line in cache ie when pulling new data into the cache.
+The programming of the h/w is done via programming  MSRs.
+
+The different cache subsets are identified by CLOS identifier (class
+of service) and each CLOS has a CBM (cache bit mask).  The CBM is a
+contiguous set of bits which defines the amount of cache resource that
+is available for each 'subset'.
+
+1.2 Why is CAT needed
+---------------------
+
+The CAT  enables more cache resources to be made available for higher
+priority applications based on guidance from the execution
+environment.
+
+The architecture also allows dynamically changing these subsets during
+runtime to further optimize the performance of the higher priority
+application with minimal degradation to the low priority app.
+Additionally, resources can be rebalanced for system throughput
+benefit.  (Refer to Section 17.15 in the Intel SDM)
+
+This technique may be useful in managing large computer systems which
+large LLC. Examples may be large servers running  instances of
+webservers or database servers. In such complex systems, these subsets
+can be used for more careful placing of the available cache
+resources.
+
+The CAT kernel patch would provide a basic kernel framework for users
+to be able to implement such cache subsets.
+
+1.3 CAT implementation Overview
+-------------------------------
+
+Kernel implements a cgroup subsystem to support cache allocation.
+
+Each cgroup has a CLOSid <-> CBM(cache bit mask) mapping.
+A CLOS(Class of service) is represented by a CLOSid.CLOSid is internal
+to the kernel and not exposed to user.  Each cgroup would have one CBM
+and would just represent one cache 'subset'.
+
+The cgroup follows cgroup hierarchy ,mkdir and adding tasks to the
+cgroup never fails.  When a child cgroup is created it inherits the
+CLOSid and the CBM from its parent.  When a user changes the default
+CBM for a cgroup, a new CLOSid may be allocated if the CBM was not
+used before.  The changing of 'cbm' may fail with -ERRNOSPC once the
+kernel runs out of maximum CLOSids it can support.
+User can create as many cgroups as he wants but having different CBMs
+at the same time is restricted by the maximum number of CLOSids
+(multiple cgroups can have the same CBM).
+Kernel maintains a CLOSid<->cbm mapping which keeps reference counter
+for each cgroup using a CLOSid.
+
+The tasks in the cgroup would get to fill the LLC cache represented by
+the cgroup's 'cbm' file.
+
+Root directory would have all available  bits set in 'cbm' file by
+default.
+
+1.4 Assignment of CBM,CLOS
+--------------------------
+
+The 'cbm' needs to be a  subset of the parent node's 'cbm'.
+Any contiguous subset of these bits(with a minimum of 2 bits) maybe
+set to indicate the cache mapping desired.  The 'cbm' between 2
+directories can overlap. The 'cbm' would represent the cache 'subset'
+of the CAT cgroup.  For ex: on a system with 16 bits of max cbm bits,

Documentation/cgroups/rdt.txt new file +74 -0

+
+1.5 Scheduling and Context Switch
+---------------------------------
+
+During context switch kernel implements this by writing the
+CLOSid (internally maintained by kernel) of the cgroup to which the
+task belongs to the CPU's IA32_PQR_ASSOC MSR. The MSR is only written
+when there is a change in the CLOSid for the CPU in order to minimize
+the latency incurred during context switch.
+
+2. Usage examples and syntax
+============================
+
+To check if CAT was enabled on your system
+
+dmesg | grep -i intel_rdt
+should output : intel_rdt: cbmlength:xx, Closs:xx
+the length of cbm and CLOS should depend on the system you use.
+
+
+Following would mount the cache allocation cgroup subsystem and create
+2 directories. Please refer to Documentation/cgroups/cgroups.txt on
+details about how to use cgroups.
+
+  cd /sys/fs/cgroup
+  mkdir rdt
+  mount -t cgroup -ordt rdt /sys/fs/cgroup/rdt
+  cd rdt
+
+Create 2 rdt cgroups
+
+  mkdir group1
+  mkdir group2
+
+Following are some of the Files in the directory
+
+  ls
+  rdt.cbm
+  tasks
+
+Say if the cache is 2MB and cbm supports 16 bits, then setting the
+below allocates the 'right 1/4th(512KB)' of the cache to group2
+
+Edit the CBM for group2 to set the least significant 4 bits.  This
+allocates 'right quarter' of the cache.
+
+  cd group2
+  /bin/echo 0xf > cat.cbm
+
+
+Edit the CBM for group2 to set the least significant 8 bits.This
+allocates the right half of the cache to 'group2'.
+
+  cd group2
+  /bin/echo 0xff > rdt.cbm
+
+Assign tasks to the group2
+
+  /bin/echo PID1 > tasks
+  /bin/echo PID2 > tasks
+
+  Meaning now threads
+  PID1 and PID2 get to fill the 'right half' of
+  the cache as the belong to cgroup group2.
+
+Create a group under group2
+
+  cd group2
+  mkdir group21
+  cat rdt.cbm
+   0xff - inherits parents mask.
+
+  /bin/echo 0xfff > rdt.cbm - throws error as mask has to parent's mask's subset
+

Signature · 7 lines

1.9.1

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Quoted 114 lines · Vikas Shivappa

This patch adds a description of Cache allocation technology, overview of kernel implementation and usage of CAT cgroup interface.

Signed-off-by Vikas Shivappa <[email protected]>

---
 Documentation/cgroups/rdt.txt | 183 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 183 insertions(+)
 create mode 100644 Documentation/cgroups/rdt.txt

Documentation/cgroups/rdt.txt new file +99 -0

new file mode 100644
index 0000000..98eb4b8
@@ -0,0 +1,183 @@
+        RDT
+        ---
+
+Copyright (C) 2014 Intel Corporation
+Written by [email protected]
+(based on contents and format from cpusets.txt)
+
+CONTENTS:
+=========
+
+1. Cache Allocation Technology
+  1.1 What is RDT and CAT ?
+  1.2 Why is CAT needed ?
+  1.3 CAT implementation overview
+  1.4 Assignment of CBM and CLOS
+  1.5 Scheduling and Context Switch
+2. Usage Examples and Syntax
+
+1. Cache Allocation Technology(CAT)
+===================================
+
+1.1 What is RDT and CAT
+-----------------------
+
+CAT is a part of Resource Director Technology(RDT) or Platform Shared
+resource control which provides support to control Platform shared
+resources like cache. Currently Cache is the only resource that is
+supported in RDT.
+More information can be found in the Intel SDM section 17.15.
+
+Cache Allocation Technology provides a way for the Software (OS/VMM)
+to restrict cache allocation to a defined 'subset' of cache which may
+be overlapping with other 'subsets'.  This feature is used when
+allocating a line in cache ie when pulling new data into the cache.
+The programming of the h/w is done via programming  MSRs.
+
+The different cache subsets are identified by CLOS identifier (class
+of service) and each CLOS has a CBM (cache bit mask).  The CBM is a
+contiguous set of bits which defines the amount of cache resource that
+is available for each 'subset'.
+
+1.2 Why is CAT needed
+---------------------
+
+The CAT  enables more cache resources to be made available for higher
+priority applications based on guidance from the execution
+environment.
+
+The architecture also allows dynamically changing these subsets during
+runtime to further optimize the performance of the higher priority
+application with minimal degradation to the low priority app.
+Additionally, resources can be rebalanced for system throughput
+benefit.  (Refer to Section 17.15 in the Intel SDM)
+
+This technique may be useful in managing large computer systems which
+large LLC. Examples may be large servers running  instances of
+webservers or database servers. In such complex systems, these subsets
+can be used for more careful placing of the available cache
+resources.
+
+The CAT kernel patch would provide a basic kernel framework for users
+to be able to implement such cache subsets.
+
+1.3 CAT implementation Overview
+-------------------------------
+
+Kernel implements a cgroup subsystem to support cache allocation.
+
+Each cgroup has a CLOSid <-> CBM(cache bit mask) mapping.
+A CLOS(Class of service) is represented by a CLOSid.CLOSid is internal
+to the kernel and not exposed to user.  Each cgroup would have one CBM
+and would just represent one cache 'subset'.
+
+The cgroup follows cgroup hierarchy ,mkdir and adding tasks to the
+cgroup never fails.  When a child cgroup is created it inherits the
+CLOSid and the CBM from its parent.  When a user changes the default
+CBM for a cgroup, a new CLOSid may be allocated if the CBM was not
+used before.  The changing of 'cbm' may fail with -ERRNOSPC once the
+kernel runs out of maximum CLOSids it can support.
+User can create as many cgroups as he wants but having different CBMs
+at the same time is restricted by the maximum number of CLOSids
+(multiple cgroups can have the same CBM).
+Kernel maintains a CLOSid<->cbm mapping which keeps reference counter
+for each cgroup using a CLOSid.
+
+The tasks in the cgroup would get to fill the LLC cache represented by
+the cgroup's 'cbm' file.
+
+Root directory would have all available  bits set in 'cbm' file by
+default.
+
+1.4 Assignment of CBM,CLOS
+--------------------------
+
+The 'cbm' needs to be a  subset of the parent node's 'cbm'.
+Any contiguous subset of these bits(with a minimum of 2 bits) maybe
+set to indicate the cache mapping desired.  The 'cbm' between 2
+directories can overlap. The 'cbm' would represent the cache 'subset'
+of the CAT cgroup.  For ex: on a system with 16 bits of max cbm bits,

Documentation/cgroups/rdt.txt line 109

Quoted 10 lines

Documentation/cgroups/rdt.txt new file +10 -0

+if the directory has the least significant 4 bits set in its 'cbm'
+file(meaning the 'cbm' is just 0xf), it would be allocated the right
+quarter of the Last level cache which means the tasks belonging to
+this CAT cgroup can use the right quarter of the cache to fill. If it
+has the most significant 8 bits set ,it would be allocated the left
+half of the cache(8 bits  out of 16 represents 50%).
+
+The cache portion defined in the CBM file is available to all tasks
+within the cgroup to fill and these task are not allowed to allocate
+space in other parts of the cache.

Is there a reason to expose the hardware interface rather than ratios to userspace ?

Say, i'd like to allocate 20% of L3 cache to cgroup A, 80% to cgroup B.

Well, you'd have to expose the shared percentages between any two cgroups (that information is there in the cbm bitmaps, but not in "ratios").

One problem i see with exposing cbm bitmasks is that on hardware updates that change cache size or bitmask length, userspace must recalculate the bitmaps.

Another is that its vendor dependant, while ratios (plus shared information for two given cgroups) is not.

Quoted 83 lines

Documentation/cgroups/rdt.txt new file +74 -1

+
+1.5 Scheduling and Context Switch
+---------------------------------
+
+During context switch kernel implements this by writing the
+CLOSid (internally maintained by kernel) of the cgroup to which the
+task belongs to the CPU's IA32_PQR_ASSOC MSR. The MSR is only written
+when there is a change in the CLOSid for the CPU in order to minimize
+the latency incurred during context switch.
+
+2. Usage examples and syntax
+============================
+
+To check if CAT was enabled on your system
+
+dmesg | grep -i intel_rdt
+should output : intel_rdt: cbmlength:xx, Closs:xx
+the length of cbm and CLOS should depend on the system you use.
+
+
+Following would mount the cache allocation cgroup subsystem and create
+2 directories. Please refer to Documentation/cgroups/cgroups.txt on
+details about how to use cgroups.
+
+  cd /sys/fs/cgroup
+  mkdir rdt
+  mount -t cgroup -ordt rdt /sys/fs/cgroup/rdt
+  cd rdt
+
+Create 2 rdt cgroups
+
+  mkdir group1
+  mkdir group2
+
+Following are some of the Files in the directory
+
+  ls
+  rdt.cbm
+  tasks
+
+Say if the cache is 2MB and cbm supports 16 bits, then setting the
+below allocates the 'right 1/4th(512KB)' of the cache to group2
+
+Edit the CBM for group2 to set the least significant 4 bits.  This
+allocates 'right quarter' of the cache.
+
+  cd group2
+  /bin/echo 0xf > cat.cbm
+
+
+Edit the CBM for group2 to set the least significant 8 bits.This
+allocates the right half of the cache to 'group2'.
+
+  cd group2
+  /bin/echo 0xff > rdt.cbm
+
+Assign tasks to the group2
+
+  /bin/echo PID1 > tasks
+  /bin/echo PID2 > tasks
+
+  Meaning now threads
+  PID1 and PID2 get to fill the 'right half' of
+  the cache as the belong to cgroup group2.
+
+Create a group under group2
+
+  cd group2
+  mkdir group21
+  cat rdt.cbm
+   0xff - inherits parents mask.
+
+  /bin/echo 0xfff > rdt.cbm - throws error as mask has to parent's mask's subset
+
--

1.9.1

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/

Hello Marcelo,

Quoted 137 lines · Vikas Shivappa

Quoted 114 lines

This patch adds a description of Cache allocation technology, overview of kernel implementation and usage of CAT cgroup interface.

Signed-off-by Vikas Shivappa <[email protected]>

---
Documentation/cgroups/rdt.txt | 183 ++++++++++++++++++++++++++++++++++++++++++

1 file changed, 183 insertions(+)

create mode 100644 Documentation/cgroups/rdt.txt

Documentation/cgroups/rdt.txt new file +99 -0

new file mode 100644
index 0000000..98eb4b8
@@ -0,0 +1,183 @@
+        RDT
+        ---
+
+Copyright (C) 2014 Intel Corporation
+Written by [email protected]
+(based on contents and format from cpusets.txt)
+
+CONTENTS:
+=========
+
+1. Cache Allocation Technology
+  1.1 What is RDT and CAT ?
+  1.2 Why is CAT needed ?
+  1.3 CAT implementation overview
+  1.4 Assignment of CBM and CLOS
+  1.5 Scheduling and Context Switch
+2. Usage Examples and Syntax
+
+1. Cache Allocation Technology(CAT)
+===================================
+
+1.1 What is RDT and CAT
+-----------------------
+
+CAT is a part of Resource Director Technology(RDT) or Platform Shared
+resource control which provides support to control Platform shared
+resources like cache. Currently Cache is the only resource that is
+supported in RDT.
+More information can be found in the Intel SDM section 17.15.
+
+Cache Allocation Technology provides a way for the Software (OS/VMM)
+to restrict cache allocation to a defined 'subset' of cache which may
+be overlapping with other 'subsets'.  This feature is used when
+allocating a line in cache ie when pulling new data into the cache.
+The programming of the h/w is done via programming  MSRs.
+
+The different cache subsets are identified by CLOS identifier (class
+of service) and each CLOS has a CBM (cache bit mask).  The CBM is a
+contiguous set of bits which defines the amount of cache resource that
+is available for each 'subset'.
+
+1.2 Why is CAT needed
+---------------------
+
+The CAT  enables more cache resources to be made available for higher
+priority applications based on guidance from the execution
+environment.
+
+The architecture also allows dynamically changing these subsets during
+runtime to further optimize the performance of the higher priority
+application with minimal degradation to the low priority app.
+Additionally, resources can be rebalanced for system throughput
+benefit.  (Refer to Section 17.15 in the Intel SDM)
+
+This technique may be useful in managing large computer systems which
+large LLC. Examples may be large servers running  instances of
+webservers or database servers. In such complex systems, these subsets
+can be used for more careful placing of the available cache
+resources.
+
+The CAT kernel patch would provide a basic kernel framework for users
+to be able to implement such cache subsets.
+
+1.3 CAT implementation Overview
+-------------------------------
+
+Kernel implements a cgroup subsystem to support cache allocation.
+
+Each cgroup has a CLOSid <-> CBM(cache bit mask) mapping.
+A CLOS(Class of service) is represented by a CLOSid.CLOSid is internal
+to the kernel and not exposed to user.  Each cgroup would have one CBM
+and would just represent one cache 'subset'.
+
+The cgroup follows cgroup hierarchy ,mkdir and adding tasks to the
+cgroup never fails.  When a child cgroup is created it inherits the
+CLOSid and the CBM from its parent.  When a user changes the default
+CBM for a cgroup, a new CLOSid may be allocated if the CBM was not
+used before.  The changing of 'cbm' may fail with -ERRNOSPC once the
+kernel runs out of maximum CLOSids it can support.
+User can create as many cgroups as he wants but having different CBMs
+at the same time is restricted by the maximum number of CLOSids
+(multiple cgroups can have the same CBM).
+Kernel maintains a CLOSid<->cbm mapping which keeps reference counter
+for each cgroup using a CLOSid.
+
+The tasks in the cgroup would get to fill the LLC cache represented by
+the cgroup's 'cbm' file.
+
+Root directory would have all available  bits set in 'cbm' file by
+default.
+
+1.4 Assignment of CBM,CLOS
+--------------------------
+
+The 'cbm' needs to be a  subset of the parent node's 'cbm'.
+Any contiguous subset of these bits(with a minimum of 2 bits) maybe
+set to indicate the cache mapping desired.  The 'cbm' between 2
+directories can overlap. The 'cbm' would represent the cache 'subset'
+of the CAT cgroup.  For ex: on a system with 16 bits of max cbm bits,

Documentation/cgroups/rdt.txt line 109

Quoted 10 lines

Documentation/cgroups/rdt.txt new file +10 -0

+if the directory has the least significant 4 bits set in its 'cbm'
+file(meaning the 'cbm' is just 0xf), it would be allocated the right
+quarter of the Last level cache which means the tasks belonging to
+this CAT cgroup can use the right quarter of the cache to fill. If it
+has the most significant 8 bits set ,it would be allocated the left
+half of the cache(8 bits  out of 16 represents 50%).
+
+The cache portion defined in the CBM file is available to all tasks
+within the cgroup to fill and these task are not allowed to allocate
+space in other parts of the cache.

Is there a reason to expose the hardware interface rather than ratios to userspace ?

Say, i'd like to allocate 20% of L3 cache to cgroup A, 80% to cgroup B.

Well, you'd have to expose the shared percentages between any two cgroups (that information is there in the cbm bitmaps, but not in "ratios").

Quoted 7 lines

One problem i see with exposing cbm bitmasks is that on hardware updates that change cache size or bitmask length, userspace must recalculate the bitmaps.

Another is that its vendor dependant, while ratios (plus shared information for two given cgroups) is not.

Agree that this interface doesnot give options to directly allocate in terms of percentage . But note that specifying in bitmasks allows the user to allocate overlapping cache areas and also since we use cgroup we naturally follow the cgroup hierarchy. User should be able to convert the bitmasks into intended percentage or size values based on the other available cache size info in hooks like cpuinfo.

We discussed more on this before in the older patches and here is one thread where we discussed it for your reference -
http://marc.info/?l=linux-kernel&m=142482002022543&w=2

Thanks,
Vikas

Quoted 2 lines · Vikas Shivappa

Hello Marcelo,

Hi Vikas,

Quoted 153 lines · Marcelo Tosatti

Quoted 137 lines · Vikas Shivappa

Quoted 114 lines

This patch adds a description of Cache allocation technology, overview of kernel implementation and usage of CAT cgroup interface.

Signed-off-by Vikas Shivappa <[email protected]>

---
Documentation/cgroups/rdt.txt | 183 ++++++++++++++++++++++++++++++++++++++++++

1 file changed, 183 insertions(+)

create mode 100644 Documentation/cgroups/rdt.txt

Documentation/cgroups/rdt.txt new file +99 -0

new file mode 100644
index 0000000..98eb4b8
@@ -0,0 +1,183 @@
+        RDT
+        ---
+
+Copyright (C) 2014 Intel Corporation
+Written by [email protected]
+(based on contents and format from cpusets.txt)
+
+CONTENTS:
+=========
+
+1. Cache Allocation Technology
+  1.1 What is RDT and CAT ?
+  1.2 Why is CAT needed ?
+  1.3 CAT implementation overview
+  1.4 Assignment of CBM and CLOS
+  1.5 Scheduling and Context Switch
+2. Usage Examples and Syntax
+
+1. Cache Allocation Technology(CAT)
+===================================
+
+1.1 What is RDT and CAT
+-----------------------
+
+CAT is a part of Resource Director Technology(RDT) or Platform Shared
+resource control which provides support to control Platform shared
+resources like cache. Currently Cache is the only resource that is
+supported in RDT.
+More information can be found in the Intel SDM section 17.15.
+
+Cache Allocation Technology provides a way for the Software (OS/VMM)
+to restrict cache allocation to a defined 'subset' of cache which may
+be overlapping with other 'subsets'.  This feature is used when
+allocating a line in cache ie when pulling new data into the cache.
+The programming of the h/w is done via programming  MSRs.
+
+The different cache subsets are identified by CLOS identifier (class
+of service) and each CLOS has a CBM (cache bit mask).  The CBM is a
+contiguous set of bits which defines the amount of cache resource that
+is available for each 'subset'.
+
+1.2 Why is CAT needed
+---------------------
+
+The CAT  enables more cache resources to be made available for higher
+priority applications based on guidance from the execution
+environment.
+
+The architecture also allows dynamically changing these subsets during
+runtime to further optimize the performance of the higher priority
+application with minimal degradation to the low priority app.
+Additionally, resources can be rebalanced for system throughput
+benefit.  (Refer to Section 17.15 in the Intel SDM)
+
+This technique may be useful in managing large computer systems which
+large LLC. Examples may be large servers running  instances of
+webservers or database servers. In such complex systems, these subsets
+can be used for more careful placing of the available cache
+resources.
+
+The CAT kernel patch would provide a basic kernel framework for users
+to be able to implement such cache subsets.
+
+1.3 CAT implementation Overview
+-------------------------------
+
+Kernel implements a cgroup subsystem to support cache allocation.
+
+Each cgroup has a CLOSid <-> CBM(cache bit mask) mapping.
+A CLOS(Class of service) is represented by a CLOSid.CLOSid is internal
+to the kernel and not exposed to user.  Each cgroup would have one CBM
+and would just represent one cache 'subset'.
+
+The cgroup follows cgroup hierarchy ,mkdir and adding tasks to the
+cgroup never fails.  When a child cgroup is created it inherits the
+CLOSid and the CBM from its parent.  When a user changes the default
+CBM for a cgroup, a new CLOSid may be allocated if the CBM was not
+used before.  The changing of 'cbm' may fail with -ERRNOSPC once the
+kernel runs out of maximum CLOSids it can support.
+User can create as many cgroups as he wants but having different CBMs
+at the same time is restricted by the maximum number of CLOSids
+(multiple cgroups can have the same CBM).
+Kernel maintains a CLOSid<->cbm mapping which keeps reference counter
+for each cgroup using a CLOSid.
+
+The tasks in the cgroup would get to fill the LLC cache represented by
+the cgroup's 'cbm' file.
+
+Root directory would have all available  bits set in 'cbm' file by
+default.
+
+1.4 Assignment of CBM,CLOS
+--------------------------
+
+The 'cbm' needs to be a  subset of the parent node's 'cbm'.
+Any contiguous subset of these bits(with a minimum of 2 bits) maybe
+set to indicate the cache mapping desired.  The 'cbm' between 2
+directories can overlap. The 'cbm' would represent the cache 'subset'
+of the CAT cgroup.  For ex: on a system with 16 bits of max cbm bits,

Documentation/cgroups/rdt.txt line 109

Quoted 10 lines

Documentation/cgroups/rdt.txt new file +10 -0

+if the directory has the least significant 4 bits set in its 'cbm'
+file(meaning the 'cbm' is just 0xf), it would be allocated the right
+quarter of the Last level cache which means the tasks belonging to
+this CAT cgroup can use the right quarter of the cache to fill. If it
+has the most significant 8 bits set ,it would be allocated the left
+half of the cache(8 bits  out of 16 represents 50%).
+
+The cache portion defined in the CBM file is available to all tasks
+within the cgroup to fill and these task are not allowed to allocate
+space in other parts of the cache.

Is there a reason to expose the hardware interface rather than ratios to userspace ?

Say, i'd like to allocate 20% of L3 cache to cgroup A, 80% to cgroup B.

Well, you'd have to expose the shared percentages between any two cgroups (that information is there in the cbm bitmaps, but not in "ratios").

Quoted 7 lines

One problem i see with exposing cbm bitmasks is that on hardware updates that change cache size or bitmask length, userspace must recalculate the bitmaps.

Another is that its vendor dependant, while ratios (plus shared information for two given cgroups) is not.

Agree that this interface doesnot give options to directly allocate in terms of percentage . But note that specifying in bitmasks allows the user to allocate overlapping cache areas and also since we use cgroup we naturally follow the cgroup hierarchy. User should be able to convert the bitmasks into intended percentage or size values based on the other available cache size info in hooks like cpuinfo.

Quoted 6 lines

We discussed more on this before in the older patches and here is one thread where we discussed it for your reference -
http://marc.info/?l=linux-kernel&m=142482002022543&w=2

Thanks,
Vikas

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?

Cpu.shares is written in ratio form, which is much more natural. Do you see any advantage in maintaining the

(ratio -> cbm bitmasks)

translation in userspace rather than in the kernel ?

What about something like:

		      root cgroup
		   /		  \
		  /		    \
		/		      \
	cgroupA-80			cgroupB-30

So that whatever exceeds 100% is the ratio of cache shared at that level (cgroup A and B share 10% of cache at that level).

https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu_and_memory-use_case.html

cpu — the cpu.shares parameter determines the share of CPU resources available to each process in all cgroups. Setting the parameter to 250, 250, and 500 in the finance, sales, and engineering cgroups respectively means that processes started in these groups will split the resources with a 1:1:2 ratio. Note that when a single process is running, it consumes as much CPU as necessary no matter which cgroup it is placed in. The CPU limitation only comes into effect when two or more processes compete for CPU resources.

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?

Cpu.shares is written in ratio form, which is much more natural. Do you see any advantage in maintaining the

(ratio -> cbm bitmasks)

translation in userspace rather than in the kernel ?

What about something like:

		      root cgroup
		   /		  \
		  /		    \
		/		      \
	cgroupA-80			cgroupB-30

So that whatever exceeds 100% is the ratio of cache shared at that level (cgroup A and B share 10% of cache at that level).

https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu_and_memory-use_case.html

cpu — the cpu.shares parameter determines the share of CPU resources available to each process in all cgroups. Setting the parameter to 250, 250, and 500 in the finance, sales, and engineering cgroups respectively means that processes started in these groups will split the resources with a 1:1:2 ratio. Note that when a single process is running, it consumes as much CPU as necessary no matter which cgroup it is placed in. The CPU limitation only comes into effect when two or more processes compete for CPU resources.

Quoted 21 lines · Marcelo Tosatti

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?

Cpu.shares is written in ratio form, which is much more natural. Do you see any advantage in maintaining the

(ratio -> cbm bitmasks)

translation in userspace rather than in the kernel ?

What about something like:

		      root cgroup
		   /		  \
		  /		    \
		/		      \
	cgroupA-80			cgroupB-30

Quoted 3 lines

So that whatever exceeds 100% is the ratio of cache shared at that level (cgroup A and B share 10% of cache at that level).

But this also means the 2 groups share all of the cache ?

Specifying the amount of bits to be shared lets you specify the exact cache area where you want to share and also when your total occupancy does not cover all of the cache. For ex: it gets more complex when you want to share say only the left quarter of the cache. cgroupA gets left half and cgroup gets left quarter. The bitmask aligns with how the h/w is designed to share the cache which gives you flexibility to define any specific overlapping areas of the cache.

Quoted 12 lines

https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu_and_memory-use_case.html

cpu — the cpu.shares parameter determines the share of CPU resources available to each process in all cgroups. Setting the parameter to 250, 250, and 500 in the finance, sales, and engineering cgroups respectively means that processes started in these groups will split the resources with a 1:1:2 ratio. Note that when a single process is running, it consumes as much CPU as necessary no matter which cgroup it is placed in. The CPU limitation only comes into effect when two or more processes compete for CPU resources.

These are more defined in terms of how many cache lines (or how many cache ways) they can use and would be difficult to define them in terms of percentage. In contrast the cpu share is a time shared thing and is much more granular where as here its not , its occupancy in terms of cache lines/ways.. (however this is not really defined as a restriction but thats the way it is now). Also note that the granularity of the bitmasks define the granularity of the percentages and in some SKUs the granularity is 2b and not 1b.. So technically you wont be able to even allocate percentage of cache even in 10% granularity for most of the cases (if there are 30MB and 25 ways like in one of hsw SKU) and this will vary for different SKUs which makes it more complicated for users. However the user library is free to define own interface based on the underlying cgroup interface say for example you never care about the overlapping and using it for a specific SKU etc.. The underlying cgroup framework is meant to be generic for all SKus and used for most of the use cases.

Also at this point I see a lot of enterprise and and other users already using the cgroup interface or shown interest in the same. However I see your point where you indicate the ease with which user can specify in size/percentage which he might be used to doing for other resources rather than bits where he needs to get an idea size by calculating it seperately - But again note that you may not be able to define percentages in many scenarios like the one above. And another question would be we would need to convince the users to adapt to the modified percentage user model (ex: like the one you say above where percentage - 100 is the one thats shared) I can review this requirements and others I have received and get back to see the closest that can be done if possible.

Thanks,
Vikas

Quoted 84 lines · Vikas Shivappa

Quoted 21 lines · Marcelo Tosatti

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?

Cpu.shares is written in ratio form, which is much more natural. Do you see any advantage in maintaining the

(ratio -> cbm bitmasks)

translation in userspace rather than in the kernel ?

What about something like:

		      root cgroup
		   /		  \
		  /		    \
		/		      \
	cgroupA-80			cgroupB-30

Quoted 3 lines

So that whatever exceeds 100% is the ratio of cache shared at that level (cgroup A and B share 10% of cache at that level).

But this also means the 2 groups share all of the cache ?

Specifying the amount of bits to be shared lets you specify the exact cache area where you want to share and also when your total occupancy does not cover all of the cache. For ex: it gets more complex when you want to share say only the left quarter of the cache. cgroupA gets left half and cgroup gets left quarter. The bitmask aligns with how the h/w is designed to share the cache which gives you flexibility to define any specific overlapping areas of the cache.

Quoted 12 lines

https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu_and_memory-use_case.html

cpu — the cpu.shares parameter determines the share of CPU resources available to each process in all cgroups. Setting the parameter to 250, 250, and 500 in the finance, sales, and engineering cgroups respectively means that processes started in these groups will split the resources with a 1:1:2 ratio. Note that when a single process is running, it consumes as much CPU as necessary no matter which cgroup it is placed in. The CPU limitation only comes into effect when two or more processes compete for CPU resources.

These are more defined in terms of how many cache lines (or how many cache ways) they can use and would be difficult to define them in terms of percentage. In contrast the cpu share is a time shared thing and is much more granular where as here its not , its occupancy in terms of cache lines/ways.. (however this is not really defined as a restriction but thats the way it is now). Also note that the granularity of the bitmasks define the granularity of the percentages and in some SKUs the granularity is 2b and not 1b.. So technically you wont be able to even allocate percentage of cache even in 10% granularity for most of the cases (if there are 30MB and 25 ways like in one of hsw SKU) and this will vary for different SKUs which makes it more complicated for users. However the user library is free to define own interface based on the underlying cgroup interface say for example you never care about the overlapping and using it for a specific SKU etc.. The underlying cgroup framework is meant to be generic for all SKus and used for most of the use cases.

Also at this point I see a lot of enterprise and and other users already using the cgroup interface or shown interest in the same. However I see your point where you indicate the ease with which user can specify in size/percentage which he might be used to doing for other resources rather than bits where he needs to get an idea size by calculating it seperately - But again note that you may not be able to define percentages in many scenarios like the one above. And another question would be we would need to convince the users to adapt to the modified percentage user model (ex: like the one you say above where percentage - 100 is the one thats shared) I can review this requirements and others I have received and get back to see the closest that can be done if possible.

Quoted 2 lines

Thanks,
Vikas

Vikas,

I see. Don't have anything against performing the translation in userspace (i agree userspace should be able to allow ratios and specific minimum/maximum counts). Can you please export the relevant information in files in /sys or cgroups itself rather than requiring userspace to parse CPUID etc? Including the EBX register from CPUID(EAX=10H, ECX=1), which is necessary to implement "reserved LLC" properly.

The current interface is unable to handle the cross CPU case, though. It would be necessary to expose per-socket masks.

Quoted 21 lines · Marcelo Tosatti

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?

Cpu.shares is written in ratio form, which is much more natural. Do you see any advantage in maintaining the

(ratio -> cbm bitmasks)

translation in userspace rather than in the kernel ?

What about something like:

		      root cgroup
		   /		  \
		  /		    \
		/		      \
	cgroupA-80			cgroupB-30

Quoted 3 lines

So that whatever exceeds 100% is the ratio of cache shared at that level (cgroup A and B share 10% of cache at that level).

But this also means the 2 groups share all of the cache ?

Specifying the amount of bits to be shared lets you specify the exact cache area where you want to share and also when your total occupancy does not cover all of the cache. For ex: it gets more complex when you want to share say only the left quarter of the cache. cgroupA gets left half and cgroup gets left quarter. The bitmask aligns with how the h/w is designed to share the cache which gives you flexibility to define any specific overlapping areas of the cache.

Quoted 13 lines

https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu_and_memory-use_case.html

cpu — the cpu.shares parameter determines the share of CPU resources available to each process in all cgroups. Setting the parameter to 250, 250, and 500 in the finance, sales, and engineering cgroups respectively means that processes started in these groups will split the resources with a 1:1:2 ratio. Note that when a single process is running, it consumes as much CPU as necessary no matter which cgroup it is placed in. The CPU limitation only comes into effect when two or more processes compete for CPU resources.

These are more defined in terms of how many cache lines (or how many cache ways) they can use and would be difficult to define them in terms of percentage. In contrast the cpu share is a time shared thing and is much more granular where as here its not , its occupancy in terms of cache lines/ways.. (however this is not really defined as a restriction but thats the way it is now). Also note that the granularity of the bitmasks define the granularity of the percentages and in some SKUs the granularity is 2b and not 1b.. So technically you wont be able to even allocate percentage of cache even in 10% granularity for most of the cases (if there are 30MB and 25 ways like in one of hsw SKU) and this will vary for different SKUs which makes it more complicated for users. However the user library is free to define own interface based on the underlying cgroup interface say for example you never care about the overlapping and using it for a specific SKU etc.. The underlying cgroup framework is meant to be generic for all SKus and used for most of the use cases.

Also at this point I see a lot of enterprise and and other users already using the cgroup interface or shown interest in the same. However I see your point where you indicate the ease with which user can specify in size/percentage which he might be used to doing for other resources rather than bits where he needs to get an idea size by calculating it seperately - But again note that you may not be able to define percentages in many scenarios like the one above. And another question would be we would need to convince the users to adapt to the modified percentage user model (ex: like the one you say above where percentage - 100 is the one thats shared) I can review this requirements and others I have received and get back to see the closest that can be done if possible.

Thanks,
Vikas

Quoted 85 lines · Vikas Shivappa

Quoted 21 lines · Marcelo Tosatti

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?

Cpu.shares is written in ratio form, which is much more natural. Do you see any advantage in maintaining the

(ratio -> cbm bitmasks)

translation in userspace rather than in the kernel ?

What about something like:

		      root cgroup
		   /		  \
		  /		    \
		/		      \
	cgroupA-80			cgroupB-30

Quoted 3 lines

So that whatever exceeds 100% is the ratio of cache shared at that level (cgroup A and B share 10% of cache at that level).

But this also means the 2 groups share all of the cache ?

Specifying the amount of bits to be shared lets you specify the exact cache area where you want to share and also when your total occupancy does not cover all of the cache. For ex: it gets more complex when you want to share say only the left quarter of the cache. cgroupA gets left half and cgroup gets left quarter. The bitmask aligns with how the h/w is designed to share the cache which gives you flexibility to define any specific overlapping areas of the cache.

Quoted 13 lines

https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/6/html/Resource_Management_Guide/sec-cpu_and_memory-use_case.html

cpu — the cpu.shares parameter determines the share of CPU resources available to each process in all cgroups. Setting the parameter to 250, 250, and 500 in the finance, sales, and engineering cgroups respectively means that processes started in these groups will split the resources with a 1:1:2 ratio. Note that when a single process is running, it consumes as much CPU as necessary no matter which cgroup it is placed in. The CPU limitation only comes into effect when two or more processes compete for CPU resources.

These are more defined in terms of how many cache lines (or how many cache ways) they can use and would be difficult to define them in terms of percentage. In contrast the cpu share is a time shared thing and is much more granular where as here its not , its occupancy in terms of cache lines/ways.. (however this is not really defined as a restriction but thats the way it is now). Also note that the granularity of the bitmasks define the granularity of the percentages and in some SKUs the granularity is 2b and not 1b.. So technically you wont be able to even allocate percentage of cache even in 10% granularity for most of the cases (if there are 30MB and 25 ways like in one of hsw SKU) and this will vary for different SKUs which makes it more complicated for users. However the user library is free to define own interface based on the underlying cgroup interface say for example you never care about the overlapping and using it for a specific SKU etc.. The underlying cgroup framework is meant to be generic for all SKus and used for most of the use cases.

Also at this point I see a lot of enterprise and and other users already using the cgroup interface or shown interest in the same. However I see your point where you indicate the ease with which user can specify in size/percentage which he might be used to doing for other resources rather than bits where he needs to get an idea size by calculating it seperately - But again note that you may not be able to define percentages in many scenarios like the one above. And another question would be we would need to convince the users to adapt to the modified percentage user model (ex: like the one you say above where percentage - 100 is the one thats shared) I can review this requirements and others I have received and get back to see the closest that can be done if possible.

Quoted 2 lines

Thanks,
Vikas

Vikas,

Three questions:

First, usage model. The usage model for CAT is the following (please correct me if i'm wrong):

1) measure application performance without L3 cache reservation.
2) measure application perf with L3 cache reservation and X number of ways until desired perf is attained.

On migration to a new hardware platform, to achieve similar benefit achieved when going from 1) to 2) is to reserve _at least_ the number of bytes that "X ways" provided when the measurement was performed. Is that correct?

If that is correct, then the user does want to record "number of bytes" that X ways on measurement CPU provided.

Second question:
Do you envision any use case which the placement of cache and not the quantity of cache is a criteria for decision? That is, two cases with the same amount of cache for each CLOSid, but with different locations inside the cache?
(except sharing of ways by two CLOSid's, of course).

Third question:
How about support for the (new) I/D cache division?

I can't find any discussion relating to exposing the CBM interface directly to userspace in that thread ?