x86/intel_rdt: Restoration of Cache Pseudo-Locked regions

index 4e588f36228f..7afbf1a746b7 100644
@@ -536,6 +536,8 @@ int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp);
 void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp);
 struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
 int update_domains(struct rdt_resource *r, int closid);
+bool closid_allocated(unsigned int closid);
+int closid_alloc(void);
 void closid_free(int closid);
 int alloc_rmid(void);
 void free_rmid(u32 rmid);

index d6d7ea7349d0..73f7ff26e704 100644
@@ -113,7 +113,7 @@ static void closid_init(void)
 	closid_free_map &= ~1;
 }
 
-static int closid_alloc(void)
+int closid_alloc(void)
 {
 	u32 closid = ffs(closid_free_map);
 

@@ -137,7 +137,7 @@ void closid_free(int closid)
  * Return: true if @closid is currently associated with a resource group,
  * false if @closid is free
  */
-static bool closid_allocated(unsigned int closid)
+bool closid_allocated(unsigned int closid)
 {
 	return (closid_free_map & (1 << closid)) == 0;
 }

index f80c58f8adc3..693f99594e9f 100644
@@ -453,6 +453,16 @@ static int pseudo_lock_fn(void *_rdtgrp)
 #endif /* CONFIG_X86_64 */
 #endif /* CONFIG_KASAN */
 
+	/*
+	 * In the case of a pseudo-locked region restore, if the thread
+	 * does not get on the CPU and the process attempting
+	 * a restore is interrupted then the thread remains in runnable
+	 * state but the closid would have been freed and nothing is
+	 * waiting on this restoration anymore.
+	 */
+	if (!closid_allocated(rdtgrp->closid))
+		return 0;
+
 	/*
 	 * Make sure none of the allocated memory is cached. If it is we
 	 * will get a cache hit in below loop from outside of pseudo-locked

@@ -1191,6 +1201,107 @@ static const struct file_operations pseudo_measure_fops = {
 	.llseek = default_llseek,
 };
 
+/**
+ * _pseudo_lock_restore - Pseudo-Lock possibly evicted memory again
+ * @rdtgrp: Resource group to which pseudo-locked region belongs
+ *
+ * Pseudo-locked memory could be evicted from the cache with an instruction
+ * like WBINVD or when the cache is affected by platform power savings. In
+ * support of recovering from these situations the pseudo-locked region can
+ * be restored by pseudo-locking the memory again.
+ *
+ * Return: 0 if region was successfully restored, <0 on failure.
+ */
+static int _pseudo_lock_restore(struct rdtgroup *rdtgrp)
+{
+	struct pseudo_lock_region *plr = rdtgrp->plr;
+	struct task_struct *thread;
+	struct rdt_resource *r;
+	struct rdt_domain *dom;
+	int closid;
+	int ret;
+
+	plr->cpu = cpumask_first(&plr->d->cpu_mask);
+
+	if (!cpu_online(plr->cpu)) {
+		pr_err("Failed to restore region %s. Associated CPU offline.\n",
+		       rdtgrp->kn->name);
+		ret = -ENODEV;
+		goto out;
+	}
+
+	closid = closid_alloc();
+	if (closid < 0) {
+		pr_err("Failed to restore region %s. No free closid.\n",
+		       rdtgrp->kn->name);
+		ret = -ENOSPC;
+		goto out;
+	}
+
+	rdtgrp->closid = closid;
+
+	/*
+	 * Associate the newly allocated closid with the CBM of the
+	 * Cache Pseudo-Locked region.
+	 */
+	for_each_alloc_enabled_rdt_resource(r) {
+		list_for_each_entry(dom, &r->domains, list)
+			dom->have_new_ctrl = false;
+	}
+
+	plr->d->new_ctrl = plr->cbm;
+	plr->d->have_new_ctrl = true;
+
+	ret = update_domains(plr->r, rdtgrp->closid);
+	if (ret < 0) {
+		pr_err("Unable to program CBM of region %s\n",
+		       rdtgrp->kn->name);
+		goto out_closid;
+	}
+
+	plr->thread_done = 0;
+
+	thread = kthread_create_on_node(pseudo_lock_fn, rdtgrp,
+					cpu_to_node(plr->cpu),
+					"pseudo_lock/%u", plr->cpu);
+	if (IS_ERR(thread)) {
+		ret = PTR_ERR(thread);
+		pr_err("Locking thread error %d during restore of %s\n",
+		       ret, rdtgrp->kn->name);
+		goto out_closid;
+	}
+
+	kthread_bind(thread, plr->cpu);
+	wake_up_process(thread);
+
+	ret = wait_event_interruptible(plr->lock_thread_wq,
+				       plr->thread_done == 1);
+	if (ret < 0) {
+		pr_err("Locking thread interrupted during restore of %s\n",
+		       rdtgrp->kn->name);
+		goto out_closid;
+	}
+
+	ret = 0;
+
+out_closid:
+	closid_free(rdtgrp->closid);
+out:
+	return ret;
+}
+
+static int __attribute__ ((unused)) pseudo_lock_restore(struct rdtgroup *rdtgrp)
+{
+	int ret;
+
+	cpus_read_lock();
+	mutex_lock(&rdtgroup_mutex);
+	ret = _pseudo_lock_restore(rdtgrp);
+	mutex_unlock(&rdtgroup_mutex);
+	cpus_read_unlock();
+	return ret;
+}
+
 /**
  * rdtgroup_pseudo_lock_create - Create a pseudo-locked region
  * @rdtgrp: resource group to which pseudo-lock region belongs

66 lines not on this page · raw message

index 246cff17ce66..d395e6982467 100644
@@ -52,6 +52,12 @@ static unsigned int pseudo_lock_major;
 static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0);
 static struct class *pseudo_lock_class;
 
+/*
+ * Support the offloading of pseudo-locked region restoration.
+ */
+static void pseudo_lock_restore_work_fn(struct work_struct *work);
+static DECLARE_WORK(pseudo_lock_restore_work, pseudo_lock_restore_work_fn);
+
 /**
  * get_prefetch_disable_bits - prefetch disable bits of supported platforms
  *

@@ -1510,6 +1516,42 @@ void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp)
 	pseudo_lock_free(rdtgrp);
 }
 
+/**
+ * pseudo_lock_restore_work_fn - Restore all pseudo-locked regions
+ * @work: the work
+ *
+ * Instructions like wbinvd would cause pseudo-locked data to be evicted.
+ * Cases like this may be detected but it may not be obvious which
+ * pseudo-locked regions are affected. To handle severe cases like this
+ * pseudo_lock_restore_work_fn() will restore all pseudo-locked regions
+ * on the system.
+ * Since this may occur automatically a WARN will be triggered to notify
+ * the user of this event.
+ */
+static void pseudo_lock_restore_work_fn(struct work_struct *work)
+{
+	struct rdtgroup *rdtgrp;
+
+	cpus_read_lock();
+	mutex_lock(&rdtgroup_mutex);
+	list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) {
+		if (WARN_ON_ONCE(rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)) {
+			pr_err("Automatic restore of Pseudo-Locked region %s\n",
+			       rdtgrp->kn->name);
+			_pseudo_lock_restore(rdtgrp);
+		}
+	}
+	mutex_unlock(&rdtgroup_mutex);
+	cpus_read_unlock();
+}
+
+void intel_rdtgroup_pseudo_lock_restore_all(void)
+{
+	if (static_branch_unlikely(&rdt_enable_key))
+		schedule_work(&pseudo_lock_restore_work);
+}
+EXPORT_SYMBOL(intel_rdtgroup_pseudo_lock_restore_all);
+
 static int pseudo_lock_dev_open(struct inode *inode, struct file *filp)
 {
 	struct rdtgroup *rdtgrp;

index 317fc59b512c..01ff02eb0e5c 100644
@@ -15,6 +15,9 @@
  * use a variable and mimic reads and writes to it to enforce serialization
  */
 extern unsigned long __force_order;
+#ifdef CONFIG_INTEL_RDT
+void intel_rdtgroup_pseudo_lock_restore_all(void);
+#endif
 
 static inline unsigned long native_read_cr0(void)
 {

@@ -131,7 +134,15 @@ static inline void __write_pkru(u32 pkru)
 
 static inline void native_wbinvd(void)
 {
-	asm volatile("wbinvd": : :"memory");
+	asm volatile("wbinvd" : : : "memory");
+#ifdef CONFIG_INTEL_RDT
+	intel_rdtgroup_pseudo_lock_restore_all();
+#endif
+}
+
+static inline void native_wbinvd_only(void)
+{
+	asm volatile("wbinvd" : : : "memory");
 }
 
 extern asmlinkage void native_load_gs_index(unsigned);

index d395e6982467..298ac2b34089 100644
@@ -477,7 +477,7 @@ static int pseudo_lock_fn(void *_rdtgrp)
 	 * increase likelihood that allocated cache portion will be filled
 	 * with associated memory.
 	 */
-	native_wbinvd();
+	native_wbinvd_only();
 
 	/*
 	 * Always called with interrupts enabled. By disabling interrupts

index 729579fb654f..325357cda202 100644
@@ -213,7 +213,7 @@ static int __init nettel_init(void)
 	maxsize = AMD_WINDOW_MAXSIZE;
 
 	*amdpar = SC520_PAR(SC520_PAR_BOOTCS, amdaddr, maxsize);
-	__asm__ ("wbinvd");
+	native_wbinvd();
 
 	nettel_amd_map.phys = amdaddr;
 	nettel_amd_map.virt = ioremap_nocache(amdaddr, maxsize);

@@ -326,7 +326,7 @@ static int __init nettel_init(void)
 	 */
 	intel1addr = intel0addr + intel0size;
 	*intel1par = SC520_PAR(intel1cs, intel1addr, maxsize);
-	__asm__ ("wbinvd");
+	native_wbinvd();
 
 	maxsize += intel0size;
 

@@ -352,7 +352,7 @@ static int __init nettel_init(void)
 	intel1size = intel_mtd->size - intel0size;
 	if (intel1size > 0) {
 		*intel1par = SC520_PAR(intel1cs, intel1addr, intel1size);
-		__asm__ ("wbinvd");
+		native_wbinvd();
 	} else {
 		*intel1par = 0;
 	}

index 7bfaaad1d0fa..37fd765c89a3 100644
@@ -54,7 +54,7 @@ static inline void i810_delete_i2c_busses(struct i810fb_par *par) { }
 #ifdef CONFIG_X86
 static inline void flush_cache(void)
 {
-	asm volatile ("wbinvd":::"memory");
+	native_wbinvd();
 }
 #else
 #define flush_cache() do { } while(0)