nitro_enclaves: Add support for Nitro Sandboxes

292 lines not on this page · raw message

index 283c17558f2a3..23a783bab3d2f 100644
@@ -54,7 +54,10 @@ struct ne_mem_region {
  *				per full CPU core in the NE CPU pool the enclave
  *				was created from.
  * @nr_vcpus:			Number of vcpus associated with the enclave.
- * @numa_node:			NUMA node of the enclave memory and CPUs.
+ * @numa_node:			NUMA node of the enclave memory and CPUs, or
+ *				NUMA_NO_NODE if the CPU pool it was created from
+ *				spans nodes, in which case its memory is not held
+ *				to a node.
  * @slot_uid:			Slot unique id mapped to the enclave.
  * @state:			Enclave state, updated during enclave lifetime.
  * @threads_per_core:		Enclave full CPU cores array. Each cpumask in the

index e808f5ba124d4..206546f24748e 100644
@@ -26,8 +26,8 @@
  *		  https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html
  *		  CPU 0 and its siblings have to remain available for the
  *		  primary / parent VM, so they cannot be set for enclaves. Full
- *		  CPU core(s), from the same NUMA node, need(s) to be included
- *		  in the CPU pool.
+ *		  CPU core(s) need(s) to be included in the CPU pool. The
+ *		  pool may span NUMA nodes.
  *
  * Context: Process context.
  * Return:

@@ -50,8 +50,9 @@
  * NE_ADD_VCPU - The command is used to set a vCPU for an enclave. The vCPU can
  *		 be auto-chosen from the NE CPU pool or it can be set by the
  *		 caller, with the note that it needs to be available in the NE
- *		 CPU pool. Full CPU core(s), from the same NUMA node, need(s) to
- *		 be associated with an enclave.
+ *		 CPU pool. Full CPU core(s) need(s) to be associated with an
+ *		 enclave. When the CPU pool spans NUMA nodes, the cores of
+ *		 one enclave can be from different nodes.
  *		 The vCPU id is an input / output parameter. If its value is 0,
  *		 then a CPU is chosen from the enclave CPU pool and returned via
  *		 this parameter.

@@ -108,8 +109,11 @@
 /**
  * NE_SET_USER_MEMORY_REGION - The command is used to set a memory region for an
  *			       enclave, given the allocated memory from the
- *			       userspace. Enclave memory needs to be from the
- *			       same NUMA node as the enclave CPUs.
+ *			       userspace. When the CPU pool sits on one
+ *			       NUMA node, enclave memory needs to be from
+ *			       that node. An enclave built out of a pool
+ *			       that spans nodes has no node of its own, and
+ *			       its memory can come from any node.
  *			       The user memory region is an input parameter. It
  *			       includes info provided by the caller - flags,
  *			       memory size and userspace address.

@@ -138,7 +142,9 @@
  * * NE_ERR_MEM_NOT_HUGE_PAGE		- The memory region is not backed by
  *					  huge pages.
  * * NE_ERR_MEM_DIFFERENT_NUMA_NODE	- The memory region is not from the same
- *					  NUMA node as the CPUs.
+ *					  NUMA node as the CPUs. Not returned
+ *					  for an enclave whose CPU pool spans
+ *					  nodes.
  * * NE_ERR_MEM_MAX_REGIONS		- The number of memory regions set for
  *					  the enclave reached maximum.
  * * NE_ERR_INVALID_PAGE_SIZE		- The memory region is not backed by

@@ -223,6 +229,8 @@
 /**
  * NE_ERR_MEM_DIFFERENT_NUMA_NODE - The user space memory region is backed by
  *				    pages from different NUMA nodes than the CPUs.
+ *				    Not returned for an enclave whose CPU pool
+ *				    spans nodes.
  */
 #define NE_ERR_MEM_DIFFERENT_NUMA_NODE		(264)
 /**

index 765b131c73190..84caf5e66fcbd 100644
@@ -11,9 +11,10 @@
  * -----
  *
  * Load the nitro_enclaves module, setting also the enclave CPU pool. The
- * enclave CPUs need to be full cores from the same NUMA node. CPU 0 and its
- * siblings have to remain available for the primary / parent VM, so they
- * cannot be included in the enclave CPU pool.
+ * enclave CPUs need to be full cores. They come from the same NUMA node
+ * unless the CPU pool spans several of them. CPU 0 and its siblings have to
+ * remain available for the primary / parent VM, so they cannot be included
+ * in the enclave CPU pool.
  *
  * See the cpu list section from the kernel documentation.
  * https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html#cpu-lists

@@ -41,7 +42,8 @@
  *	dmesg
  *
  * Setup hugetlbfs huge pages. The memory needs to be from the same NUMA node as
- * the enclave CPUs.
+ * the enclave CPUs, unless the CPU pool spans nodes, in which case the enclave
+ * is not held to a node and its memory can come from any of them.
  *
  * https://www.kernel.org/doc/html/latest/admin-guide/mm/hugetlbpage.html
  *

index 6e36abeff01a2..576f1909c7304 100644
@@ -9,11 +9,13 @@
  */
 
 #include <linux/anon_inodes.h>
+#include <linux/build_bug.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
 #include <linux/device.h>
 #include <linux/file.h>
 #include <linux/hugetlb.h>
+#include <linux/ioctl.h>
 #include <linux/kernel.h>
 #include <linux/limits.h>
 #include <linux/list.h>

@@ -24,12 +26,15 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/nitro_enclaves.h>
+#include <linux/nodemask.h>
 #include <linux/numa.h>
 #include <linux/pci.h>
 #include <linux/poll.h>
 #include <linux/range.h>
 #include <linux/slab.h>
+#include <linux/stddef.h>
 #include <linux/types.h>
+#include <linux/uaccess.h>
 #include <uapi/linux/vm_sockets.h>
 
 #include "ne_misc_dev.h"

@@ -42,6 +47,23 @@
  */
 #define NE_CPUS_SIZE		(512)
 
+/**
+ * NE_EA_IOCTL() - Mask the size and direction bits out of an ioctl command, so
+ *		   that an Extensible Argument ioctl matches whatever size of
+ *		   argument the caller passed. The direction bits go with them,
+ *		   so a caller that spells the command _IOW(), _IOR() or
+ *		   _IOWR() reaches the same handler, and the handler decides
+ *		   what it reads and writes.
+ * @cmd:	The ioctl command to mask.
+ */
+#define NE_EA_IOCTL(cmd)	((cmd) & ~(IOC_INOUT | IOCSIZE_MASK))
+
+/*
+ * NE_NUMA_ANY reaches the core allocator as a node id, so the uapi spelling
+ * and the kernel one have to agree.
+ */
+static_assert(NE_NUMA_ANY == NUMA_NO_NODE);
+
 /**
  * NE_EIF_LOAD_OFFSET - The offset where to copy the Enclave Image Format (EIF)
  *			image in enclave memory.

@@ -334,7 +356,8 @@ static int ne_setup_cpu_pool(const struct cpumask *cpu_pool)
 	 * multiple NUMA nodes, set numa_node to NUMA_NO_NODE; the pool is then
 	 * treated as node-agnostic for placement and memory-locality checks.
 	 * A CPU that reports no node at all is refused, because otherwise such
-	 * a CPU would join a single-node pool and turn it into a spanning one.
+	 * a CPU would join a single-node pool and turn it into a spanning one,
+	 * and its core would be unreachable through a node-named pick.
 	 */
 	for_each_cpu(cpu, cpu_pool) {
 		int nid = cpu_to_node(cpu);

@@ -1592,6 +1615,73 @@ static long ne_enclave_do_ioctl(struct file *file, unsigned int cmd, unsigned lo
 
 		return 0;
 	}
+	}
+
+	/* Extensible Argument ioctls */
+	switch (NE_EA_IOCTL(cmd)) {
+	case NE_EA_IOCTL(NE_ADD_ANY_VCPU): {
+		struct ne_add_any_vcpu_args __user *uargs = (void __user *)arg;
+		unsigned int usize = _IOC_SIZE(cmd);
+		struct ne_add_any_vcpu_args args;
+		u32 vcpu_id = 0;
+		int rc;
+
+		if (usize < offsetofend(struct ne_add_any_vcpu_args, vcpu_id))
+			return -EINVAL;
+
+		rc = copy_struct_from_user(&args, sizeof(args), uargs, usize);
+		if (rc)
+			return rc;
+
+		if (args.numa_node != NE_NUMA_ANY &&
+		    (args.numa_node < 0 || args.numa_node >= nr_node_ids ||
+		     !node_state(args.numa_node, N_POSSIBLE))) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "Invalid NUMA node %d\n", args.numa_node);
+
+			return -EINVAL;
+		}
+
+		rc = mutex_lock_killable(&ne_enclave->enclave_info_mutex);
+		if (rc)
+			return rc;
+
+		if (ne_enclave->state != NE_STATE_INIT) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "Enclave is not in init state\n");
+
+			mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+			return -NE_ERR_NOT_IN_INIT_STATE;
+		}
+
+		rc = ne_get_cpu_from_cpu_pool(ne_enclave, &vcpu_id, args.numa_node);
+		if (rc < 0) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "Error in get CPU from pool [rc=%d node=%d]\n",
+					    rc, args.numa_node);
+
+			mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+			return rc;
+		}
+
+		rc = ne_add_vcpu_ioctl(ne_enclave, vcpu_id);
+		if (rc < 0) {
+			mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+			return rc;
+		}
+
+		mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+		args.vcpu_id = vcpu_id;
+
+		if (copy_to_user(uargs, &args, min_t(size_t, usize, sizeof(args))))
+			return -EFAULT;
+
+		return 0;
+	}
 
 	default:
 		return -ENOTTY;

60 lines not on this page · raw message

493 lines not on this page · raw message

index 23a783bab3d2f..483037061eafe 100644
@@ -115,4 +115,7 @@ struct ne_devs {
 /* Nitro Enclaves (NE) data structure for keeping refs to the NE misc and PCI devices. */
 extern struct ne_devs ne_devs;
 
+int ne_cpu_pool_sysfs_init(void);
+void ne_cpu_pool_sysfs_exit(void);
+
 #endif /* _NE_MISC_DEV_H_ */

index edc72d4f623e5..2d44cae4ac0cf 100644
@@ -695,8 +695,20 @@ static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	up_write(&ne_devs.lock);
 
+	rc = ne_cpu_pool_sysfs_init();
+	if (rc < 0) {
+		dev_err(&pdev->dev, "Error in cpu pool sysfs init [rc=%d]\n",
+			rc);
+
+		goto deregister_misc_dev;
+	}
+
 	return 0;
 
+deregister_misc_dev:
+	down_write(&ne_devs.lock);
+	ne_pci_dev_detach_enclaves(ne_pci_dev);
+	misc_deregister(ne_devs.ne_misc_dev);
 disable_ne_pci_dev:
 	ne_devs.ne_pci_dev = NULL;
 unlock_ne_devs:

@@ -727,6 +739,8 @@ static void ne_pci_remove(struct pci_dev *pdev)
 {
 	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
 
+	ne_cpu_pool_sysfs_exit();
+
 	down_write(&ne_devs.lock);
 
 	ne_pci_dev_detach_enclaves(ne_pci_dev);

@@ -765,6 +779,8 @@ static void ne_pci_shutdown(struct pci_dev *pdev)
 	if (!ne_pci_dev)
 		return;
 
+	ne_cpu_pool_sysfs_exit();
+
 	down_write(&ne_devs.lock);
 
 	ne_pci_dev_detach_enclaves(ne_pci_dev);

index afaf2c12189b8..5f7138ea6899a 100644
@@ -93,6 +93,7 @@ static inline void set_dma_ops(struct device *dev,
 #ifdef CONFIG_DMA_CMA
 struct cma *dev_get_cma_area(struct device *dev);
 struct cma *dma_contiguous_get_area_by_idx(unsigned int idx);
+struct cma *dma_get_numa_cma_area(int nid);
 
 void dma_contiguous_reserve(phys_addr_t addr_limit);
 int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base,

@@ -115,6 +116,12 @@ static inline struct cma *dma_contiguous_get_area_by_idx(unsigned int idx)
 {
 	return NULL;
 }
+
+static inline struct cma *dma_get_numa_cma_area(int nid)
+{
+	return NULL;
+}
+
 static inline void dma_contiguous_reserve(phys_addr_t limit)
 {
 }

index 90937dc7c627d..363b84284d087 100644
@@ -443,6 +443,42 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
 	return cma_alloc_aligned(dma_contiguous_default_area, size, gfp);
 }
 
+/**
+ * dma_get_numa_cma_area() - get the CMA area a NUMA node can allocate from
+ * @nid: NUMA node id
+ *
+ * dma_alloc_contiguous() picks a per-node CMA area from dev_to_node(dev),
+ * which a caller cannot override on a per-request basis. This accessor
+ * returns the area a specific node can allocate from, so a caller can
+ * cma_alloc() from the node it wants.
+ *
+ * A per-node area comes from numa_cma= or cma_pernuma=, or from
+ * CONFIG_CMA_SIZE_PERNUMA, which reserves the default area's size on every
+ * online node on top of the default area itself, all three of which are
+ * CONFIG_DMA_NUMA_CMA. Where there is one online node this returns the default
+ * area, so a host that reserved only cma= reaches its reservation through this
+ * accessor as well, and that holds however CONFIG_DMA_NUMA_CMA is set.
+ *
+ * Return: the area @nid can allocate from, or NULL if @nid has none or is not
+ * a valid node (including NUMA_NO_NODE).
+ */
+struct cma *dma_get_numa_cma_area(int nid)
+{
+	if ((unsigned int)nid >= MAX_NUMNODES)
+		return NULL;
+
+#ifdef CONFIG_DMA_NUMA_CMA
+	if (dma_contiguous_numa_area[nid])
+		return dma_contiguous_numa_area[nid];
+#endif
+
+	if (nr_online_nodes == 1 && node_online(nid))
+		return dma_contiguous_default_area;
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(dma_get_numa_cma_area);
+
 /**
  * dma_free_contiguous() - release allocated pages
  * @dev:   Pointer to device for which the pages were allocated.

791 lines not on this page · raw message

85 lines not on this page · raw message

108 lines not on this page · raw message

index 56ca1a9d99d41..55dbeb5b14d29 100644
@@ -2597,17 +2597,21 @@ static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
  * serving such a request out of movable memory would take that memory from
  * the rest of the host and leave the reservation bounding nothing.
  *
- * A request larger than one contiguous run is satisfied by several runs, which
- * the driver keeps on a list of its own so that an enclave can be given every
- * one of them.
+ * A request larger than one contiguous run is satisfied by several runs, and
+ * mmap() maps them back to back into one VM_PFNMAP VMA, so user space sees a
+ * single region. The driver keeps the runs on a list of its own, so that an
+ * enclave can be given them without anyone reading them back out of the page
+ * tables.
  *
- * An allocation is never mapped, and the enclave image reaches it through
- * write().
+ * An allocation is either private or shared. A private one is never mapped, and
+ * the enclave image reaches it through write() instead. A shared one can also be
+ * mapped, at an offset and a length its own size bounds.
  *
  * The runs belong to the file the allocation is handed out on. Zeroing a fresh
  * run and a write into an allocation go through ne_walk_run(), so that neither
- * can run for as long as the allocation is large; the release path walks the
- * runs with a preemption point of its own.
+ * can run for as long as the allocation is large. A fault resolves the run its
+ * offset falls in with a binary search instead of walking, and the release path
+ * walks the runs with a preemption point of its own.
  */
 
 /**

@@ -2788,6 +2792,82 @@ static int ne_walk_allocs(struct ne_contig_mem *mem, u64 off, u64 len,
 	return 0;
 }
 
+/**
+ * ne_contig_fault_order() - Install one page table entry for a fault.
+ * @vmf:	The fault.
+ * @order:	Order of the entry to install, zero for a PTE.
+ *
+ * A mapping is populated as it is touched, so a caller that maps a whole
+ * allocation and builds an enclave image at the front of it pays page tables
+ * for the part it touched.
+ *
+ * An entry of @order needs that much of one run behind it, at a matching
+ * physical alignment. A run is a power-of-two multiple of 2 MiB aligned to at
+ * least 2 MiB, so a PMD sized entry fits inside one whenever the address it
+ * lands at is aligned too, and anything that does not fit falls back to the
+ * next order down.
+ *
+ * Return: a VM_FAULT_* value.
+ */
+static vm_fault_t ne_contig_fault_order(struct vm_fault *vmf, unsigned int order)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct ne_contig_mem *mem = vma->vm_file->private_data;
+	unsigned long len = PAGE_SIZE << order;
+	unsigned long addr = ALIGN_DOWN(vmf->address, len);
+	struct ne_contig_alloc *alloc;
+	unsigned long pfn;
+	u64 off;
+
+	if (addr < vma->vm_start || addr + len > vma->vm_end)
+		return VM_FAULT_FALLBACK;
+
+	off = ((u64)vma->vm_pgoff << PAGE_SHIFT) + (addr - vma->vm_start);
+
+	alloc = ne_find_alloc(mem, off);
+	if (!alloc)
+		return VM_FAULT_SIGBUS;
+
+	if (off + len > alloc->off + ((u64)alloc->nr_pages << PAGE_SHIFT))
+		return VM_FAULT_FALLBACK;
+
+	pfn = page_to_pfn(alloc->page) + ((off - alloc->off) >> PAGE_SHIFT);
+	if (!IS_ALIGNED(pfn, len >> PAGE_SHIFT))
+		return VM_FAULT_FALLBACK;
+
+	if (!order)
+		return vmf_insert_pfn(vma, addr, pfn);
+
+	if (IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP) && order == PMD_ORDER)
+		return vmf_insert_pfn_pmd(vmf, pfn,
+					  vmf->flags & FAULT_FLAG_WRITE);
+
+	if (IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP) && order == PUD_ORDER)
+		return vmf_insert_pfn_pud(vmf, pfn,
+					  vmf->flags & FAULT_FLAG_WRITE);
+
+	return VM_FAULT_FALLBACK;
+}
+
+static vm_fault_t ne_contig_vma_fault(struct vm_fault *vmf)
+{
+	return ne_contig_fault_order(vmf, 0);
+}
+
+#ifdef CONFIG_ARCH_SUPPORTS_HUGE_PFNMAP
+static vm_fault_t ne_contig_huge_fault(struct vm_fault *vmf, unsigned int order)
+{
+	return ne_contig_fault_order(vmf, order);
+}
+#endif
+
+static const struct vm_operations_struct ne_contig_vm_ops = {
+	.fault		= ne_contig_vma_fault,
+#ifdef CONFIG_ARCH_SUPPORTS_HUGE_PFNMAP
+	.huge_fault	= ne_contig_huge_fault,
+#endif
+};
+
 static struct cma *ne_numa_cma_area(int nid)
 {
 	struct cma *cma = dma_get_numa_cma_area(nid);

@@ -2915,6 +2995,69 @@ struct page *ne_alloc_contig(unsigned long nr_pages, int nid,
 	return page;
 }
 
+/**
+ * ne_contig_supports_mmap() - Whether an allocation may be mapped at all.
+ * @file:	The allocation's file.
+ *
+ * A private allocation is the one an enclave gets to itself, and it is never
+ * mapped. The attribute is fixed when the allocation is made and the file is
+ * not published until it is set, so this reads it without a lock.
+ *
+ * Return: true when the caller may map the allocation.
+ */
+static bool ne_contig_supports_mmap(struct file *file)
+{
+	struct ne_contig_mem *mem = file->private_data;
+
+	return mem->shared;
+}
+
+/**
+ * ne_contig_mmap_prepare() - Set a mapping of part or all of an allocation up.
+ * @desc:	The mapping to set up.
+ *
+ * The offset and the length both have to be multiples of 2 MiB, which is the
+ * unit the allocation itself is made in, and the mapping has to stay inside
+ * the allocation. A caller may map part of an allocation, and may map one
+ * allocation more than once; the runs are not mutated after the file is
+ * published and the file holds them for as long as any mapping exists.
+ *
+ * ne_contig_fault_order() populates the mapping as it is touched. The mapping
+ * asks for VM_HUGEPAGE because a fault on a VM_PFNMAP mapping without it only
+ * reaches ->huge_fault where the transparent_hugepage policy is "always".
+ *
+ * Return: 0 on success, negative return value on failure.
+ */
+static int ne_contig_mmap_prepare(struct vm_area_desc *desc)
+{
+	struct ne_contig_mem *mem = desc->file->private_data;
+	unsigned long size = vma_desc_size(desc);
+	u64 offset = (u64)desc->pgoff << PAGE_SHIFT;
+
+	if (!ne_contig_supports_mmap(desc->file))
+		return -ENODEV;
+
+	/*
+	 * MAP_PRIVATE of an allocation has no meaning: the memory exists to be
+	 * given to an enclave, and a copy that only this process can see never
+	 * reaches one.
+	 */
+	if (!vma_desc_test(desc, VMA_SHARED_BIT))
+		return -EINVAL;
+
+	if ((size | offset) & (NE_MIN_MEM_REGION_SIZE - 1))
+		return -EINVAL;
+
+	if (offset > mem->size || size > mem->size - offset)
+		return -EINVAL;
+
+	desc->vm_ops = &ne_contig_vm_ops;
+	vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
+			   VMA_DONTDUMP_BIT, VMA_DONTCOPY_BIT, VMA_HUGEPAGE_BIT);
+
+	return 0;
+}
+
 /**
  * struct ne_write_ctx - Where the write walk has got to.
  * @iter:	Where the bytes come from.

@@ -3030,6 +3173,8 @@ static int ne_contig_release(struct inode *inode, struct file *file)
 static const struct file_operations ne_contig_fops = {
 	.owner			= THIS_MODULE,
 	.llseek			= generic_file_llseek,
+	.mmap_prepare		= ne_contig_mmap_prepare,
+	.get_unmapped_area	= thp_get_unmapped_area,
 	.write_iter		= ne_contig_write_iter,
 	.release		= ne_contig_release,
 };

@@ -3176,6 +3321,7 @@ static int ne_alloc_memory_ioctl(struct ne_enclave *ne_enclave,
 	mem->mm = current->mm;
 	mem->nr_pinned = nr_pages;
 	mem->size = args->size;
+	mem->shared = args->flags == NE_MEMORY_SHARED;
 	mem->enclave_id = ne_enclave->id;
 
 	/*

index 1a76c12815298..fcdf457af5800 100644
@@ -38,6 +38,7 @@ struct ne_contig_alloc {
  * @allocs:	Contiguous runs backing the allocation, ordered by offset.
  * @nr_allocs:	Number of runs in @allocs.
  * @size:	Size of the allocation in bytes, mirrored in i_size.
+ * @shared:	Whether the allocation may be mapped by the caller.
  * @enclave_id:	Id of the enclave the allocation was made for.
  * @mm:		The mm charged for the pinned pages, held via mmgrab().
  * @nr_pinned:	Pages charged to @mm->pinned_vm.

@@ -47,13 +48,14 @@ struct ne_contig_alloc {
  *
  * The array is built before the file is published and not mutated afterwards, so
  * any holder of a reference can read it without a lock of its own, which is what
- * lets a page fault resolve an offset to a run. @size is set in the same place
- * and is read the same way.
+ * lets a page fault resolve an offset to a run. @shared and @size are set in the
+ * same place and are read the same way.
  */
 struct ne_contig_mem {
 	struct ne_contig_alloc	*allocs;
 	unsigned int		nr_allocs;
 	u64			size;
+	bool			shared;
 	u64			enclave_id;
 	struct mm_struct	*mm;
 	unsigned long		nr_pinned;

index a2919565bc883..f1315c9c876de 100644
@@ -215,7 +215,7 @@ struct ne_add_any_vcpu_args {
  * @size:	Number of bytes to allocate, a multiple of 2 MiB.
  * @nid:	Host NUMA node to allocate from. One node, named; unlike
  *		%NE_ADD_ANY_VCPU it does not take %NE_NUMA_ANY.
- * @flags:	%NE_MEMORY_PRIVATE.
+ * @flags:	%NE_MEMORY_PRIVATE or %NE_MEMORY_SHARED.
  */
 struct ne_alloc_memory_args {
 	__u64	size;

@@ -237,7 +237,16 @@ struct ne_alloc_memory_args {
  *		     goes into one through write() or pwrite() and not through
  *		     a copy into a mapping, and every write is a call the
  *		     kernel handles and can act on or refuse.
- *		     The allocation lives until the descriptor is gone.
+ *		     A %NE_MEMORY_SHARED allocation can also be mapped, at
+ *		     offset 0 or at any other multiple of 2 MiB, for a multiple
+ *		     of 2 MiB, with %MAP_SHARED. A caller maps one to keep an
+ *		     address for memory it goes on using after it has handed the
+ *		     allocation over. Such a mapping is readable too, which is
+ *		     the only way to read allocated memory back. The mapping is
+ *		     not inherited across fork(); a child that needs one maps
+ *		     the descriptor itself.
+ *		     The allocation lives until the last of the descriptor and
+ *		     its mappings is gone.
  *		     The ioctl can be invoked on the enclave fd, before an
  *		     enclave is started.
  *

@@ -276,6 +285,17 @@ struct ne_alloc_memory_args {
  *				  write, so a caller that has to know how far it
  *				  got looks at the return value. A fatal signal
  *				  part way through is reported the same way.
+ *
+ * An mmap() of the descriptor sets errno to:
+ * * ENODEV			- The allocation is %NE_MEMORY_PRIVATE.
+ * * EINVAL			- The mapping is %MAP_PRIVATE, or its offset or
+ *				  length is not a multiple of 2 MiB, or it does
+ *				  not fit inside the allocation.
+ * A mapping is populated as it is touched, so the page tables for a page are
+ * allocated at first touch and not by mmap(). A touch by the process itself
+ * that fails to get them is retried until it succeeds or the process is
+ * killed. A touch the kernel makes for the process, as read() into the
+ * mapping does, reports EFAULT or a short transfer instead.
  */
 #define NE_ALLOC_MEMORY			_IOW(0xAE, 0x2E, struct ne_alloc_memory_args)
 

@@ -472,8 +492,18 @@ struct ne_user_memory_region {
  *		       way to put an enclave image in one.
  */
 #define NE_MEMORY_PRIVATE		(0x00)
+/**
+ * NE_MEMORY_SHARED - An allocation the parent may also map. A mapping is what a
+ *		      caller wants for memory it goes on reading or writing
+ *		      after it has handed the allocation over. It costs page
+ *		      tables for the part of it a caller touches, and it leaves
+ *		      the memory reachable from the parent for as long as it
+ *		      lives, so an allocation that is only being handed over
+ *		      does not ask for one.
+ */
+#define NE_MEMORY_SHARED		(0x01)
 
-#define NE_ALLOC_MEMORY_MAX_FLAG_VAL	(0x01)
+#define NE_ALLOC_MEMORY_MAX_FLAG_VAL	(0x02)
 
 /**
  * DOC: Enclave start info flags

index 55dbeb5b14d29..38345ad7c7985 100644
@@ -119,6 +119,8 @@ static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
  */
 static atomic64_t ne_enclave_seq = ATOMIC64_INIT(0);
 
+static int ne_add_memory_region_ioctl(struct ne_enclave *ne_enclave,
+				      struct ne_add_memory_region_args *args);
 static int ne_alloc_memory_ioctl(struct ne_enclave *ne_enclave,
 				 struct ne_alloc_memory_args *args);
 static const struct file_operations ne_fops = {

@@ -2069,6 +2071,39 @@ static long ne_enclave_do_ioctl(struct file *file, unsigned int cmd, unsigned lo
 		return 0;
 	}
 
+	case NE_EA_IOCTL(NE_ADD_MEMORY_REGION): {
+		struct ne_add_memory_region_args __user *uargs = (void __user *)arg;
+		unsigned int usize = _IOC_SIZE(cmd);
+		struct ne_add_memory_region_args args;
+		int rc;
+
+		if (usize < offsetofend(struct ne_add_memory_region_args, flags))
+			return -EINVAL;
+
+		rc = copy_struct_from_user(&args, sizeof(args), uargs, usize);
+		if (rc)
+			return rc;
+
+		rc = mutex_lock_killable(&ne_enclave->enclave_info_mutex);
+		if (rc)
+			return rc;
+
+		if (ne_enclave->state != NE_STATE_INIT) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "Enclave is not in init state\n");
+
+			mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+			return -NE_ERR_NOT_IN_INIT_STATE;
+		}
+
+		rc = ne_add_memory_region_ioctl(ne_enclave, &args);
+
+		mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+		return rc;
+	}
+
 	case NE_EA_IOCTL(NE_ALLOC_MEMORY): {
 		struct ne_alloc_memory_args __user *uargs = (void __user *)arg;
 		unsigned int usize = _IOC_SIZE(cmd);

@@ -2139,6 +2174,33 @@ static long ne_enclave_ioctl(struct file *file, unsigned int cmd, unsigned long
 	return rc;
 }
 
+/**
+ * ne_enclave_remove_all_contig_regions() - Drop the enclave's references to the
+ *					    allocations it was given.
+ * @ne_enclave :	Private data associated with the current enclave.
+ *
+ * The last reference on an allocation's file frees its runs and gives back its
+ * %RLIMIT_MEMLOCK charge, so an allocation whose descriptor and mappings are
+ * already gone is freed here.
+ *
+ * Context: Process context. This function is called on an enclave that is on
+ *	    no list and has no file left.
+ */
+static void ne_enclave_remove_all_contig_regions(struct ne_enclave *ne_enclave)
+{
+	struct ne_contig_region *ne_contig_region = NULL;
+	struct ne_contig_region *tmp = NULL;
+
+	list_for_each_entry_safe(ne_contig_region, tmp,
+				 &ne_enclave->contig_regions_list, list) {
+		list_del(&ne_contig_region->list);
+
+		fput(ne_contig_region->file);
+
+		kfree(ne_contig_region);
+	}
+}
+
 /**
  * ne_enclave_remove_all_mem_region_entries() - Remove all memory region entries
  *						from the enclave data structure.

@@ -2278,6 +2340,7 @@ static int ne_enclave_free_slot(struct ne_pci_dev *ne_pci_dev,
  */
 static void ne_enclave_free(struct ne_enclave *ne_enclave)
 {
+	ne_enclave_remove_all_contig_regions(ne_enclave);
 	ne_enclave_remove_all_mem_region_entries(ne_enclave);
 	ne_enclave_remove_all_vcpu_id_entries(ne_enclave);
 

@@ -2494,6 +2557,7 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_ui
 	ne_enclave->has_event = false;
 	mutex_init(&ne_enclave->enclave_info_mutex);
 	ne_enclave->max_mem_regions = cmd_reply.mem_regions;
+	INIT_LIST_HEAD(&ne_enclave->contig_regions_list);
 	INIT_LIST_HEAD(&ne_enclave->mem_regions_list);
 	ne_enclave->mm = current->mm;
 	ne_enclave->slot_uid = cmd_reply.slot_uid;

@@ -3116,6 +3180,12 @@ static int ne_write_span(void *arg, struct page *page, u64 off, u64 len)
  * refusal is -EFBIG, the same as generic_write_check_limits() gives for a write
  * beyond s_maxbytes. There is no RLIMIT_FSIZE in it, so no SIGXFSZ either.
  *
+ * Donating an allocation closes this path, and @lock is what makes the two
+ * orderly. It is taken here and in ne_add_memory_region_ioctl() and nowhere
+ * else; setting a mapping up and faulting one in both read only fields that
+ * never change, so nothing takes it under mmap_write_lock or in a fault, and
+ * copying from user space under it cannot invert against the fault path.
+ *
  * Return: the number of bytes written, or a negative return value on failure.
  */
 static ssize_t ne_contig_write_iter(struct kiocb *iocb, struct iov_iter *from)

@@ -3132,6 +3202,11 @@ static ssize_t ne_contig_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	if (pos >= mem->size || count > mem->size - pos)
 		return -EFBIG;
 
+	guard(mutex)(&mem->lock);
+
+	if (mem->donated)
+		return -EPERM;
+
 	rc = ne_walk_allocs(mem, pos, count, NE_WRITE_SPAN_PAGES, ne_write_span,
 			    &ctx);
 

@@ -3165,6 +3240,8 @@ static int ne_contig_release(struct inode *inode, struct file *file)
 
 	mmdrop(mem->mm);
 
+	mutex_destroy(&mem->lock);
+
 	kfree(mem);
 
 	return 0;

@@ -3317,6 +3394,7 @@ static int ne_alloc_memory_ioctl(struct ne_enclave *ne_enclave,
 		return -ENOMEM;
 	}
 
+	mutex_init(&mem->lock);
 	mmgrab(current->mm);
 	mem->mm = current->mm;
 	mem->nr_pinned = nr_pages;

@@ -3385,11 +3463,195 @@ static int ne_alloc_memory_ioctl(struct ne_enclave *ne_enclave,
 	ne_contig_free_allocs(mem);
 	atomic64_sub(nr_pages, &mem->mm->pinned_vm);
 	mmdrop(mem->mm);
+	mutex_destroy(&mem->lock);
 	kfree(mem);
 
 	return rc;
 }
 
+/**
+ * ne_add_memory_region_ioctl() - Give the slot associated with the current
+ *				  enclave a whole allocation.
+ * @ne_enclave:	Private data associated with the current enclave.
+ * @args:	Descriptor for the allocation, and flags.
+ *
+ * Every run the allocation is made of becomes a memory region, except that runs
+ * which turn out to be physically adjacent are merged into one, so a request
+ * that took several runs takes up several of the enclave's regions or fewer.
+ * The runs are 2 MiB aligned and a multiple of 2 MiB long as they come out of
+ * NE_ALLOC_MEMORY, which is what the hypervisor asks of a region, and no
+ * virtual address is involved on this path at all.
+ *
+ * The reference on the file is what keeps the runs and their %RLIMIT_MEMLOCK
+ * charge alive for as long as the enclave has them, whatever user space does
+ * with the descriptor and its mappings afterwards.
+ *
+ * Context: Process context. This function is called with the ne_enclave mutex held.
+ * Return:
+ * * 0 on success.
+ * * Negative return value on failure.
+ */
+static int ne_add_memory_region_ioctl(struct ne_enclave *ne_enclave,
+				      struct ne_add_memory_region_args *args)
+{
+	struct ne_phys_contig_mem_regions phys_contig_mem_regions = {};
+	struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev;
+	struct ne_contig_region *ne_contig_region = NULL;
+	struct ne_contig_region *held = NULL;
+	struct ne_contig_mem *mem = NULL;
+	unsigned long i = 0;
+	int rc = -EINVAL;
+
+	if (ne_enclave->mm != current->mm)
+		return -EIO;
+
+	if (args->flags) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Incorrect flag for memory region\n");
+
+		return -NE_ERR_INVALID_FLAG_VALUE;
+	}
+
+	CLASS(fd, f)(args->fd);
+
+	if (fd_empty(f))
+		return -EBADF;
+
+	if (fd_file(f)->f_op != &ne_contig_fops) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Not an enclave memory allocation\n");
+
+		return -EINVAL;
+	}
+
+	/*
+	 * Every file with these operations came from ne_contig_create_file(),
+	 * which hands the allocation to anon_inode_create_getfile() before there
+	 * is a descriptor to pass in here, so it is there to be read.
+	 */
+	mem = fd_file(f)->private_data;
+
+	if (mem->enclave_id != ne_enclave->id) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Allocation was made for another enclave\n");
+
+		return -EINVAL;
+	}
+
+	/*
+	 * The parent may map a shared allocation and keeps any mapping it has
+	 * for as long as the allocation lives. Memory a region is made of is
+	 * the enclave's alone.
+	 */
+	if (mem->shared) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The memory region allocation is shared\n");
+
+		return -EINVAL;
+	}
+
+	list_for_each_entry(held, &ne_enclave->contig_regions_list, list) {
+		if (held->file == fd_file(f)) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "The memory region is already used\n");
+
+			return -NE_ERR_MEM_REGION_ALREADY_USED;
+		}
+	}
+
+	ne_contig_region = kzalloc_obj(*ne_contig_region);
+	if (!ne_contig_region)
+		return -ENOMEM;
+
+	/*
+	 * One entry per run is the worst case, since merging only ever reduces
+	 * the count. That is 4 MiB of entries for a 512 GiB allocation the
+	 * allocator had to satisfy in 2 MiB runs, so this uses kvmalloc like
+	 * the run array.
+	 */
+	phys_contig_mem_regions.regions = kvzalloc_objs(*phys_contig_mem_regions.regions,
+							mem->nr_allocs);
+	if (!phys_contig_mem_regions.regions) {
+		rc = -ENOMEM;
+
+		goto free_contig_region;
+	}
+
+	for (i = 0; i < mem->nr_allocs; i++) {
+		struct ne_contig_alloc *alloc = &mem->allocs[i];
+
+		rc = ne_merge_phys_contig_memory_regions(&phys_contig_mem_regions,
+							 page_to_phys(alloc->page),
+							 (u64)alloc->nr_pages << PAGE_SHIFT);
+		if (rc < 0)
+			goto free_regions;
+	}
+
+	if ((ne_enclave->nr_mem_regions + phys_contig_mem_regions.num) >
+	    ne_enclave->max_mem_regions) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Reached max memory regions %lld\n",
+				    ne_enclave->max_mem_regions);
+
+		rc = -NE_ERR_MEM_MAX_REGIONS;
+
+		goto free_regions;
+	}
+
+	/*
+	 * From here the allocation is the enclave's. Closing the write path under
+	 * the same lock a write holds means a write either finished before this
+	 * point or is refused after it, so a donation cannot land in the middle of
+	 * one. Whether the whole image got there is the caller's to check: a write
+	 * cut short by an unreadable buffer reports how far it got.
+	 */
+	scoped_guard(mutex, &mem->lock)
+		mem->donated = true;
+
+	ne_contig_region->file = get_file(fd_file(f));
+
+	list_add(&ne_contig_region->list, &ne_enclave->contig_regions_list);
+
+	for (i = 0; i < phys_contig_mem_regions.num; i++) {
+		struct ne_pci_dev_cmd_reply cmd_reply = {};
+		struct slot_add_mem_req slot_add_mem_req = {};
+
+		slot_add_mem_req.slot_uid = ne_enclave->slot_uid;
+		slot_add_mem_req.paddr = phys_contig_mem_regions.regions[i].start;
+		slot_add_mem_req.size = range_len(&phys_contig_mem_regions.regions[i]);
+
+		rc = ne_do_request(pdev, SLOT_ADD_MEM,
+				   &slot_add_mem_req, sizeof(slot_add_mem_req),
+				   &cmd_reply, sizeof(cmd_reply));
+		if (rc < 0) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "Error in slot add mem [rc=%d]\n", rc);
+
+			kvfree(phys_contig_mem_regions.regions);
+
+			/*
+			 * Exit here without dropping the reference on the file
+			 * as memory regions may already been added.
+			 */
+			return rc;
+		}
+
+		ne_enclave->mem_size += slot_add_mem_req.size;
+		ne_enclave->nr_mem_regions++;
+	}
+
+	kvfree(phys_contig_mem_regions.regions);
+
+	return 0;
+
+free_regions:
+	kvfree(phys_contig_mem_regions.regions);
+free_contig_region:
+	kfree(ne_contig_region);
+
+	return rc;
+}
+
 #if defined(CONFIG_NITRO_ENCLAVES_MISC_DEV_TEST)
 #include "ne_misc_dev_test.c"
 #endif

index fcdf457af5800..2502cca3f3f64 100644
@@ -42,14 +42,23 @@ struct ne_contig_alloc {
  * @enclave_id:	Id of the enclave the allocation was made for.
  * @mm:		The mm charged for the pinned pages, held via mmgrab().
  * @nr_pinned:	Pages charged to @mm->pinned_vm.
+ * @lock:	Serialises a write against the donation that ends writing.
+ * @donated:	Whether the allocation has been given to its enclave.
  *
- * The runs hang off the file, and a mapping's page table entries cannot outlive
- * them because a VMA holds vm_file for as long as it exists.
+ * The runs hang off the file, so that closing the last descriptor does not free
+ * memory an enclave is still using: a donated region holds the file with
+ * get_file(). That is also what keeps a mapping's page table entries from
+ * outliving the runs, since a VMA holds vm_file for as long as it exists.
  *
  * The array is built before the file is published and not mutated afterwards, so
  * any holder of a reference can read it without a lock of its own, which is what
  * lets a page fault resolve an offset to a run. @shared and @size are set in the
- * same place and are read the same way.
+ * same place and are read the same way. @donated is the one field that changes
+ * after publication, and @lock covers it.
+ *
+ * @enclave_id is compared, never dereferenced, and the counter it comes from is
+ * 64 bits wide, so an allocation cannot be donated to a later enclave that
+ * happens to have been placed at the address of the one it was made for.
  */
 struct ne_contig_mem {
 	struct ne_contig_alloc	*allocs;

@@ -59,6 +68,20 @@ struct ne_contig_mem {
 	u64			enclave_id;
 	struct mm_struct	*mm;
 	unsigned long		nr_pinned;
+	/* Serialises a write against the donation that ends writing. */
+	struct mutex		lock;
+	bool			donated;
+};
+
+/**
+ * struct ne_contig_region - Enclave memory given from one allocation.
+ * @list:	Entry in the enclave's list of allocations given to it.
+ * @file:	File holding the runs, held with get_file() so that they outlive the
+ *		descriptor the allocation was handed out on and its mappings.
+ */
+struct ne_contig_region {
+	struct list_head	list;
+	struct file		*file;
 };
 
 /**

@@ -82,6 +105,8 @@ struct ne_mem_region {
  * @enclave_info_mutex :	Mutex for accessing this internal state.
  * @detached:			The NE PCI device the enclave was created on is
  *				gone. Written with ne_devs.lock held for writing.
+ * @contig_regions_list:	List of %NE_ALLOC_MEMORY allocations given to the
+ *				enclave with %NE_ADD_MEMORY_REGION.
  * @enclave_list_entry :	Entry in the list of created enclaves.
  * @eventq:			Wait queue used for out-of-band event notifications
  *				triggered from the PCI device event handler to

@@ -116,6 +141,7 @@ struct ne_mem_region {
 struct ne_enclave {
 	struct mutex		enclave_info_mutex;
 	bool			detached;
+	struct list_head	contig_regions_list;
 	struct list_head	enclave_list_entry;
 	wait_queue_head_t	eventq;
 	bool			has_event;

141 lines not on this page · raw message

index 38345ad7c7985..68c497df4f14b 100644
@@ -2452,6 +2452,8 @@ static const struct file_operations ne_enclave_fops = {
  *			  enclave file descriptor to be further used for enclave
  *			  resources handling e.g. memory regions and CPUs.
  * @ne_pci_dev :	Private data associated with the PCI device.
+ * @flags:		Launch-time flag word from NE_CREATE_VM2, or zero when
+ *			the caller used the flag-less NE_CREATE_VM.
  * @slot_uid:		User pointer to store the generated unique slot id
  *			associated with an enclave to.
  *

@@ -2461,7 +2463,8 @@ static const struct file_operations ne_enclave_fops = {
  * * Enclave fd on success.
  * * Negative return value on failure.
  */
-static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_uid)
+static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 flags,
+			      u64 __user *slot_uid)
 {
 	struct ne_pci_dev_cmd_reply cmd_reply = {};
 	int enclave_fd = -1;

@@ -2472,6 +2475,13 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_ui
 	int rc = -EINVAL;
 	struct slot_alloc_req slot_alloc_req = {};
 
+	if (flags) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Unknown launch flag bit set 0x%llx\n", flags);
+
+		return -NE_ERR_INVALID_FLAG_VALUE;
+	}
+
 	mutex_lock(&ne_cpu_pool.mutex);
 
 	for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)

@@ -2626,7 +2636,44 @@ static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		if (ne_pci_dev) {
 			rc = mutex_lock_killable(&ne_pci_dev->enclaves_list_mutex);
 			if (!rc) {
-				rc = ne_create_vm_ioctl(ne_pci_dev, slot_uid);
+				rc = ne_create_vm_ioctl(ne_pci_dev, 0, slot_uid);
+				mutex_unlock(&ne_pci_dev->enclaves_list_mutex);
+			}
+		}
+
+		up_read(&ne_devs.lock);
+
+		return rc;
+	}
+	}
+
+	/* Extensible Argument ioctls */
+	switch (NE_EA_IOCTL(cmd)) {
+	case NE_EA_IOCTL(NE_CREATE_VM2): {
+		struct ne_create_vm_args args;
+		struct ne_create_vm_args __user *uargs = (void __user *)arg;
+		unsigned int usize = _IOC_SIZE(cmd);
+		struct ne_pci_dev *ne_pci_dev;
+		int rc;
+
+		if (usize < offsetofend(struct ne_create_vm_args, slot_uid))
+			return -EINVAL;
+
+		rc = copy_struct_from_user(&args, sizeof(args), uargs, usize);
+		if (rc)
+			return rc;
+
+		rc = down_read_killable(&ne_devs.lock);
+		if (rc)
+			return rc;
+
+		rc = -ENODEV;
+		ne_pci_dev = ne_devs.ne_pci_dev;
+		if (ne_pci_dev) {
+			rc = mutex_lock_killable(&ne_pci_dev->enclaves_list_mutex);
+			if (!rc) {
+				rc = ne_create_vm_ioctl(ne_pci_dev, args.flags,
+							&uargs->slot_uid);
 				mutex_unlock(&ne_pci_dev->enclaves_list_mutex);
 			}
 		}

@@ -2639,8 +2686,6 @@ static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 	default:
 		return -ENOTTY;
 	}
-
-	return 0;
 }
 
 /*

54 lines not on this page · raw message

index 68c497df4f14b..0bf62f703806e 100644
@@ -2475,13 +2475,30 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 flags,
 	int rc = -EINVAL;
 	struct slot_alloc_req slot_alloc_req = {};
 
-	if (flags) {
+	if (flags & ~(u64)NE_ENCLAVE_PCIE_MODE) {
 		dev_err_ratelimited(ne_misc_dev.this_device,
 				    "Unknown launch flag bit set 0x%llx\n", flags);
 
 		return -NE_ERR_INVALID_FLAG_VALUE;
 	}
 
+	/*
+	 * A device that takes launch flags publishes a vendor-specific
+	 * capability. One without it takes no launch flags and would allocate a
+	 * slot without the properties they ask for.
+	 */
+	if (flags && !pci_find_capability(ne_pci_dev->pdev, PCI_CAP_ID_VNDR)) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Device takes no launch flags\n");
+
+		return -EOPNOTSUPP;
+	}
+
+	if (flags) {
+		slot_alloc_req.payload_version = 1;
+		slot_alloc_req.flags = flags;
+	}
+
 	mutex_lock(&ne_cpu_pool.mutex);
 
 	for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)

index 19e129d14a3a0..b0957e2606ec3 100644
@@ -7,10 +7,12 @@
 #define _NE_PCI_DEV_H_
 
 #include <linux/atomic.h>
+#include <linux/build_bug.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
+#include <linux/stddef.h>
 #include <linux/wait.h>
 
 /**

@@ -174,12 +176,29 @@ struct enclave_stop_req {
 
 /**
  * struct slot_alloc_req - SLOT_ALLOC request.
- * @unused:	In order to avoid weird sizeof edge cases.
+ * @payload_version:	Wire version at offset 0. Zero is the value a driver
+ *			without launch flags leaves here, and tells the device
+ *			to disregard @reserved and @flags. 1 means @flags is
+ *			valid.
+ * @reserved:		Alignment padding between @payload_version and
+ *			@flags. Zero.
+ * @flags:		Launch flag word from NE_CREATE_VM2, valid when
+ *			@payload_version >= 1.
  */
 struct slot_alloc_req {
-	u8	unused;
+	u8	payload_version;
+	u8	reserved[3];
+	u32	flags;
 };
 
+/*
+ * The device picks these fields out of the payload by offset, so a change
+ * to the layout here changes the SLOT_ALLOC wire format.
+ */
+static_assert(sizeof(struct slot_alloc_req) == 8);
+static_assert(offsetof(struct slot_alloc_req, reserved) == 1);
+static_assert(offsetof(struct slot_alloc_req, flags) == 4);
+
 /**
  * struct slot_free_req - SLOT_FREE request.
  * @slot_uid:	Slot unique id mapped to the slot to free.

index 8e43be9a0a1fa..8ccf16e9cf5fb 100644
@@ -46,6 +46,18 @@
  */
 #define NE_CREATE_VM			_IOR(0xAE, 0x20, __u64)
 
+/**
+ * DOC: NE_CREATE_VM2 launch flags
+ */
+
+/**
+ * NE_ENCLAVE_PCIE_MODE - Allocate a slot with a PCIe bus. The
+ *			  hypervisor selects PCIe backing for the slot at
+ *			  SLOT_ALLOC time, so a caller that wants it asks
+ *			  for it here and not at enclave start.
+ */
+#define NE_ENCLAVE_PCIE_MODE		(1u << 6)
+
 /**
  * struct ne_create_vm_args - NE_CREATE_VM2 argument (in / out).
  * @flags:	Launch-time flag word (in). Flags select properties of the

@@ -82,6 +94,8 @@ struct ne_create_vm_args {
  * * EFAULT				- Argument read or write failure.
  * * NE_ERR_INVALID_FLAG_VALUE		- Unknown flag bit set, or
  *					  NE_ENCLAVE_DEBUG_MODE set.
+ * * EOPNOTSUPP			- A launch flag is set and the NE PCI
+ *					  device takes no launch flags.
  * * EINVAL				- The argument stops before @slot_uid.
  * * E2BIG				- The argument carries a field this kernel
  *					  does not know, set to a non-zero value.

index 8ccf16e9cf5fb..6dd00c41c07d5 100644
@@ -567,6 +567,148 @@ struct ne_add_memory_region_args {
  */
 #define NE_ERR_INVALID_ENCLAVE_CID		(275)
 
+/**
+ * DOC: NE error code ranges
+ *
+ * The NE_ERR_ codes come from three ranges:
+ *
+ * * 256 to 275 - The driver's original codes. The NE PCI device also
+ *   returns NE_ERR_MEM_MAX_REGIONS, with the same meaning.
+ * * 276 to 511 - Codes the NE PCI device returns. The driver passes them on
+ *   to the caller unchanged, and the device adds new codes upward from the
+ *   last one defined here. The range ends below the kernel-internal error
+ *   codes, which start at 512.
+ * * 1024 and above - Codes only the driver returns, added upward from 1024.
+ */
+
+/**
+ * NE_ERR_INVALID_FLAGS - The command's reserved flags field was not zero.
+ */
+#define NE_ERR_INVALID_FLAGS			(276)
+/**
+ * NE_ERR_INVALID_ALIGNMENT - An address or a size the command carries is
+ *			      misaligned or too small.
+ */
+#define NE_ERR_INVALID_ALIGNMENT		(277)
+/**
+ * NE_ERR_INVALID_SIZE - A size or count the command carries is zero, too large
+ *			 or too short.
+ */
+#define NE_ERR_INVALID_SIZE			(278)
+/**
+ * NE_ERR_INVALID_SLOT - The slot the command names does not exist.
+ */
+#define NE_ERR_INVALID_SLOT			(279)
+/**
+ * NE_ERR_INVALID_DEVICE - The command names a device, an address or a memory
+ *			   range the device does not accept.
+ */
+#define NE_ERR_INVALID_DEVICE			(280)
+/**
+ * NE_ERR_TOO_MANY_DEVICES - The enclave already has as many devices as it may.
+ */
+#define NE_ERR_TOO_MANY_DEVICES			(281)
+/**
+ * NE_ERR_TOO_MANY_VQS - The device already has as many queues as it may.
+ */
+#define NE_ERR_TOO_MANY_VQS			(282)
+/**
+ * NE_ERR_NO_RESOURCES - The device could not put together something of its own
+ *			 that the command needs.
+ */
+#define NE_ERR_NO_RESOURCES			(283)
+/**
+ * NE_ERR_ALREADY_EXISTS - The slot already has what the command gives it.
+ */
+#define NE_ERR_ALREADY_EXISTS			(284)
+/**
+ * NE_ERR_MISSING_INFO_PAGE - The slot has no info page, or the one it has is too
+ *			      small for what the command writes there.
+ */
+#define NE_ERR_MISSING_INFO_PAGE		(285)
+/**
+ * NE_ERR_INVALID_FLAG - A flag bit the device does not know, or does not allow
+ *			 here, was set.
+ */
+#define NE_ERR_INVALID_FLAG			(286)
+/**
+ * NE_ERR_RESERVED_FLAG - A reserved flag bit was set.
+ */
+#define NE_ERR_RESERVED_FLAG			(287)
+/**
+ * NE_ERR_MISSING_PCIE - A flag that depends on the PCIe flag was set without it.
+ */
+#define NE_ERR_MISSING_PCIE			(288)
+/**
+ * NE_ERR_INCOMPATIBLE_FLAGS - The flags asked for cannot be had together.
+ */
+#define NE_ERR_INCOMPATIBLE_FLAGS		(289)
+/**
+ * NE_ERR_EIF_FLAG_MISMATCH - The image's matched flags are not the ones the
+ *			      enclave was started with.
+ */
+#define NE_ERR_EIF_FLAG_MISMATCH		(290)
+/**
+ * NE_ERR_DEVICE_DISALLOWED - The device type asked for is not allowed.
+ */
+#define NE_ERR_DEVICE_DISALLOWED		(291)
+/**
+ * NE_ERR_HOTPLUG_DISABLED - The enclave does not take devices once it is
+ *			     running.
+ */
+#define NE_ERR_HOTPLUG_DISABLED			(292)
+/**
+ * NE_ERR_BAD_EIF - The enclave image is missing, or did not pass validation when
+ *		    the enclave was started.
+ */
+#define NE_ERR_BAD_EIF				(293)
+/**
+ * NE_ERR_MISSING_PCIE_VIRTIO - A flag that depends on the virtio-PCIe flag was
+ *				set without it.
+ */
+#define NE_ERR_MISSING_PCIE_VIRTIO		(294)
+/**
+ * NE_ERR_BAD_PAYLOAD_VERSION - The request's payload version is one the device
+ *				does not take.
+ */
+#define NE_ERR_BAD_PAYLOAD_VERSION		(295)
+/**
+ * NE_ERR_INFO_VECTORS_EXHAUSTED - No notification vector is left.
+ */
+#define NE_ERR_INFO_VECTORS_EXHAUSTED		(296)
+/**
+ * NE_ERR_TOO_MANY_ENCLAVES - The instance already has as many enclaves as its
+ *			      quota allows. A terminal answer: neither
+ *			      resending the command nor donating more memory
+ *			      changes it.
+ */
+#define NE_ERR_TOO_MANY_ENCLAVES		(297)
+/**
+ * NE_ERR_NO_MEM_SLOTS_ON_NODE - The memory budget of one NUMA node is used up,
+ *				 which -ENOSPC does not distinguish from the
+ *				 budget as a whole.
+ */
+#define NE_ERR_NO_MEM_SLOTS_ON_NODE		(298)
+/**
+ * NE_ERR_NUMA_SPAN - The enclave's CPUs and memory span more than one host NUMA
+ *		      node, and the enclave was not set up to span nodes.
+ */
+#define NE_ERR_NUMA_SPAN			(299)
+/**
+ * NE_ERR_NUMA_ALIGNMENT - The memory of a guest NUMA node other than the last
+ *			   is not a multiple of the guest's memory section size.
+ */
+#define NE_ERR_NUMA_ALIGNMENT			(300)
+/**
+ * NE_ERR_LOW_RAM_TOO_SMALL - The lowest contiguous run of enclave memory is too
+ *			      small for the boot layout the enclave needs.
+ */
+#define NE_ERR_LOW_RAM_TOO_SMALL		(301)
+/**
+ * NE_ERR_ZERO_REGION - The enclave's memory layout has a region of size zero.
+ */
+#define NE_ERR_ZERO_REGION			(302)
+
 /**
  * DOC: Image load info flags
  */

index 2d44cae4ac0cf..f258e04ee9e47 100644
@@ -261,8 +261,18 @@ int ne_do_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
 	if (cmd_reply->rc < 0) {
 		rc = cmd_reply->rc;
 
-		dev_err_ratelimited(&pdev->dev, "Error in cmd process logic, cmd type=%u [rc=%d]\n",
-				    cmd_type, rc);
+		if (rc == -ENOSPC) {
+			dev_info_ratelimited(&pdev->dev,
+					     "No room for cmd type=%u [rc=%d]\n",
+					     cmd_type, rc);
+		} else if (rc == -NE_ERR_TOO_MANY_ENCLAVES) {
+			dev_info_ratelimited(&pdev->dev,
+					     "Enclave quota reached for cmd type=%u [rc=%d]\n",
+					     cmd_type, rc);
+		} else {
+			dev_err_ratelimited(&pdev->dev, "Error in cmd process logic, cmd type=%u [rc=%d]\n",
+					    cmd_type, rc);
+		}
 
 		goto unlock_mutex;
 	}

index 0bf62f703806e..610d98264e5a1 100644
@@ -123,6 +123,8 @@ static int ne_add_memory_region_ioctl(struct ne_enclave *ne_enclave,
 				      struct ne_add_memory_region_args *args);
 static int ne_alloc_memory_ioctl(struct ne_enclave *ne_enclave,
 				 struct ne_alloc_memory_args *args);
+static int ne_set_info_page_ioctl(struct ne_enclave *ne_enclave,
+				  struct ne_set_info_page_args *args);
 static const struct file_operations ne_fops = {
 	.owner		= THIS_MODULE,
 	.llseek		= noop_llseek,

@@ -2137,6 +2139,39 @@ static long ne_enclave_do_ioctl(struct file *file, unsigned int cmd, unsigned lo
 		return rc;
 	}
 
+	case NE_EA_IOCTL(NE_SET_INFO_PAGE): {
+		struct ne_set_info_page_args __user *uargs = (void __user *)arg;
+		unsigned int usize = _IOC_SIZE(cmd);
+		struct ne_set_info_page_args args;
+		int rc;
+
+		if (usize < offsetofend(struct ne_set_info_page_args, flags))
+			return -EINVAL;
+
+		rc = copy_struct_from_user(&args, sizeof(args), uargs, usize);
+		if (rc)
+			return rc;
+
+		rc = mutex_lock_killable(&ne_enclave->enclave_info_mutex);
+		if (rc)
+			return rc;
+
+		if (ne_enclave->state != NE_STATE_INIT) {
+			dev_err_ratelimited(ne_misc_dev.this_device,
+					    "Enclave is not in init state\n");
+
+			mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+			return -NE_ERR_NOT_IN_INIT_STATE;
+		}
+
+		rc = ne_set_info_page_ioctl(ne_enclave, &args);
+
+		mutex_unlock(&ne_enclave->enclave_info_mutex);
+
+		return rc;
+	}
+
 	default:
 		return -ENOTTY;
 	}

@@ -2344,6 +2379,9 @@ static void ne_enclave_free(struct ne_enclave *ne_enclave)
 	ne_enclave_remove_all_mem_region_entries(ne_enclave);
 	ne_enclave_remove_all_vcpu_id_entries(ne_enclave);
 
+	if (ne_enclave->info_file)
+		fput(ne_enclave->info_file);
+
 	kfree(ne_enclave);
 }
 

@@ -3152,6 +3190,9 @@ static bool ne_contig_supports_mmap(struct file *file)
  * asks for VM_HUGEPAGE because a fault on a VM_PFNMAP mapping without it only
  * reaches ->huge_fault where the transparent_hugepage policy is "always".
  *
+ * An allocation that is a slot's info page is mapped without write access, and
+ * without the right to gain it with mprotect().
+ *
  * Return: 0 on success, negative return value on failure.
  */
 static int ne_contig_mmap_prepare(struct vm_area_desc *desc)

@@ -3177,6 +3218,13 @@ static int ne_contig_mmap_prepare(struct vm_area_desc *desc)
 	if (offset > mem->size || size > mem->size - offset)
 		return -EINVAL;
 
+	if (READ_ONCE(mem->read_only)) {
+		if (vma_desc_test(desc, VMA_WRITE_BIT))
+			return -EACCES;
+
+		vma_desc_clear_flags(desc, VMA_MAYWRITE_BIT);
+	}
+
 	desc->vm_ops = &ne_contig_vm_ops;
 	vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
 			   VMA_DONTDUMP_BIT, VMA_DONTCOPY_BIT, VMA_HUGEPAGE_BIT);

@@ -3242,11 +3290,12 @@ static int ne_write_span(void *arg, struct page *page, u64 off, u64 len)
  * refusal is -EFBIG, the same as generic_write_check_limits() gives for a write
  * beyond s_maxbytes. There is no RLIMIT_FSIZE in it, so no SIGXFSZ either.
  *
- * Donating an allocation closes this path, and @lock is what makes the two
- * orderly. It is taken here and in ne_add_memory_region_ioctl() and nowhere
- * else; setting a mapping up and faulting one in both read only fields that
- * never change, so nothing takes it under mmap_write_lock or in a fault, and
- * copying from user space under it cannot invert against the fault path.
+ * Donating an allocation closes this path, and so does making it an info page.
+ * This function, ne_add_memory_region_ioctl(), ne_contig_make_read_only() and
+ * ne_contig_allow_writes() take @lock, which orders those state changes against
+ * a write. Setting a mapping up and faulting one in do not take it, so nothing
+ * takes it under mmap_write_lock or in a fault, and copying from user space
+ * under it cannot invert against the fault path.
  *
  * Return: the number of bytes written, or a negative return value on failure.
  */

@@ -3266,7 +3315,7 @@ static ssize_t ne_contig_write_iter(struct kiocb *iocb, struct iov_iter *from)
 
 	guard(mutex)(&mem->lock);
 
-	if (mem->donated)
+	if (mem->donated || mem->read_only)
 		return -EPERM;
 
 	rc = ne_walk_allocs(mem, pos, count, NE_WRITE_SPAN_PAGES, ne_write_span,

@@ -3281,6 +3330,60 @@ static ssize_t ne_contig_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	return rc;
 }
 
+/**
+ * ne_contig_make_read_only() - Take write access to an allocation away from
+ *				user space.
+ * @file:	The allocation's file.
+ *
+ * From here on a write to the descriptor fails and a new mapping has no write
+ * access, so only the device and the kernel change the memory. The descriptor
+ * is open for writing, so a mapping made before could be made writable with
+ * mprotect(), and one that exists makes this fail.
+ *
+ * mmap_region() raises i_mmap_writable with full ordering before it calls
+ * ne_contig_mmap_prepare() for a mapping that may write, and the mapping keeps
+ * it raised for as long as it exists. @read_only is set before the count is
+ * read, so a mapping that ne_contig_mmap_prepare() set up without seeing it is
+ * counted here. A mapping that is being set up as this runs counts too, even
+ * one that ends up without write access, so a caller sees -EBUSY for it and
+ * can try again.
+ *
+ * Return: 0 on success, -EBUSY if the allocation is mapped.
+ */
+static int ne_contig_make_read_only(struct file *file)
+{
+	struct ne_contig_mem *mem = file->private_data;
+
+	guard(mutex)(&mem->lock);
+
+	WRITE_ONCE(mem->read_only, true);
+
+	/* Pairs with the full ordering of mapping_map_writable(). */
+	smp_mb();
+
+	if (mapping_writably_mapped(file->f_mapping)) {
+		WRITE_ONCE(mem->read_only, false);
+
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+/**
+ * ne_contig_allow_writes() - Give write access to an allocation back to user
+ *			      space.
+ * @file:	The allocation's file.
+ */
+static void ne_contig_allow_writes(struct file *file)
+{
+	struct ne_contig_mem *mem = file->private_data;
+
+	guard(mutex)(&mem->lock);
+
+	WRITE_ONCE(mem->read_only, false);
+}
+
 /**
  * ne_contig_release() - Free an allocation once nothing holds it any more.
  * @inode:	The allocation's inode, unused.

@@ -3612,6 +3715,13 @@ static int ne_add_memory_region_ioctl(struct ne_enclave *ne_enclave,
 		return -EINVAL;
 	}
 
+	if (fd_file(f) == ne_enclave->info_file) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The allocation is the enclave's info page\n");
+
+		return -NE_ERR_MEM_REGION_ALREADY_USED;
+	}
+
 	list_for_each_entry(held, &ne_enclave->contig_regions_list, list) {
 		if (held->file == fd_file(f)) {
 			dev_err_ratelimited(ne_misc_dev.this_device,

@@ -3714,6 +3824,160 @@ static int ne_add_memory_region_ioctl(struct ne_enclave *ne_enclave,
 	return rc;
 }
 
+/**
+ * ne_set_info_page_ioctl() - Give the slot associated with the current enclave
+ *			      the info page.
+ * @ne_enclave:	Private data associated with the current enclave.
+ * @args:	Descriptor for the allocation, and flags.
+ *
+ * The device writes the info page, so the run behind it has to stay where the
+ * device was told it is for as long as the enclave lives. The runs belong to the
+ * file, so a reference on it keeps them whatever user space does with the
+ * descriptor and its mappings, and ne_enclave_free() drops that reference
+ * after SLOT_FREE and after the enclave's memory regions. The reference is taken
+ * before the request, since a request that timed out may still be completed by
+ * the device. A rejected request drops it again.
+ *
+ * Only the device writes the page, and the parent reads back from it what the
+ * device reports about the slot, so user space loses write access to the
+ * allocation before the device is told about it and gets it back only if the
+ * device rejects the request.
+ *
+ * Context: Process context. This function is called with the ne_enclave mutex held.
+ * Return:
+ * * 0 on success.
+ * * Negative return value on failure.
+ * * -ETIMEDOUT if the device did not answer, in which case the slot keeps the
+ *   allocation until the enclave goes away.
+ */
+static int ne_set_info_page_ioctl(struct ne_enclave *ne_enclave,
+				  struct ne_set_info_page_args *args)
+{
+	struct ne_pci_dev_cmd_reply cmd_reply = {};
+	struct slot_add_info_page_req req = {};
+	struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev;
+	struct ne_contig_mem *mem = NULL;
+	int rc = -EINVAL;
+
+	if (ne_enclave->mm != current->mm)
+		return -EIO;
+
+	if (args->flags) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Incorrect flag for info page\n");
+
+		return -NE_ERR_INVALID_FLAG_VALUE;
+	}
+
+	if (ne_enclave->info_file) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The slot already has an info page\n");
+
+		return -EEXIST;
+	}
+
+	CLASS(fd, f)(args->fd);
+
+	if (fd_empty(f))
+		return -EBADF;
+
+	if (fd_file(f)->f_op != &ne_contig_fops) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Not an enclave memory allocation\n");
+
+		return -EINVAL;
+	}
+
+	mem = fd_file(f)->private_data;
+
+	if (mem->enclave_id != ne_enclave->id) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Allocation was made for another enclave\n");
+
+		return -EINVAL;
+	}
+
+	if (!mem->shared) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The info page allocation is not shared\n");
+
+		return -EINVAL;
+	}
+
+	/*
+	 * The device takes one address and one length, so an allocation the
+	 * allocator had to satisfy in several runs has no single address to
+	 * name. A request of NE_MIN_MEM_REGION_SIZE cannot be split, since the
+	 * allocator has nothing smaller to fall back to.
+	 */
+	if (mem->nr_allocs != 1) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The info page allocation is not one run\n");
+
+		return -EINVAL;
+	}
+
+	/*
+	 * An allocation the enclave already has as memory is not available to
+	 * be its info page as well, since the device would be handed the same
+	 * run twice in two roles. @donated is only ever set from
+	 * ne_add_memory_region_ioctl(), which runs under the same enclave mutex
+	 * as this function for the one enclave the allocation was made for, so
+	 * the read needs no lock of its own.
+	 */
+	if (mem->donated) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The info page allocation was donated\n");
+
+		return -NE_ERR_MEM_REGION_ALREADY_USED;
+	}
+
+	rc = ne_contig_make_read_only(fd_file(f));
+	if (rc < 0) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "The info page allocation is mapped\n");
+
+		return rc;
+	}
+
+	req.slot_uid = ne_enclave->slot_uid;
+	req.info_phys = page_to_phys(mem->allocs[0].page);
+	req.info_size = mem->size;
+
+	/*
+	 * Take the reference before the request, so that the run stays where the
+	 * device is told it is from the moment the device is told, as the memory
+	 * region path does.
+	 */
+	ne_enclave->info_file = get_file(fd_file(f));
+
+	rc = ne_do_request(pdev, SLOT_ADD_INFO_PAGE,
+			   &req, sizeof(req), &cmd_reply, sizeof(cmd_reply));
+	if (rc < 0) {
+		dev_err_ratelimited(ne_misc_dev.this_device,
+				    "Error in slot add info page [rc=%d]\n", rc);
+
+		/*
+		 * A request the device never answered was submitted with the
+		 * run's physical address in it, so the device may write the page
+		 * yet. Keep the reference and let ne_enclave_free() drop it
+		 * after SLOT_FREE. Every other error means the device either
+		 * never saw the request or answered to reject it, and the run is
+		 * the parent's again. ne_request_unanswered() tells the two
+		 * apart.
+		 */
+		if (!ne_request_unanswered(rc)) {
+			ne_contig_allow_writes(ne_enclave->info_file);
+			fput(ne_enclave->info_file);
+			ne_enclave->info_file = NULL;
+		}
+
+		return rc;
+	}
+
+	return 0;
+}
+
 #if defined(CONFIG_NITRO_ENCLAVES_MISC_DEV_TEST)
 #include "ne_misc_dev_test.c"
 #endif

index 2502cca3f3f64..d0d1973bfbe9d 100644
@@ -38,27 +38,37 @@ struct ne_contig_alloc {
  * @allocs:	Contiguous runs backing the allocation, ordered by offset.
  * @nr_allocs:	Number of runs in @allocs.
  * @size:	Size of the allocation in bytes, mirrored in i_size.
- * @shared:	Whether the allocation may be mapped by the caller.
+ * @shared:	Whether the caller asked for a shared allocation. A shared one may
+ *		be mapped by the caller and may be a slot's info page; a private
+ *		one is neither.
  * @enclave_id:	Id of the enclave the allocation was made for.
  * @mm:		The mm charged for the pinned pages, held via mmgrab().
  * @nr_pinned:	Pages charged to @mm->pinned_vm.
- * @lock:	Serialises a write against the donation that ends writing.
+ * @lock:	Serialises a write against the donation or the info page that
+ *		ends writing.
  * @donated:	Whether the allocation has been given to its enclave.
+ * @read_only:	Whether user space has lost write access to the allocation,
+ *		which it does when the allocation becomes a slot's info page.
  *
  * The runs hang off the file, so that closing the last descriptor does not free
  * memory an enclave is still using: a donated region holds the file with
- * get_file(). That is also what keeps a mapping's page table entries from
- * outliving the runs, since a VMA holds vm_file for as long as it exists.
+ * get_file(), and so does the enclave the allocation is the info page of. That
+ * is also what keeps a mapping's page table entries from outliving the runs,
+ * since a VMA holds vm_file for as long as it exists.
  *
  * The array is built before the file is published and not mutated afterwards, so
  * any holder of a reference can read it without a lock of its own, which is what
  * lets a page fault resolve an offset to a run. @shared and @size are set in the
- * same place and are read the same way. @donated is the one field that changes
- * after publication, and @lock covers it.
+ * same place and are read the same way. @donated and @read_only are the fields
+ * that change after publication. @lock covers both on the write path;
+ * ne_set_info_page_ioctl() reads @donated under the enclave mutex its writer also
+ * holds, and ne_contig_mmap_prepare() reads @read_only without a lock, which
+ * ne_contig_make_read_only() orders against.
  *
  * @enclave_id is compared, never dereferenced, and the counter it comes from is
- * 64 bits wide, so an allocation cannot be donated to a later enclave that
- * happens to have been placed at the address of the one it was made for.
+ * 64 bits wide, so an allocation cannot be donated to, or become the info page
+ * of, a later enclave that happens to have been placed at the address of the one
+ * it was made for.
  */
 struct ne_contig_mem {
 	struct ne_contig_alloc	*allocs;

@@ -68,9 +78,10 @@ struct ne_contig_mem {
 	u64			enclave_id;
 	struct mm_struct	*mm;
 	unsigned long		nr_pinned;
-	/* Serialises a write against the donation that ends writing. */
+	/* Serialises a write against what ends writing. */
 	struct mutex		lock;
 	bool			donated;
+	bool			read_only;
 };
 
 /**

@@ -116,6 +127,11 @@ struct ne_mem_region {
  * @id:				Id of this enclave, unique for the lifetime of the
  *				driver, recorded in an %NE_ALLOC_MEMORY allocation
  *				as the enclave it was made for.
+ * @info_file:			File of the %NE_ALLOC_MEMORY allocation the slot
+ *				was given as its info page, held with get_file()
+ *				so that the device keeps the page after user space
+ *				has closed the descriptor. NULL until
+ *				%NE_SET_INFO_PAGE is called.
  * @max_mem_regions:		The maximum number of memory regions that can be
  *				handled by the hypervisor.
  * @mem_regions_list:		Enclave user space memory regions list.

@@ -146,6 +162,7 @@ struct ne_enclave {
 	wait_queue_head_t	eventq;
 	bool			has_event;
 	u64			id;
+	struct file		*info_file;
 	u64			max_mem_regions;
 	struct list_head	mem_regions_list;
 	u64			mem_size;

index f258e04ee9e47..a07dbc6d8a9b7 100644
@@ -128,6 +128,31 @@ static int ne_wait_for_reply(struct pci_dev *pdev, unsigned int timeout_msecs)
 	return 0;
 }
 
+/**
+ * ne_request_unanswered() - Whether a failure left a submitted request in
+ *			     flight.
+ * @rc:		Failure a command request came back with.
+ *
+ * A request the device never answered was submitted with whatever the caller
+ * put in it, and there is nothing to cancel it with, so a resource whose
+ * address is in it may still be written by the device. A caller that has to
+ * decide who owns such a resource asks here instead of naming an errno of its
+ * own.
+ *
+ * Every code the wait for a reply can fail a submitted request with belongs
+ * here. ne_wait_for_reply() times out, so -ETIMEDOUT is named. A code raised
+ * before ne_submit_request(), and a code the device answered with, do not:
+ * nothing is in flight in either case. A
+ * device that answers one of the codes named here of its own accord lands here
+ * too and costs one held resource, which is the side to be wrong on.
+ *
+ * Return: true if the request may still be in flight.
+ */
+bool ne_request_unanswered(int rc)
+{
+	return rc == -ETIMEDOUT;
+}
+
 /**
  * ne_cmd_frees_slot() - Whether a command is one of the two that free a slot.
  * @cmd_type:	Command type of the request.

index b0957e2606ec3..4f670c53dd3c0 100644
@@ -123,6 +123,8 @@
  * @NEXT_SLOT:			Get the next slot in the list of allocated slots.
  * @SLOT_INFO:			Get the info for a slot e.g. slot uid, vCPUs count.
  * @SLOT_ADD_BULK_VCPUS:	Add a number of vCPUs, not providing CPU ids.
+ * @SLOT_ADD_INFO_PAGE:		Give a slot the info page, a memory region the
+ *				device writes.
  * @MAX_CMD:			A gatekeeper for max possible command type.
  */
 enum ne_pci_dev_cmd_type {

@@ -138,6 +140,8 @@ enum ne_pci_dev_cmd_type {
 	NEXT_SLOT		= 9,
 	SLOT_INFO		= 10,
 	SLOT_ADD_BULK_VCPUS	= 11,
+	/* 12 is a command this driver does not implement. */
+	SLOT_ADD_INFO_PAGE	= 13,
 	MAX_CMD,
 };
 

@@ -266,6 +270,20 @@ struct slot_add_bulk_vcpus_req {
 	u64	nr_vcpus;
 };
 
+/**
+ * struct slot_add_info_page_req - SLOT_ADD_INFO_PAGE request.
+ * @slot_uid:	Slot unique id mapped to the slot to give the info page to.
+ * @flags:	Flags for the info page. Zero is the only value the driver sends.
+ * @info_phys:	Physical address of the info page.
+ * @info_size:	Size, in bytes, of the info page.
+ */
+struct slot_add_info_page_req {
+	u64	slot_uid;
+	u64	flags;
+	u64	info_phys;
+	u64	info_size;
+};
+
 /**
  * struct ne_pci_dev_cmd_reply - NE PCI device command reply.
  * @rc :		Return code of the logic that processed the request.

@@ -327,6 +345,9 @@ struct ne_pci_dev {
 	struct pci_dev		*pdev;
 };
 
+/* Whether a failure left a submitted request in flight. */
+bool ne_request_unanswered(int rc);
+
 /**
  * ne_do_request() - Submit command request to the PCI device based on the command
  *		     type and retrieve the associated reply.

162 lines not on this page · raw message

index 610d98264e5a1..77b15eda8d120 100644
@@ -1749,7 +1749,7 @@ static int ne_start_enclave_ioctl(struct ne_enclave *ne_enclave,
 
 	ne_enclave->state = NE_STATE_RUNNING;
 
-	enclave_start_info->enclave_cid = cmd_reply.enclave_cid;
+	enclave_start_info->enclave_cid = cmd_reply.start.enclave_cid;
 
 	return 0;
 }

@@ -2621,11 +2621,11 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 flags,
 	init_waitqueue_head(&ne_enclave->eventq);
 	ne_enclave->has_event = false;
 	mutex_init(&ne_enclave->enclave_info_mutex);
-	ne_enclave->max_mem_regions = cmd_reply.mem_regions;
+	ne_enclave->max_mem_regions = cmd_reply.alloc.mem_regions;
 	INIT_LIST_HEAD(&ne_enclave->contig_regions_list);
 	INIT_LIST_HEAD(&ne_enclave->mem_regions_list);
 	ne_enclave->mm = current->mm;
-	ne_enclave->slot_uid = cmd_reply.slot_uid;
+	ne_enclave->slot_uid = cmd_reply.alloc.slot_uid;
 	ne_enclave->state = NE_STATE_INIT;
 
 	list_add(&ne_enclave->enclave_list_entry, &ne_pci_dev->enclaves_list);

index a07dbc6d8a9b7..8ed2349dea878 100644
@@ -404,8 +404,8 @@ static void ne_event_work_handler(struct work_struct *work)
 		}
 
 		/* Notify enclave process that the enclave state changed. */
-		if (ne_enclave->state != cmd_reply.state) {
-			ne_enclave->state = cmd_reply.state;
+		if (ne_enclave->state != cmd_reply.info.state) {
+			ne_enclave->state = cmd_reply.info.state;
 
 			ne_enclave->has_event = true;
 

98 lines not on this page · raw message

index 835413fca8040..e651e74ab656e 100644
@@ -95,6 +95,42 @@
  */
 #define NE_RECV_DATA_SIZE	(240)
 
+/**
+ * DOC: Slot pool registers in the NE PCI device MMIO BAR
+ */
+
+/**
+ * NE_SLOT_POOL_MAX - (2 bytes) Register reporting the highest number of
+ *		      slots the device lets the pool grow to, or zero on a
+ *		      device that needs no donation (Read-Only).
+ */
+#define NE_SLOT_POOL_MAX	(0x01f2)
+
+/**
+ * NE_SLOT_POOL_LIMIT - (2 bytes) Register reporting how many slots the
+ *			pool has room for with the pages donated so far
+ *			(Read-Only).
+ */
+#define NE_SLOT_POOL_LIMIT	(0x01f6)
+
+/**
+ * NE_SLOT_POOL_GRAN_SIZE - (4 bytes) Register reporting the size, in bytes,
+ *			    of one donation to the pool (Read-Only).
+ */
+#define NE_SLOT_POOL_GRAN_SIZE	(0x0200)
+
+/**
+ * NE_SLOT_POOL_GRAN_ALIGN - (4 bytes) Register reporting the alignment a
+ *			     donation to the pool has to meet (Read-Only).
+ */
+#define NE_SLOT_POOL_GRAN_ALIGN	(0x0204)
+
+/**
+ * NE_SLOT_POOL_BASE_SIZE - (4 bytes) Register reporting the size, in bytes,
+ *			    of the base donation (Read-Only).
+ */
+#define NE_SLOT_POOL_BASE_SIZE	(0x0208)
+
 /**
  * DOC: MSI-X interrupt vectors
  */

@@ -125,6 +161,10 @@
  * @SLOT_ADD_BULK_VCPUS:	Add a number of vCPUs, not providing CPU ids.
  * @SLOT_ADD_INFO_PAGE:		Give a slot the info page, a memory region the
  *				device writes.
+ * @SLOT_POOL_DONATE:		Donate one page of parent memory to the slot
+ *				pool.
+ * @SLOT_POOL_SHUTDOWN:		Give the whole slot pool back.
+ * @SLOT_POOL_BASE_DONATE:	Donate the base run of the slot pool.
  * @MAX_CMD:			A gatekeeper for max possible command type.
  */
 enum ne_pci_dev_cmd_type {

@@ -142,6 +182,9 @@ enum ne_pci_dev_cmd_type {
 	SLOT_ADD_BULK_VCPUS	= 11,
 	/* 12 is a command this driver does not implement. */
 	SLOT_ADD_INFO_PAGE	= 13,
+	SLOT_POOL_DONATE	= 16,
+	SLOT_POOL_SHUTDOWN	= 17,
+	SLOT_POOL_BASE_DONATE	= 25,
 	MAX_CMD,
 };
 

@@ -284,6 +327,23 @@ struct slot_add_info_page_req {
 	u64	info_size;
 };
 
+/**
+ * struct slot_pool_donate_req - SLOT_POOL_DONATE and SLOT_POOL_BASE_DONATE
+ *				 request.
+ * @paddr:	Physical address of the donated run of pages.
+ */
+struct slot_pool_donate_req {
+	u64	paddr;
+};
+
+/**
+ * struct slot_pool_shutdown_req - SLOT_POOL_SHUTDOWN request.
+ * @unused:	In order to avoid weird sizeof edge cases.
+ */
+struct slot_pool_shutdown_req {
+	u8	unused;
+};
+
 /**
  * struct ne_pci_dev_cmd_reply - NE PCI device command reply.
  *

index 77b15eda8d120..2fae731acb977 100644
@@ -201,6 +201,11 @@ static const struct kernel_param *ne_cpus_kp;
  *				and leaves when the enclave is freed, in
  *				the same two places that move the core out of
  *				and back into @avail_threads_per_core.
+ * @nr_enclaves:			How many enclaves were created from the pool and
+ *				have not been freed yet, whether or not they are
+ *				on the list of a device. The pool is not replaced
+ *				while any is, so the cores an enclave gives back
+ *				go back to the pool they came from.
  */
 struct ne_cpu_pool {
 	cpumask_var_t	*avail_threads_per_core;

@@ -211,6 +216,7 @@ struct ne_cpu_pool {
 	cpumask_var_t	*node_cpus;
 	struct cpumask	total_cpus;
 	struct cpumask	used_cpus;
+	unsigned int	nr_enclaves;
 };
 
 static struct ne_cpu_pool ne_cpu_pool = {

@@ -931,6 +937,7 @@ static int ne_replace_cpu_pool(const char *val, const struct kernel_param *kp)
 	cpumask_var_t cpu_pool;
 	cpumask_var_t old_pool;
 	struct ne_pci_dev *ne_pci_dev = NULL;
+	unsigned int nr_enclaves = 0;
 	int rc = -EINVAL;
 
 	if (!zalloc_cpumask_var(&cpu_pool, GFP_KERNEL))

@@ -961,7 +968,11 @@ static int ne_replace_cpu_pool(const char *val, const struct kernel_param *kp)
 			goto unlock_devs;
 	}
 
-	if (ne_pci_dev && !list_empty(&ne_pci_dev->enclaves_list)) {
+	mutex_lock(&ne_cpu_pool.mutex);
+	nr_enclaves = ne_cpu_pool.nr_enclaves;
+	mutex_unlock(&ne_cpu_pool.mutex);
+
+	if (nr_enclaves) {
 		pr_err("%s: The CPU pool is used by enclave(s)\n", ne_misc_dev.name);
 
 		rc = -EPERM;

@@ -2293,6 +2304,8 @@ static void ne_enclave_remove_all_vcpu_id_entries(struct ne_enclave *ne_enclave)
 	for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++)
 		free_cpumask_var(ne_enclave->threads_per_core[i]);
 
+	ne_cpu_pool.nr_enclaves--;
+
 	mutex_unlock(&ne_cpu_pool.mutex);
 
 	kfree(ne_enclave->threads_per_core);

@@ -2337,8 +2350,8 @@ static void ne_pci_dev_remove_enclave_entry(struct ne_enclave *ne_enclave,
  * * 0 once the device has freed the slot.
  * * Negative return value on failure.
  */
-static int ne_enclave_free_slot(struct ne_pci_dev *ne_pci_dev,
-				struct ne_enclave *ne_enclave)
+int ne_enclave_free_slot(struct ne_pci_dev *ne_pci_dev,
+			 struct ne_enclave *ne_enclave)
 {
 	struct ne_pci_dev_cmd_reply cmd_reply = {};
 	struct enclave_stop_req enclave_stop_request = {};

@@ -2346,17 +2359,20 @@ static int ne_enclave_free_slot(struct ne_pci_dev *ne_pci_dev,
 	struct pci_dev *pdev = ne_pci_dev->pdev;
 	int rc;
 
-	if (ne_enclave->state != NE_STATE_INIT && ne_enclave->state != NE_STATE_STOPPED) {
-		enclave_stop_request.slot_uid = ne_enclave->slot_uid;
+	/*
+	 * An enclave that exited may still be scrubbed, so the stop goes out
+	 * whatever the state, and the device answers it once the slot is
+	 * neither running nor being scrubbed.
+	 */
+	enclave_stop_request.slot_uid = ne_enclave->slot_uid;
 
-		rc = ne_do_request(pdev, ENCLAVE_STOP,
-				   &enclave_stop_request, sizeof(enclave_stop_request),
-				   &cmd_reply, sizeof(cmd_reply));
-		if (rc < 0)
-			return rc;
+	rc = ne_do_request(pdev, ENCLAVE_STOP,
+			   &enclave_stop_request, sizeof(enclave_stop_request),
+			   &cmd_reply, sizeof(cmd_reply));
+	if (rc < 0)
+		return rc;
 
-		memset(&cmd_reply, 0, sizeof(cmd_reply));
-	}
+	memset(&cmd_reply, 0, sizeof(cmd_reply));
 
 	slot_free_req.slot_uid = ne_enclave->slot_uid;
 

@@ -2416,10 +2432,13 @@ static int ne_enclave_release(struct inode *inode, struct file *file)
 	/*
 	 * The device the enclave was created on is gone, and with it the only
 	 * party that can confirm that the slot no longer uses its memory and
-	 * its CPUs, so the enclave keeps both.
+	 * its CPUs, so the enclave keeps both, unless the device freed the
+	 * slot before it went away.
 	 */
 	if (ne_enclave->detached) {
-		rc = -ENODEV;
+		rc = ne_enclave->slot_freed ? 0 : -ENODEV;
+		if (!rc)
+			ne_enclave_free(ne_enclave);
 
 		goto unlock_devs;
 	}

@@ -2525,7 +2544,7 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 flags,
 	 * capability. One without it takes no launch flags and would allocate a
 	 * slot without the properties they ask for.
 	 */
-	if (flags && !pci_find_capability(ne_pci_dev->pdev, PCI_CAP_ID_VNDR)) {
+	if (flags && !ne_pci_dev->has_vendor_cap) {
 		dev_err_ratelimited(ne_misc_dev.this_device,
 				    "Device takes no launch flags\n");
 

@@ -2630,6 +2649,10 @@ static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 flags,
 
 	list_add(&ne_enclave->enclave_list_entry, &ne_pci_dev->enclaves_list);
 
+	mutex_lock(&ne_cpu_pool.mutex);
+	ne_cpu_pool.nr_enclaves++;
+	mutex_unlock(&ne_cpu_pool.mutex);
+
 	if (copy_to_user(slot_uid, &ne_enclave->slot_uid, sizeof(ne_enclave->slot_uid))) {
 		/*
 		 * As we're holding the only reference to 'enclave_file', fput()

index d0d1973bfbe9d..59374a35c3c7d 100644
@@ -116,6 +116,9 @@ struct ne_mem_region {
  * @enclave_info_mutex :	Mutex for accessing this internal state.
  * @detached:			The NE PCI device the enclave was created on is
  *				gone. Written with ne_devs.lock held for writing.
+ * @slot_freed:			The device freed the slot of a detached enclave
+ *				before it went away. Written with ne_devs.lock
+ *				held for writing.
  * @contig_regions_list:	List of %NE_ALLOC_MEMORY allocations given to the
  *				enclave with %NE_ADD_MEMORY_REGION.
  * @enclave_list_entry :	Entry in the list of created enclaves.

@@ -157,6 +160,7 @@ struct ne_mem_region {
 struct ne_enclave {
 	struct mutex		enclave_info_mutex;
 	bool			detached;
+	bool			slot_freed;
 	struct list_head	contig_regions_list;
 	struct list_head	enclave_list_entry;
 	wait_queue_head_t	eventq;

@@ -221,6 +225,9 @@ void ne_contig_free_run(struct cma *cma, struct page *page,
 /* Nitro Enclaves (NE) data structure for keeping refs to the NE misc and PCI devices. */
 extern struct ne_devs ne_devs;
 
+int ne_enclave_free_slot(struct ne_pci_dev *ne_pci_dev,
+			 struct ne_enclave *ne_enclave);
+
 int ne_cpu_pool_sysfs_init(void);
 void ne_cpu_pool_sysfs_exit(void);
 

403 lines not on this page · raw message

index e651e74ab656e..4e991c4b0e70f 100644
@@ -433,10 +433,27 @@ static_assert(offsetof(struct ne_pci_dev_cmd_reply, info.state) == 64);
  *				triggered by the Nitro Hypervisor which require
  *				enclave state scanning and propagation to the
  *				enclave process.
+ * @has_vendor_cap:		Whether the PCI device publishes a vendor-specific
+ *				capability, and with it the registers past the
+ *				command interface.
  * @iomem_base :		MMIO region of the PCI device.
  * @notify_work:		Work item for every received out-of-band event.
  * @pci_dev_mutex:		Mutex for accessing the PCI device MMIO space.
  * @pdev:			PCI device data structure.
+ * @slot_pool_max:		Highest number of slots the device lets the slot
+ *				pool grow to, zero on a device that takes no
+ *				donation.
+ * @slot_pool_gran:		Size, in bytes, of one donation to the slot pool.
+ *				Written once at probe, before any donation is
+ *				made, so a later reader of it sees the size the
+ *				donation was allocated with.
+ * @slot_pool_base_page:	First page of the base donation, NULL until it
+ *				has been made.
+ * @slot_pool_base_cma:		CMA area the base donation came from.
+ * @slot_pool_base_nr_pages:	Number of pages in the base donation.
+ * @slot_pool_page:		First page of the donation to the slot pool, NULL
+ *				until it has been made.
+ * @slot_pool_cma:		CMA area the donation came from.
  */
 struct ne_pci_dev {
 	atomic_t		cmd_reply_avail;

@@ -445,10 +462,18 @@ struct ne_pci_dev {
 	struct list_head	enclaves_list;
 	struct mutex		enclaves_list_mutex;
 	struct workqueue_struct	*event_wq;
+	bool			has_vendor_cap;
 	void __iomem		*iomem_base;
 	struct work_struct	notify_work;
 	struct mutex		pci_dev_mutex;
 	struct pci_dev		*pdev;
+	u16			slot_pool_max;
+	u32			slot_pool_gran;
+	struct page		*slot_pool_base_page;
+	struct cma		*slot_pool_base_cma;
+	unsigned long		slot_pool_base_nr_pages;
+	struct page		*slot_pool_page;
+	struct cma		*slot_pool_cma;
 };
 
 /* Whether a failure left a submitted request in flight. */

index 4e991c4b0e70f..3718f893e4f65 100644
@@ -164,6 +164,10 @@
  * @SLOT_POOL_DONATE:		Donate one page of parent memory to the slot
  *				pool.
  * @SLOT_POOL_SHUTDOWN:		Give the whole slot pool back.
+ * @SLOT_POOL_MEM_DONATE:	Donate a run of parent memory to the memory the
+ *				device runs a slot out of.
+ * @SLOT_POOL_GET_LIMITS:	Report how many slots each half of the pool has
+ *				room for.
  * @SLOT_POOL_BASE_DONATE:	Donate the base run of the slot pool.
  * @MAX_CMD:			A gatekeeper for max possible command type.
  */

@@ -185,6 +189,8 @@ enum ne_pci_dev_cmd_type {
 	SLOT_POOL_DONATE	= 16,
 	SLOT_POOL_SHUTDOWN	= 17,
 	SLOT_POOL_BASE_DONATE	= 25,
+	SLOT_POOL_MEM_DONATE	= 26,
+	SLOT_POOL_GET_LIMITS	= 27,
 	MAX_CMD,
 };
 

@@ -344,6 +350,16 @@ struct slot_pool_shutdown_req {
 	u8	unused;
 };
 
+/**
+ * struct slot_pool_mem_donate_req - SLOT_POOL_MEM_DONATE request.
+ * @paddr:	Physical address of the donated run of memory.
+ * @size:	Size, in bytes, of the donated run.
+ */
+struct slot_pool_mem_donate_req {
+	u64	paddr;
+	u64	size;
+};
+
 /**
  * struct ne_pci_dev_cmd_reply - NE PCI device command reply.
  *

@@ -377,6 +393,21 @@ struct slot_pool_shutdown_req {
  * @info.state:			SLOT_INFO. Slot state.
  * @info.padding1:		SLOT_INFO. Padding for the overall data
  *				structure.
+ * @mem_donate:			Fields SLOT_POOL_MEM_DONATE fills.
+ * @mem_donate.consumed_paddr:	SLOT_POOL_MEM_DONATE. Physical address of the
+ *				run the device took out of the donation.
+ * @mem_donate.consumed_size:	SLOT_POOL_MEM_DONATE. Size, in bytes, of the
+ *				run the device took, zero where it took none.
+ * @limits:			Fields SLOT_POOL_GET_LIMITS fills.
+ * @limits.meta_limit:		SLOT_POOL_GET_LIMITS. Slots the descriptors
+ *				donated so far have room for.
+ * @limits.mem_limit:		SLOT_POOL_GET_LIMITS. Slots the memory donated
+ *				so far has room to run.
+ * @limits.slots_per_donation:	SLOT_POOL_GET_LIMITS. Slots one donation of
+ *				memory adds to the memory the device runs a
+ *				slot out of.
+ * @limits.reserved14:		SLOT_POOL_GET_LIMITS. Bytes the device writes as
+ *				zero.
  */
 struct ne_pci_dev_cmd_reply {
 	s32	rc;

@@ -402,6 +433,16 @@ struct ne_pci_dev_cmd_reply {
 			u16	state;
 			u8	padding1[6];
 		} info;
+		struct {
+			u64	consumed_paddr;
+			u64	consumed_size;
+		} mem_donate;
+		struct {
+			u16	meta_limit;
+			u16	mem_limit;
+			u16	slots_per_donation;
+			u16	reserved14;
+		} limits;
 	};
 };
 

@@ -417,6 +458,15 @@ static_assert(offsetof(struct ne_pci_dev_cmd_reply, info.mem_size) == 40);
 static_assert(offsetof(struct ne_pci_dev_cmd_reply, info.nr_vcpus) == 48);
 static_assert(offsetof(struct ne_pci_dev_cmd_reply, info.flags) == 56);
 static_assert(offsetof(struct ne_pci_dev_cmd_reply, info.state) == 64);
+static_assert(offsetof(struct ne_pci_dev_cmd_reply,
+		       mem_donate.consumed_paddr) == 8);
+static_assert(offsetof(struct ne_pci_dev_cmd_reply,
+		       mem_donate.consumed_size) == 16);
+static_assert(offsetof(struct ne_pci_dev_cmd_reply, limits.meta_limit) == 8);
+static_assert(offsetof(struct ne_pci_dev_cmd_reply, limits.mem_limit) == 10);
+static_assert(offsetof(struct ne_pci_dev_cmd_reply,
+		       limits.slots_per_donation) == 12);
+static_assert(offsetof(struct ne_pci_dev_cmd_reply, limits.reserved14) == 14);
 
 /**
  * struct ne_pci_dev - Nitro Enclaves (NE) PCI device.

index e42be1742a39c..9e304ffc2bd13 100644
@@ -18,8 +18,10 @@
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/nitro_enclaves.h>
+#include <linux/nodemask.h>
 #include <linux/pci.h>
 #include <linux/sizes.h>
+#include <linux/slab.h>
 #include <linux/sysfs.h>
 #include <linux/types.h>
 #include <linux/wait.h>

@@ -692,6 +694,61 @@ static void ne_slot_pool_read_geometry(struct pci_dev *pdev)
 	ne_pci_dev->slot_pool_gran = size;
 }
 
+/**
+ * ne_slot_pool_alloc_run() - Allocate one run for the slot pool.
+ * @nr_pages:	Number of pages in the run.
+ * @pref_nid:	NUMA node to try first.
+ * @out_cma:	Set to the CMA area the run came from.
+ *
+ * ne_alloc_contig() confines a run to the node it is given, which is what an
+ * enclave needs. A run in the slot pool is read and written by the device
+ * alone, so it can sit on any node that has memory, and a pool that stops
+ * growing because one node ran out is worth less than a pool spread over
+ * several. So walk the nodes that have memory, starting at @pref_nid, and take
+ * the first run any of them serves.
+ *
+ * A caller runs on a node that need not have memory of its own, and there is no
+ * run to be had from such a node, so a @pref_nid outside the set of nodes that
+ * have memory starts the walk at the first node that does. The walk then covers
+ * every node that can serve it.
+ *
+ * Context: Process context.
+ * Return: the first page of the run, NULL when no node served one, or the
+ *	   ERR_PTR() ne_alloc_contig() returned.
+ */
+static struct page *ne_slot_pool_alloc_run(unsigned long nr_pages,
+					   int pref_nid, struct cma **out_cma)
+{
+	int nid = pref_nid;
+	struct page *page;
+	int nr_nodes;
+	int i;
+
+	nr_nodes = num_node_state(N_MEMORY);
+	if (nr_nodes < 1)
+		return NULL;
+
+	if (pref_nid < 0 || pref_nid >= MAX_NUMNODES ||
+	    !node_state(pref_nid, N_MEMORY))
+		nid = first_node(node_states[N_MEMORY]);
+
+	for (i = 0; i < nr_nodes; i++) {
+		page = ne_alloc_contig(nr_pages, nid, out_cma);
+
+		/*
+		 * An error ends the walk along with a run: only a node that had
+		 * no run to give leaves NULL behind, and only then does the next
+		 * node get asked.
+		 */
+		if (page)
+			return page;
+
+		nid = next_node_in(nid, node_states[N_MEMORY]);
+	}
+
+	return NULL;
+}
+
 /**
  * ne_slot_pool_donate_base() - Donate the base run of the slot pool.
  * @pdev:	PCI device to donate to.

@@ -708,6 +765,7 @@ static void ne_slot_pool_read_geometry(struct pci_dev *pdev)
  *	    mutex held.
  * Return:
  * * 0 on success or if the base run has already been donated.
+ * * -EINTR if a fatal signal arrived while a run was being zeroed.
  * * Negative return value on failure.
  * * -ETIMEDOUT, or any other code ne_request_unanswered() names, if the
  *   request went unanswered, in which case the run stays recorded as donated

@@ -737,13 +795,13 @@ static int ne_slot_pool_donate_base(struct pci_dev *pdev)
 
 	nr_pages = size >> PAGE_SHIFT;
 
-	page = ne_alloc_contig(nr_pages, numa_node_id(), &cma);
+	page = ne_slot_pool_alloc_run(nr_pages, numa_node_id(), &cma);
 	if (IS_ERR_OR_NULL(page)) {
 		rc = page ? PTR_ERR(page) : -ENOMEM;
 
 		dev_warn(&pdev->dev,
-			 "Slot pool base size=%u not allocated on node %d [rc=%d]\n",
-			 size, numa_node_id(), rc);
+			 "Slot pool base size=%u not allocated [rc=%d]\n",
+			 size, rc);
 
 		return rc;
 	}

@@ -798,6 +856,7 @@ static int ne_slot_pool_donate_base(struct pci_dev *pdev)
  * Return:
  * * 0 on success.
  * * -ENOSPC if the pool has reached the ceiling the device reports.
+ * * -EINTR if a fatal signal arrived while a run was being zeroed.
  * * Negative return value on failure.
  * * -ETIMEDOUT, or any other code ne_request_unanswered() names, if the
  *   request went unanswered, in which case the run stays recorded as donated

@@ -809,14 +868,12 @@ static int ne_slot_pool_donate(struct pci_dev *pdev)
 	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
 	struct ne_pci_dev_cmd_reply cmd_reply = {};
 	struct slot_pool_donate_req donate_req = {};
+	struct ne_slot_pool_run *run;
 	struct cma *cma = NULL;
 	unsigned long nr_pages;
 	struct page *page;
 	int rc;
 
-	if (ne_pci_dev->slot_pool_page)
-		return 0;
-
 	if (!ne_pci_dev->slot_pool_gran)
 		return -EOPNOTSUPP;
 

@@ -826,17 +883,27 @@ static int ne_slot_pool_donate(struct pci_dev *pdev)
 
 	nr_pages = ne_pci_dev->slot_pool_gran >> PAGE_SHIFT;
 
-	page = ne_alloc_contig(nr_pages, numa_node_id(), &cma);
+	run = kzalloc_obj(*run);
+	if (!run)
+		return -ENOMEM;
+
+	page = ne_slot_pool_alloc_run(nr_pages, numa_node_id(), &cma);
 	if (IS_ERR_OR_NULL(page)) {
 		rc = page ? PTR_ERR(page) : -ENOMEM;
 
 		dev_warn(&pdev->dev,
-			 "Slot pool granularity size=%u not allocated on node %d [rc=%d]\n",
-			 ne_pci_dev->slot_pool_gran, numa_node_id(), rc);
+			 "Slot pool granularity size=%u not allocated [rc=%d]\n",
+			 ne_pci_dev->slot_pool_gran, rc);
+
+		kfree(run);
 
 		return rc;
 	}
 
+	run->page = page;
+	run->cma = cma;
+	run->nr_pages = nr_pages;
+
 	donate_req.paddr = page_to_phys(page);
 
 	rc = ne_do_request(pdev, SLOT_POOL_DONATE,

@@ -851,21 +918,362 @@ static int ne_slot_pool_donate(struct pci_dev *pdev)
 		 * does and for the same reason.
 		 */
 		if (ne_request_unanswered(rc)) {
-			ne_pci_dev->slot_pool_page = page;
-			ne_pci_dev->slot_pool_cma = cma;
+			list_add(&run->node, &ne_pci_dev->slot_pool_meta_runs);
+			ne_pci_dev->slot_pool_meta_nr++;
 		} else {
 			ne_contig_free_run(cma, page, nr_pages);
+			kfree(run);
 		}
 
 		return rc;
 	}
 
-	ne_pci_dev->slot_pool_page = page;
-	ne_pci_dev->slot_pool_cma = cma;
+	list_add(&run->node, &ne_pci_dev->slot_pool_meta_runs);
+	ne_pci_dev->slot_pool_meta_nr++;
 
 	return 0;
 }
 
+/*
+ * Size of one run of memory the slot pool takes, and of the span the driver
+ * offers so that the device can place such a run inside it. A run begins on a
+ * boundary of its own size in the device's address space, and an address the
+ * parent aligns does not stay aligned once the device has mapped it, so the
+ * parent cannot place the run itself. A span of twice the size holds a run the
+ * device can place wherever the span begins.
+ */
+#define NE_SLOT_POOL_MEM_SIZE	SZ_128M
+#define NE_SLOT_POOL_MEM_SPAN	(2 * NE_SLOT_POOL_MEM_SIZE)
+
+/**
+ * ne_slot_pool_donate_nid() - Pick the node the next memory donation comes from.
+ * @nr_donated:	Number of runs of memory the slot pool already holds.
+ *
+ * The device turns a run of donated memory into memory it gives to enclaves, and
+ * the host grows the zone it takes that memory from on the node the run sits on.
+ * Every donation coming from one node grows that node's zone alone, so the
+ * enclaves the host places on any other node run on memory out of a zone that
+ * never grew, and the host runs that zone out and takes every enclave on the
+ * machine down with it. So spread the donations instead: step through the nodes
+ * that have memory in time with the runs already held, and successive donations
+ * come from successive nodes.
+ *
+ * A node that has no run to give costs a donation nothing, because
+ * ne_slot_pool_alloc_run() carries on from the node named here to the rest.
+ *
+ * Context: Process context.
+ * Return: the node the donation is to come from first.
+ */
+static int ne_slot_pool_donate_nid(unsigned int nr_donated)
+{
+	int nr_nodes = num_node_state(N_MEMORY);
+	unsigned int steps;
+	unsigned int i;
+	int nid;
+
+	if (nr_nodes < 1)
+		return numa_node_id();
+
+	steps = nr_donated % (unsigned int)nr_nodes;
+	nid = first_node(node_states[N_MEMORY]);
+
+	for (i = 0; i < steps; i++)
+		nid = next_node_in(nid, node_states[N_MEMORY]);
+
+	return nid;
+}
+
+/**
+ * ne_slot_pool_mem_donate() - Donate a run of memory to the slot pool.
+ * @pdev:	PCI device to donate to.
+ *
+ * The device counts the slots the pool's records support and the slots its
+ * memory supports separately, and hands out the smaller of the two counts.
+ * This donation raises the second count.
+ *
+ * The driver offers a span and the device answers with the run it took out of
+ * it, so the pages before and after that run go back to the allocator and the
+ * run alone stays recorded. A device that answers without naming a run inside
+ * the span leaves the whole span recorded, because which of the span's pages
+ * the device is using is then not established.
+ *
+ * Context: Process context. This function is called with the enclaves list
+ *	    mutex held.
+ * Return:
+ * * 0 on success.
+ * * -EINTR if a fatal signal arrived while the span was being zeroed.
+ * * Negative return value on failure.
+ * * -ETIMEDOUT, or any other code ne_request_unanswered() names, if the
+ *   request went unanswered, in which case the span stays recorded as donated
+ *   because the driver cannot establish that the device refused it. A device
+ *   that answers such a code itself lands here too and costs one held span.
+ */
+static int ne_slot_pool_mem_donate(struct pci_dev *pdev)
+{
+	const unsigned long span_pages = NE_SLOT_POOL_MEM_SPAN >> PAGE_SHIFT;
+	const unsigned long run_pages = NE_SLOT_POOL_MEM_SIZE >> PAGE_SHIFT;
+	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
+	struct slot_pool_mem_donate_req donate_req = {};
+	struct ne_pci_dev_cmd_reply cmd_reply = {};
+	u64 span_paddr, taken_paddr, taken_size;
+	unsigned long head_pages, tail_pages;
+	struct ne_slot_pool_run *run;
+	struct cma *cma = NULL;
+	struct page *page;
+	int rc;
+
+	run = kzalloc_obj(*run);
+	if (!run)
+		return -ENOMEM;
+
+	page = ne_slot_pool_alloc_run(span_pages,
+				      ne_slot_pool_donate_nid(ne_pci_dev->slot_pool_mem_nr),
+				      &cma);
+	if (IS_ERR_OR_NULL(page)) {
+		rc = page ? PTR_ERR(page) : -ENOMEM;
+
+		dev_warn(&pdev->dev,
+			 "Slot pool memory span size=%u not allocated [rc=%d]\n",
+			 NE_SLOT_POOL_MEM_SPAN, rc);
+
+		kfree(run);
+
+		return rc;
+	}
+
+	span_paddr = page_to_phys(page);
+
+	run->page = page;
+	run->cma = cma;
+	run->nr_pages = span_pages;
+
+	donate_req.paddr = span_paddr;
+	donate_req.size = NE_SLOT_POOL_MEM_SPAN;
+
+	rc = ne_do_request(pdev, SLOT_POOL_MEM_DONATE,
+			   &donate_req, sizeof(donate_req),
+			   &cmd_reply, sizeof(cmd_reply));
+	if (rc < 0) {
+		dev_err_ratelimited(&pdev->dev,
+				    "Error in slot pool memory donate [rc=%d]\n",
+				    rc);
+
+		/*
+		 * Record a span whose donation went unanswered as donated
+		 * instead of giving it back, as ne_slot_pool_donate_base() does
+		 * and for the same reason. The whole span is recorded, because
+		 * the run the device took out of it is what the answer that
+		 * never came would have named.
+		 */
+		if (ne_request_unanswered(rc)) {
+			list_add(&run->node, &ne_pci_dev->slot_pool_mem_runs);
+			ne_pci_dev->slot_pool_mem_nr++;
+		} else {
+			ne_contig_free_run(cma, page, span_pages);
+			kfree(run);
+		}
+
+		return rc;
+	}
+
+	taken_paddr = cmd_reply.mem_donate.consumed_paddr;
+	taken_size = cmd_reply.mem_donate.consumed_size;
+
+	/*
+	 * The run the device names has to be one run's worth, inside the span
+	 * it was offered, and to begin on a page boundary of the span, since
+	 * the pages given back are counted from where it begins. The device
+	 * places the run on a boundary of its own address space, which the
+	 * span's physical address says nothing about, so where in the span it
+	 * begins is only bounded, not fixed.
+	 */
+	if (taken_size == NE_SLOT_POOL_MEM_SIZE &&
+	    taken_paddr >= span_paddr &&
+	    taken_paddr - span_paddr <=
+		    NE_SLOT_POOL_MEM_SPAN - NE_SLOT_POOL_MEM_SIZE &&
+	    IS_ALIGNED(taken_paddr - span_paddr, PAGE_SIZE)) {
+		head_pages = (taken_paddr - span_paddr) >> PAGE_SHIFT;
+		tail_pages = span_pages - head_pages - run_pages;
+
+		run->page = page + head_pages;
+		run->nr_pages = run_pages;
+
+		if (head_pages)
+			ne_contig_free_run(cma, page, head_pages);
+
+		if (tail_pages)
+			ne_contig_free_run(cma, run->page + run_pages,
+					   tail_pages);
+	} else {
+		dev_warn(&pdev->dev,
+			 "Slot pool memory span kept, the device took paddr=0x%llx size=0x%llx\n",
+			 taken_paddr, taken_size);
+	}
+
+	list_add(&run->node, &ne_pci_dev->slot_pool_mem_runs);
+	ne_pci_dev->slot_pool_mem_nr++;
+
+	return 0;
+}
+
+/**
+ * ne_slot_pool_limits() - Read how many slots each half of the slot pool has
+ *			   room for.
+ * @pdev:	PCI device to read the slot pool limits of.
+ * @cmd_reply:	Reply the limits are read out of, valid on success.
+ *
+ * The reply is a snapshot. A caller that donates on the strength of it and then
+ * reads it again holds the enclaves list mutex across both.
+ *
+ * Context: Process context.
+ * Return:
+ * * 0 on success.
+ * * Negative return value on failure.
+ */
+static int ne_slot_pool_limits(struct pci_dev *pdev,
+			       struct ne_pci_dev_cmd_reply *cmd_reply)
+{
+	u8 unused = 0;
+
+	return ne_do_request(pdev, SLOT_POOL_GET_LIMITS,
+			     &unused, sizeof(unused),
+			     cmd_reply, sizeof(*cmd_reply));
+}
+
+/**
+ * ne_slot_pool_effective_limit() - Read how many slots the parent can use out
+ *				    of a reply the device filled.
+ * @cmd_reply:	Reply a SLOT_POOL_GET_LIMITS filled.
+ *
+ * A slot needs room in both halves of the pool, so the number of slots the
+ * parent can use is the smaller of the two the device reports.
+ *
+ * Context: Process context.
+ * Return: the number of slots the parent can use.
+ */
+static u16 ne_slot_pool_effective_limit(const struct ne_pci_dev_cmd_reply *cmd_reply)
+{
+	return min(cmd_reply->limits.meta_limit, cmd_reply->limits.mem_limit);
+}
+
+/**
+ * ne_slot_pool_grow() - Grow the slot pool until it holds @target slots.
+ * @pdev:	PCI device to donate to.
+ * @target:	Number of slots the pool is to hold.
+ *
+ * The device hands out the smaller of the slots its records support and the
+ * slots its memory supports, so each round donates to whichever of the two is
+ * behind and then asks the device for both counts again. Neither is behind
+ * when the two are level, and a round donates to both then, because a
+ * donation to one half of a level pair leaves the smaller count where it was.
+ *
+ * A round that leaves both counts where they were ends the growth: the device
+ * took every donation the round offered and got no further with any of them,
+ * and another round would allocate again for the same answer.
+ *
+ * The pool only grows. A run the device accepted cannot be taken back on its
+ * own, so growth that stops short leaves the pool at the size it reached.
+ *
+ * Context: Process context. This function is called with the enclaves list
+ *	    mutex held.
+ * Return:
+ * * 0 once the pool holds @target slots or more.
+ * * -EOPNOTSUPP on a device that takes no donation.
+ * * -EINVAL where @target is above the number of slots the device allows.
+ * * -ENOSPC where a round raised neither count and the pool is below @target.
+ * * -EINTR if a fatal signal arrived while a run was being zeroed.
+ * * Negative return value on failure.
+ */
+int ne_slot_pool_grow(struct pci_dev *pdev, u16 target)
+{
+	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
+	struct ne_pci_dev_cmd_reply cmd_reply = {};
+	u16 effective;
+	int rc;
+
+	if (!ne_pci_dev->slot_pool_max)
+		return -EOPNOTSUPP;
+
+	if (target > ne_pci_dev->slot_pool_max)
+		return -EINVAL;
+
+	rc = ne_slot_pool_donate_base(pdev);
+	if (rc < 0)
+		return rc;
+
+	rc = ne_slot_pool_limits(pdev, &cmd_reply);
+	if (rc < 0)
+		return rc;
+
+	effective = ne_slot_pool_effective_limit(&cmd_reply);
+
+	while (effective < target) {
+		u16 meta_before = cmd_reply.limits.meta_limit;
+		u16 mem_before = cmd_reply.limits.mem_limit;
+
+		if (cmd_reply.limits.mem_limit <= cmd_reply.limits.meta_limit) {
+			rc = ne_slot_pool_mem_donate(pdev);
+			if (rc < 0)
+				return rc;
+		}
+
+		if (cmd_reply.limits.meta_limit <= cmd_reply.limits.mem_limit) {
+			rc = ne_slot_pool_donate(pdev);
+			if (rc < 0)
+				return rc;
+		}
+
+		rc = ne_slot_pool_limits(pdev, &cmd_reply);
+		if (rc < 0)
+			return rc;
+
+		if (cmd_reply.limits.meta_limit <= meta_before &&
+		    cmd_reply.limits.mem_limit <= mem_before)
+			return -ENOSPC;
+
+		effective = ne_slot_pool_effective_limit(&cmd_reply);
+	}
+
+	return 0;
+}
+
+/**
+ * enum ne_slot_pool_runs_owner - Who keeps the runs of a drained list.
+ * @NE_RUNS_TO_ALLOCATOR:	The device handed the runs back, so they go back
+ *				to the allocator.
+ * @NE_RUNS_STAY_WITH_DEVICE:	The device may still use the runs, so only the
+ *				records of them go away.
+ */
+enum ne_slot_pool_runs_owner {
+	NE_RUNS_TO_ALLOCATOR,
+	NE_RUNS_STAY_WITH_DEVICE,
+};
+
+/**
+ * ne_slot_pool_runs_drain() - Empty a list of donated runs.
+ * @runs:	List of runs to empty.
+ * @nr:		Count of runs on @runs, set to zero.
+ * @owner:	Who keeps the runs.
+ *
+ * Context: Process context. This function is called with the enclaves list
+ *	    mutex held.
+ */
+static void ne_slot_pool_runs_drain(struct list_head *runs, unsigned int *nr,
+				    enum ne_slot_pool_runs_owner owner)
+{
+	struct ne_slot_pool_run *run, *next;
+
+	list_for_each_entry_safe(run, next, runs, node) {
+		if (owner == NE_RUNS_TO_ALLOCATOR)
+			ne_contig_free_run(run->cma, run->page, run->nr_pages);
+
+		list_del(&run->node);
+		kfree(run);
+	}
+
+	*nr = 0;
+}
+
 /**
  * ne_slot_pool_release() - Take the whole slot pool back from the device.
  * @pdev:	PCI device to take the pool back from.

@@ -904,17 +1312,15 @@ static void ne_slot_pool_release(struct pci_dev *pdev)
 	if (rc < 0) {
 		dev_err(&pdev->dev, "Error in slot pool shutdown [rc=%d]\n", rc);
 
-		goto unlock_mutex;
+		goto forget_runs;
 	}
 
-	if (ne_pci_dev->slot_pool_page) {
-		ne_contig_free_run(ne_pci_dev->slot_pool_cma,
-				   ne_pci_dev->slot_pool_page,
-				   ne_pci_dev->slot_pool_gran >> PAGE_SHIFT);
-
-		ne_pci_dev->slot_pool_page = NULL;
-		ne_pci_dev->slot_pool_cma = NULL;
-	}
+	ne_slot_pool_runs_drain(&ne_pci_dev->slot_pool_meta_runs,
+				&ne_pci_dev->slot_pool_meta_nr,
+				NE_RUNS_TO_ALLOCATOR);
+	ne_slot_pool_runs_drain(&ne_pci_dev->slot_pool_mem_runs,
+				&ne_pci_dev->slot_pool_mem_nr,
+				NE_RUNS_TO_ALLOCATOR);
 
 	ne_contig_free_run(ne_pci_dev->slot_pool_base_cma,
 			   ne_pci_dev->slot_pool_base_page,

@@ -923,6 +1329,20 @@ static void ne_slot_pool_release(struct pci_dev *pdev)
 	ne_pci_dev->slot_pool_base_page = NULL;
 	ne_pci_dev->slot_pool_base_cma = NULL;
 
+	goto unlock_mutex;
+
+forget_runs:
+	/*
+	 * The device keeps these runs, so only the records of them go, and the
+	 * pages they name stay the device's for as long as it is bound.
+	 */
+	ne_slot_pool_runs_drain(&ne_pci_dev->slot_pool_meta_runs,
+				&ne_pci_dev->slot_pool_meta_nr,
+				NE_RUNS_STAY_WITH_DEVICE);
+	ne_slot_pool_runs_drain(&ne_pci_dev->slot_pool_mem_runs,
+				&ne_pci_dev->slot_pool_mem_nr,
+				NE_RUNS_STAY_WITH_DEVICE);
+
 unlock_mutex:
 	mutex_unlock(&ne_pci_dev->enclaves_list_mutex);
 }

@@ -976,6 +1396,8 @@ static int ne_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	atomic_set(&ne_pci_dev->cmd_reply_avail, 0);
 	init_waitqueue_head(&ne_pci_dev->cmd_reply_wait_q);
 	INIT_LIST_HEAD(&ne_pci_dev->enclaves_list);
+	INIT_LIST_HEAD(&ne_pci_dev->slot_pool_meta_runs);
+	INIT_LIST_HEAD(&ne_pci_dev->slot_pool_mem_runs);
 	mutex_init(&ne_pci_dev->enclaves_list_mutex);
 	mutex_init(&ne_pci_dev->pci_dev_mutex);
 	ne_pci_dev->pdev = pdev;

index 3718f893e4f65..cab5434fc3e0d 100644
@@ -468,6 +468,24 @@ static_assert(offsetof(struct ne_pci_dev_cmd_reply,
 		       limits.slots_per_donation) == 12);
 static_assert(offsetof(struct ne_pci_dev_cmd_reply, limits.reserved14) == 14);
 
+/**
+ * struct ne_slot_pool_run - One run of pages the driver gave the slot pool.
+ * @node:	Entry on the list of runs donated to one of the pools.
+ * @page:	First page of the run.
+ * @cma:	CMA area the run came from.
+ * @nr_pages:	Number of pages in the run.
+ *
+ * A record is allocated before the run it describes, and the run is recorded
+ * before the request carrying its address goes out, so a run the device
+ * accepted is a run ne_slot_pool_release() finds.
+ */
+struct ne_slot_pool_run {
+	struct list_head	node;
+	struct page		*page;
+	struct cma		*cma;
+	unsigned long		nr_pages;
+};
+
 /**
  * struct ne_pci_dev - Nitro Enclaves (NE) PCI device.
  * @cmd_reply_avail:		Variable set if a reply has been sent by the

@@ -501,9 +519,11 @@ static_assert(offsetof(struct ne_pci_dev_cmd_reply, limits.reserved14) == 14);
  *				has been made.
  * @slot_pool_base_cma:		CMA area the base donation came from.
  * @slot_pool_base_nr_pages:	Number of pages in the base donation.
- * @slot_pool_page:		First page of the donation to the slot pool, NULL
- *				until it has been made.
- * @slot_pool_cma:		CMA area the donation came from.
+ * @slot_pool_meta_runs:	Runs donated to hold the pool's slot descriptors.
+ * @slot_pool_meta_nr:		Number of runs on @slot_pool_meta_runs.
+ * @slot_pool_mem_runs:		Runs donated for the enclaves in the pool's
+ *				slots to run in.
+ * @slot_pool_mem_nr:		Number of runs on @slot_pool_mem_runs.
  */
 struct ne_pci_dev {
 	atomic_t		cmd_reply_avail;

@@ -522,13 +542,18 @@ struct ne_pci_dev {
 	struct page		*slot_pool_base_page;
 	struct cma		*slot_pool_base_cma;
 	unsigned long		slot_pool_base_nr_pages;
-	struct page		*slot_pool_page;
-	struct cma		*slot_pool_cma;
+	struct list_head	slot_pool_meta_runs;
+	unsigned int		slot_pool_meta_nr;
+	struct list_head	slot_pool_mem_runs;
+	unsigned int		slot_pool_mem_nr;
 };
 
 /* Whether a failure left a submitted request in flight. */
 bool ne_request_unanswered(int rc);
 
+/* Grow the slot pool until it holds a given number of slots. */
+int ne_slot_pool_grow(struct pci_dev *pdev, u16 target);
+
 /**
  * ne_do_request() - Submit command request to the PCI device based on the command
  *		     type and retrieve the associated reply.

index 2fae731acb977..208c1997609d3 100644
@@ -425,11 +425,17 @@ static const struct attribute_group ne_cpu_pool_node_group = {
 static struct kobject *ne_cpu_pool_kobj;
 static struct kobject **ne_cpu_pool_node_kobj;
 
+static const struct attribute_group *ne_groups[] = {
+	&ne_slot_pool_group,
+	NULL,
+};
+
 static struct miscdevice ne_misc_dev = {
 	.minor	= MISC_DYNAMIC_MINOR,
 	.name	= "nitro_enclaves",
 	.fops	= &ne_fops,
 	.mode	= 0660,
+	.groups	= ne_groups,
 };
 
 struct ne_devs ne_devs = {

191 lines not on this page · raw message

index cab5434fc3e0d..f5cb24312d442 100644
@@ -578,4 +578,7 @@ int ne_do_request(struct pci_dev *pdev, enum ne_pci_dev_cmd_type cmd_type,
 /* Nitro Enclaves (NE) PCI device driver */
 extern struct pci_driver ne_pci_driver;
 
+/* slot_pool/ sysfs group on the NE misc device */
+extern const struct attribute_group ne_slot_pool_group;
+
 #endif /* _NE_PCI_DEV_H_ */

index f5cb24312d442..3445318b84672 100644
@@ -161,6 +161,7 @@
  * @SLOT_ADD_BULK_VCPUS:	Add a number of vCPUs, not providing CPU ids.
  * @SLOT_ADD_INFO_PAGE:		Give a slot the info page, a memory region the
  *				device writes.
+ * @SLOT_ADD_DEVICE:		Give a slot a device the enclave will drive.
  * @SLOT_POOL_DONATE:		Donate one page of parent memory to the slot
  *				pool.
  * @SLOT_POOL_SHUTDOWN:		Give the whole slot pool back.

@@ -186,6 +187,7 @@ enum ne_pci_dev_cmd_type {
 	SLOT_ADD_BULK_VCPUS	= 11,
 	/* 12 is a command this driver does not implement. */
 	SLOT_ADD_INFO_PAGE	= 13,
+	SLOT_ADD_DEVICE		= 14,
 	SLOT_POOL_DONATE	= 16,
 	SLOT_POOL_SHUTDOWN	= 17,
 	SLOT_POOL_BASE_DONATE	= 25,

@@ -333,6 +335,73 @@ struct slot_add_info_page_req {
 	u64	info_size;
 };
 
+/**
+ * NE_MAX_VQS_PER_DEVICE - Highest number of virtqueues one device may have.
+ */
+#define NE_MAX_VQS_PER_DEVICE		(64)
+
+/**
+ * NE_MAX_BACKING_RANGES - Highest number of memory ranges one device may be
+ *			   backed by.
+ */
+#define NE_MAX_BACKING_RANGES		(4)
+
+/**
+ * struct ne_backing_range - One run of enclave memory a device works out of.
+ * @phys_addr:	Physical address of the run, in the parent's address space.
+ * @size:	Size, in bytes, of the run.
+ */
+struct ne_backing_range {
+	u64	phys_addr;
+	u64	size;
+};
+
+/**
+ * struct ne_vq_wiring - How the enclave reaches one virtqueue of a device.
+ * @msix_vector:		Interrupt vector the device raises for the queue.
+ * @notify_index:		Index the enclave writes to notify the device.
+ * @msix_entry:			Entry in the enclave's interrupt table holding
+ *				@msix_vector.
+ * @doorbell_offset:		Offset of the queue's doorbell in the device's
+ *				doorbell window.
+ */
+struct ne_vq_wiring {
+	u16	msix_vector;
+	u16	notify_index;
+	u16	msix_entry;
+	u16	doorbell_offset;
+};
+
+static_assert(sizeof(struct ne_vq_wiring) == 8);
+
+/**
+ * struct slot_add_device_req - SLOT_ADD_DEVICE request.
+ * @slot_uid:			Slot unique id mapped to the slot to give the
+ *				device to.
+ * @device_type:		Type of device to give the slot.
+ * @num_vqs:			Number of virtqueues the device has, at most
+ *				NE_MAX_VQS_PER_DEVICE.
+ * @device_features:		Feature bits the parent offers the enclave for
+ *				the device.
+ * @config_size:		Number of bytes of @device_config the device
+ *				reads.
+ * @num_backing_ranges:		Number of entries of @backing_ranges the
+ *				device reads, at most NE_MAX_BACKING_RANGES.
+ * @device_config:		Type-specific configuration of the device.
+ * @backing_ranges:		Runs of enclave memory the device works out of.
+ */
+struct slot_add_device_req {
+	u64	slot_uid;
+	u32	device_type;
+	u32	num_vqs;
+	u8	device_features[64];
+	u32	config_size;
+	u32	num_backing_ranges;
+	u8	device_config[64];
+	struct ne_backing_range backing_ranges[NE_MAX_BACKING_RANGES];
+} __packed;
+static_assert(sizeof(struct slot_add_device_req) <= NE_SEND_DATA_SIZE);
+
 /**
  * struct slot_pool_donate_req - SLOT_POOL_DONATE and SLOT_POOL_BASE_DONATE
  *				 request.