Skip to contentSkip to conversation

expandable grant table

PATCH In review Keir Fraser 4 participants 9 messages Jan 5, 2007 – Feb 27, 2007 1 message not in the archive

lore
Share & tools
lore

A message that is not in the archive (sent off-list or not yet received): [email protected]

#1 lore raw

PATCH 1/1 expandable grant table

This one is waiting for final review before being checked in. Some people want the functionality this enables (>3 VIFs per guest) so I'm posting the patch in advance of applying it to xen-unstable.

-- Keir

linux-2.6-xen-sparse/drivers/xen/core/gnttab.c +216 -49
diff -r f026d4091322 linux-2.6-xen-sparse/drivers/xen/core/gnttab.c
@@ -3,7 +3,7 @@
  *
  * Granting foreign access to our memory reservation.
  *
- * Copyright (c) 2005, Christopher Clark
+ * Copyright (c) 2005-2006, Christopher Clark
  * Copyright (c) 2004-2005, K A Fraser
  *
  * This program is free software; you can redistribute it and/or
@@ -35,7 +35,6 @@
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
-#include <linux/vmalloc.h>
 #include <xen/interface/xen.h>
 #include <xen/gnttab.h>
 #include <asm/pgtable.h>
@@ -50,37 +49,63 @@
 
 /* External tools reserve first few grant table entries. */
 #define NR_RESERVED_ENTRIES 8
-
-#define NR_GRANT_ENTRIES \
-	(NR_GRANT_FRAMES * PAGE_SIZE / sizeof(struct grant_entry))
-#define GNTTAB_LIST_END (NR_GRANT_ENTRIES + 1)
-
-static grant_ref_t gnttab_list[NR_GRANT_ENTRIES];
+#define GNTTAB_LIST_END 0xffffffff
+#define GREFS_PER_GRANT_FRAME (PAGE_SIZE / sizeof(grant_entry_t))
+
+static grant_ref_t **gnttab_list;
+static unsigned int nr_grant_frames;
+static unsigned int boot_max_nr_grant_frames;
 static int gnttab_free_count;
 static grant_ref_t gnttab_free_head;
 static DEFINE_SPINLOCK(gnttab_list_lock);
 
 static struct grant_entry *shared;
+#ifndef CONFIG_XEN
+static unsigned long resume_frames;
+#endif
 
 static struct gnttab_free_callback *gnttab_free_callback_list;
 
+static int gnttab_expand(unsigned int req_entries);
+
+static inline unsigned int
+gnttab_list_table(grant_ref_t ref)
+{
+    return (ref / (PAGE_SIZE / sizeof(grant_ref_t *)));
+}
+
+static inline unsigned int
+gnttab_list_index(grant_ref_t ref)
+{
+    return (ref % (PAGE_SIZE / sizeof(grant_ref_t *)));
+}
+
 static int get_free_entries(int count)
 {
 	unsigned long flags;
-	int ref;
+	int ref, rc;
 	grant_ref_t head;
+
 	spin_lock_irqsave(&gnttab_list_lock, flags);
-	if (gnttab_free_count < count) {
+
+	if ((gnttab_free_count < count) &&
+	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
 		spin_unlock_irqrestore(&gnttab_list_lock, flags);
-		return -1;
-	}
+		return rc;
+	}
+
 	ref = head = gnttab_free_head;
 	gnttab_free_count -= count;
 	while (count-- > 1)
-		head = gnttab_list[head];
-	gnttab_free_head = gnttab_list[head];
-	gnttab_list[head] = GNTTAB_LIST_END;
+		head = gnttab_list[gnttab_list_table(head)]
+				  [gnttab_list_index(head)];
+	gnttab_free_head = gnttab_list[gnttab_list_table(head)]
+				      [gnttab_list_index(head)];
+	gnttab_list[gnttab_list_table(head)]
+		   [gnttab_list_index(head)] = GNTTAB_LIST_END;
+
 	spin_unlock_irqrestore(&gnttab_list_lock, flags);
+
 	return ref;
 }
 
@@ -116,7 +141,8 @@ static void put_free_entry(grant_ref_t r
 {
 	unsigned long flags;
 	spin_lock_irqsave(&gnttab_list_lock, flags);
-	gnttab_list[ref] = gnttab_free_head;
+	gnttab_list[gnttab_list_table(ref)]
+		   [gnttab_list_index(ref)] = gnttab_free_head;
 	gnttab_free_head = ref;
 	gnttab_free_count++;
 	check_free_callbacks();
@@ -132,7 +158,7 @@ int gnttab_grant_foreign_access(domid_t 
 {
 	int ref;
 
-	if (unlikely((ref = get_free_entry()) == -1))
+	if (unlikely((ref = get_free_entry()) < 0))
 		return -ENOSPC;
 
 	shared[ref].frame = frame;
@@ -202,7 +228,7 @@ int gnttab_grant_foreign_transfer(domid_
 {
 	int ref;
 
-	if (unlikely((ref = get_free_entry()) == -1))
+	if (unlikely((ref = get_free_entry()) < 0))
 		return -ENOSPC;
 	gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
 
@@ -273,11 +299,14 @@ void gnttab_free_grant_references(grant_
 		return;
 	spin_lock_irqsave(&gnttab_list_lock, flags);
 	ref = head;
-	while (gnttab_list[ref] != GNTTAB_LIST_END) {
-		ref = gnttab_list[ref];
+	while (gnttab_list[gnttab_list_table(ref)]
+			  [gnttab_list_index(ref)] != GNTTAB_LIST_END) {
+		ref = gnttab_list[gnttab_list_table(ref)]
+				 [gnttab_list_index(ref)];
 		count++;
 	}
-	gnttab_list[ref] = gnttab_free_head;
+	gnttab_list[gnttab_list_table(ref)]
+		   [gnttab_list_index(ref)] = gnttab_free_head;
 	gnttab_free_head = head;
 	gnttab_free_count += count;
 	check_free_callbacks();
@@ -289,7 +318,7 @@ int gnttab_alloc_grant_references(u16 co
 {
 	int h = get_free_entries(count);
 
-	if (h == -1)
+	if (h < 0)
 		return -ENOSPC;
 
 	*head = h;
@@ -309,7 +338,8 @@ int gnttab_claim_grant_reference(grant_r
 	grant_ref_t g = *private_head;
 	if (unlikely(g == GNTTAB_LIST_END))
 		return -ENOSPC;
-	*private_head = gnttab_list[g];
+	*private_head = gnttab_list[gnttab_list_table(g)]
+				   [gnttab_list_index(g)];
 	return g;
 }
 EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
@@ -317,7 +347,8 @@ void gnttab_release_grant_reference(gran
 void gnttab_release_grant_reference(grant_ref_t *private_head,
 				    grant_ref_t release)
 {
-	gnttab_list[release] = *private_head;
+	gnttab_list[gnttab_list_table(release)]
+		   [gnttab_list_index(release)] = *private_head;
 	*private_head = release;
 }
 EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
@@ -356,6 +387,65 @@ void gnttab_cancel_free_callback(struct 
 }
 EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
 
+static int grow_gnttab_list(unsigned int more_frames)
+{
+	unsigned int new_nr_grant_frames, extra_entries, i;
+
+	new_nr_grant_frames = nr_grant_frames + more_frames;
+	extra_entries       = more_frames * GREFS_PER_GRANT_FRAME;
+
+	for (i = nr_grant_frames; i < new_nr_grant_frames; i++)
+	{
+		gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
+		if (!gnttab_list[i])
+			goto grow_nomem;
+	}
+
+
+	for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
+	     i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
+		gnttab_list[gnttab_list_table(i)][gnttab_list_index(i)] = i + 1;
+
+	gnttab_list[new_nr_grant_frames - 1][GREFS_PER_GRANT_FRAME - 1] =
+		gnttab_free_head;
+	gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
+	gnttab_free_count += extra_entries;
+
+	nr_grant_frames = new_nr_grant_frames;
+
+	check_free_callbacks();
+
+	return 0;
+	
+grow_nomem:
+	for ( ; i >= nr_grant_frames; i--)
+		free_page((unsigned long) gnttab_list[i]);
+	return -ENOMEM;
+}
+
+static unsigned int __max_nr_grant_frames(void)
+{
+	struct gnttab_query_size query;
+	int rc;
+
+	query.dom = DOMID_SELF;
+
+	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
+	if ((rc < 0) || (query.status != GNTST_okay))
+		return 4; /* Legacy max supported number of frames */
+
+	return query.max_nr_frames;
+}
+
+static inline unsigned int max_nr_grant_frames(void)
+{
+	unsigned int xen_max = __max_nr_grant_frames();
+
+	if (xen_max > boot_max_nr_grant_frames)
+		return boot_max_nr_grant_frames;
+	return xen_max;
+}
+
 #ifdef CONFIG_XEN
 
 #ifndef __ia64__
@@ -378,49 +468,68 @@ static int unmap_pte_fn(pte_t *pte, stru
 }
 #endif
 
-int gnttab_resume(void)
+static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
 {
 	struct gnttab_setup_table setup;
-	unsigned long frames[NR_GRANT_FRAMES];
+	unsigned long *frames;
+	unsigned int nr_gframes = end_idx + 1;
 	int rc;
 #ifndef __ia64__
-	void *pframes = frames;
+	void *pframes;
 	struct vm_struct *area;
 #endif
 
+	frames = kmalloc(nr_gframes * sizeof(unsigned long), GFP_ATOMIC);
+	if (!frames)
+		return -ENOMEM;
+
 	setup.dom        = DOMID_SELF;
-	setup.nr_frames  = NR_GRANT_FRAMES;
+	setup.nr_frames  = nr_gframes;
 	set_xen_guest_handle(setup.frame_list, frames);
 
 	rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
-	if (rc == -ENOSYS)
+	if (rc == -ENOSYS) {
+		kfree(frames);
 		return -ENOSYS;
+	}
 
 	BUG_ON(rc || setup.status);
 
 #ifndef __ia64__
 	if (shared == NULL) {
-		area = get_vm_area(PAGE_SIZE * NR_GRANT_FRAMES, VM_IOREMAP);
+		area = get_vm_area(PAGE_SIZE * max_nr_grant_frames(),
+				   VM_IOREMAP);
 		BUG_ON(area == NULL);
 		shared = area->addr;
 	}
+	/* The duplicate pointer pframes exists because apply_to_page_range
+	 * modifies the pointer it uses. This complicates the kfree.
+	 */
+	pframes = frames;
 	rc = apply_to_page_range(&init_mm, (unsigned long)shared,
-				 PAGE_SIZE * NR_GRANT_FRAMES,
+				 PAGE_SIZE * nr_gframes,
 				 map_pte_fn, &pframes);
 	BUG_ON(rc);
 #else
 	shared = __va(frames[0] << PAGE_SHIFT);
-	printk("grant table at %p\n", shared);
-#endif
-
-	return 0;
+#endif
+	kfree(frames);
+
+	return 0;
+}
+
+int gnttab_resume(void)
+{
+	if (max_nr_grant_frames() < nr_grant_frames)
+		return -ENOSYS;
+	return gnttab_map(0, nr_grant_frames - 1);
 }
 
 int gnttab_suspend(void)
 {
 #ifndef __ia64__
 	apply_to_page_range(&init_mm, (unsigned long)shared,
-			    PAGE_SIZE * NR_GRANT_FRAMES,
+			    PAGE_SIZE * nr_grant_frames,
 			    unmap_pte_fn, NULL);
 #endif
 	return 0;
@@ -430,53 +539,111 @@ int gnttab_suspend(void)
 
 #include <platform-pci.h>
 
-int gnttab_resume(void)
-{
-	unsigned long frames;
+static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
+{
 	struct xen_add_to_physmap xatp;
 	unsigned int i;
 
-	frames = alloc_xen_mmio(PAGE_SIZE * NR_GRANT_FRAMES);
-
-	for (i = 0; i < NR_GRANT_FRAMES; i++) {
+	/* Loop backwards, so that the first hypercall has the largest index,
+	 * ensuring that the table will only grow once.
+	 */
+	for (i = end_idx; i >= start_idx; i--) {
 		xatp.domid = DOMID_SELF;
 		xatp.idx = i;
 		xatp.space = XENMAPSPACE_grant_table;
-		xatp.gpfn = (frames >> PAGE_SHIFT) + i;
+		xatp.gpfn = (resume_frames >> PAGE_SHIFT) + i;
 		if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp))
 			BUG();
 	}
-
-	shared = ioremap(frames, PAGE_SIZE * NR_GRANT_FRAMES);
+}
+
+int gnttab_resume(void)
+{
+	struct xen_add_to_physmap xatp;
+	unsigned int i, max_nr_gframes, nr_gframes;
+
+	nr_gframes = nr_grant_frames;
+	max_nr_gframes = max_nr_grant_frames();
+	if (max_nr_gframes < nr_gframes)
+		return -ENOSYS;
+
+	resume_frames = alloc_xen_mmio(PAGE_SIZE * max_nr_gframes);
+
+	gnttab_map_shared(0, nr_gframes - 1);
+
+	shared = ioremap(resume_frames, PAGE_SIZE * max_nr_gframes);
 	if (shared == NULL) {
 		printk("error to ioremap gnttab share frames\n");
 		return -1;
 	}
-
 	return 0;
 }
 
 int gnttab_suspend(void)
 {
 	iounmap(shared);
+	resume_frames = 0;
 	return 0;
 }
 
 #endif /* !CONFIG_XEN */
 
+static int gnttab_expand(unsigned int req_entries)
+{
+	int rc;
+	unsigned int cur, extra;
+
+	cur = nr_grant_frames;
+	extra = ((req_entries + (GREFS_PER_GRANT_FRAME-1)) /
+		 GREFS_PER_GRANT_FRAME);
+	if (cur + extra > max_nr_grant_frames())
+		return -ENOSPC;
+
+	if ((rc = gnttab_map(cur, cur + extra - 1)) == 0)
+		rc = grow_gnttab_list(extra);
+
+	return rc;
+}
+
 int __init gnttab_init(void)
 {
 	int i;
+	unsigned int max_nr_glist_frames;
 
 	if (!is_running_on_xen())
 		return -ENODEV;
 
+	nr_grant_frames = 1;
+	boot_max_nr_grant_frames = __max_nr_grant_frames();
+
+	/* Determine the maximum number of frames required for the
+	 * grant reference free list on the current hypervisor.
+	 */
+	max_nr_glist_frames = boot_max_nr_grant_frames * GREFS_PER_GRANT_FRAME /
+			      (PAGE_SIZE / sizeof(grant_ref_t));
+
+	gnttab_list = kmalloc(max_nr_glist_frames * sizeof(grant_ref_t *),
+			      GFP_KERNEL);
+	if (gnttab_list == NULL)
+		return -ENOMEM;
+
+	gnttab_list[0] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
+	if (gnttab_list[0] == NULL)
+	{
+		kfree(gnttab_list);
+		return -ENOMEM;
+	}
+
 	if (gnttab_resume() < 0)
 		return -ENODEV;
 
-	for (i = NR_RESERVED_ENTRIES; i < NR_GRANT_ENTRIES; i++)
-		gnttab_list[i] = i + 1;
-	gnttab_free_count = NR_GRANT_ENTRIES - NR_RESERVED_ENTRIES;
+	for (i = NR_RESERVED_ENTRIES; i < GREFS_PER_GRANT_FRAME - 1; i++)
+		gnttab_list[gnttab_list_table(i)][gnttab_list_index(i)] = i + 1;
+
+	gnttab_list[gnttab_list_table(GREFS_PER_GRANT_FRAME - 1)]
+		   [gnttab_list_index(GREFS_PER_GRANT_FRAME - 1)]
+		   = GNTTAB_LIST_END;
+	gnttab_free_count = GREFS_PER_GRANT_FRAME - NR_RESERVED_ENTRIES;
 	gnttab_free_head  = NR_RESERVED_ENTRIES;
 
 	printk("Grant table initialized\n");
linux-2.6-xen-sparse/include/xen/gnttab.h +0 -13
diff -r f026d4091322 linux-2.6-xen-sparse/include/xen/gnttab.h
@@ -42,13 +42,6 @@
 #include <asm/maddr.h> /* maddr_t */
 #include <xen/interface/grant_table.h>
 #include <xen/features.h>
-
-/* NR_GRANT_FRAMES must be less than or equal to that configured in Xen */
-#ifdef __ia64__
-#define NR_GRANT_FRAMES 1
-#else
-#define NR_GRANT_FRAMES 4
-#endif
 
 struct gnttab_free_callback {
 	struct gnttab_free_callback *next;
@@ -109,12 +102,6 @@ void gnttab_grant_foreign_transfer_ref(g
 void gnttab_grant_foreign_transfer_ref(grant_ref_t, domid_t domid,
 				       unsigned long pfn);
 
-#ifdef __ia64__
-#define gnttab_map_vaddr(map) __va(map.dev_bus_addr)
-#else
-#define gnttab_map_vaddr(map) ((void *)(map.host_virt_addr))
-#endif
-
 int gnttab_suspend(void);
 int gnttab_resume(void);
 
xen/arch/ia64/xen/mm.c +3 -1
diff -r f026d4091322 xen/arch/ia64/xen/mm.c
@@ -1983,8 +1983,10 @@ arch_memory_op(int op, XEN_GUEST_HANDLE(
                 mfn = virt_to_mfn(d->shared_info);
             break;
         case XENMAPSPACE_grant_table:
-            if (xatp.idx < NR_GRANT_FRAMES)
+            spin_lock(d->grant_table->lock);
+            if ( xatp.idx < nr_grant_frames(d->grant_table) )
                 mfn = virt_to_mfn(d->grant_table->shared) + xatp.idx;
+            spin_unlock(d->grant_table->lock);
             break;
         default:
             break;
xen/arch/x86/mm.c +10 -2
diff -r f026d4091322 xen/arch/x86/mm.c
@@ -2932,8 +2932,16 @@ long arch_memory_op(int op, XEN_GUEST_HA
                 mfn = virt_to_mfn(d->shared_info);
             break;
         case XENMAPSPACE_grant_table:
-            if ( xatp.idx < NR_GRANT_FRAMES )
-                mfn = virt_to_mfn(d->grant_table->shared) + xatp.idx;
+            spin_lock(&d->grant_table->lock);
+
+            if ( (xatp.idx >= nr_grant_frames(d->grant_table)) &&
+                 (xatp.idx < max_nr_grant_frames) )
+                gnttab_grow_table(d, xatp.idx + 1);
+
+            if ( xatp.idx < nr_grant_frames(d->grant_table) )
+                mfn = virt_to_mfn(d->grant_table->shared[xatp.idx]);
+
+            spin_unlock(&d->grant_table->lock);
             break;
         default:
             break;
xen/common/grant_table.c +430 -111

770 lines not on this page · raw message

xen/include/asm-ia64/grant_table.h +1 -1
diff -r f026d4091322 xen/include/asm-ia64/grant_table.h
@@ -5,7 +5,7 @@
 #ifndef __ASM_GRANT_TABLE_H__
 #define __ASM_GRANT_TABLE_H__
 
-#define ORDER_GRANT_FRAMES 0
+#define INITIAL_NR_GRANT_FRAMES 1
 
 // for grant map/unmap
 int create_grant_host_mapping(unsigned long gpaddr, unsigned long mfn, unsigned int flags);
xen/include/asm-powerpc/grant_table.h +1 -1
diff -r f026d4091322 xen/include/asm-powerpc/grant_table.h
@@ -23,7 +23,7 @@
 
 #include <asm/mm.h>
 
-#define ORDER_GRANT_FRAMES 2
+#define INITIAL_NR_GRANT_FRAMES 4
 
 /*
  * Caller must own caller's BIGLOCK, is responsible for flushing the TLB, and
xen/include/asm-x86/grant_table.h +3 -3
diff -r f026d4091322 xen/include/asm-x86/grant_table.h
@@ -7,7 +7,7 @@
 #ifndef __ASM_GRANT_TABLE_H__
 #define __ASM_GRANT_TABLE_H__
 
-#define ORDER_GRANT_FRAMES 2
+#define INITIAL_NR_GRANT_FRAMES 1
 
 /*
  * Caller must own caller's BIGLOCK, is responsible for flushing the TLB, and
@@ -21,12 +21,12 @@ int destroy_grant_host_mapping(
 #define gnttab_create_shared_page(d, t, i)                               \
     do {                                                                 \
         share_xen_page_with_guest(                                       \
-            virt_to_page((char *)(t)->shared + ((i) * PAGE_SIZE)),       \
+            virt_to_page((char *)(t)->shared[i]),                        \
             (d), XENSHARE_writable);                                     \
     } while ( 0 )
 
 #define gnttab_shared_mfn(d, t, i)                      \
-    ((virt_to_maddr((t)->shared) >> PAGE_SHIFT) + (i))
+    ((virt_to_maddr((t)->shared[i]) >> PAGE_SHIFT))
 
 #define gnttab_shared_gmfn(d, t, i)                     \
     (mfn_to_gmfn(d, gnttab_shared_mfn(d, t, i)))
xen/include/public/grant_table.h +19 -0
diff -r f026d4091322 xen/include/public/grant_table.h
@@ -308,6 +308,25 @@ typedef struct gnttab_copy {
     int16_t       status;
 } gnttab_copy_t;
 DEFINE_XEN_GUEST_HANDLE(gnttab_copy_t);
+
+/*
+ * GNTTABOP_query_size: Query the current and maximum sizes of the shared
+ * grant table.
+ * NOTES:
+ *  1. <dom> may be specified as DOMID_SELF.
+ *  2. Only a sufficiently-privileged domain may specify <dom> != DOMID_SELF.
+ */
+#define GNTTABOP_query_size           6
+struct gnttab_query_size {
+    /* IN parameters. */
+    domid_t  dom;
+    /* OUT parameters. */
+    uint32_t nr_frames;
+    uint32_t max_nr_frames;
+    int16_t  status;              /* GNTST_* */
+};
+typedef struct gnttab_query_size gnttab_query_size_t;
+DEFINE_XEN_GUEST_HANDLE(gnttab_query_size_t);
 
 
 /*
xen/include/xen/grant_table.h +31 -7
diff -r f026d4091322 xen/include/xen/grant_table.h
@@ -52,9 +52,14 @@ struct active_grant_entry {
 #define GNTPIN_devr_inc      (1 << GNTPIN_devr_shift)
 #define GNTPIN_devr_mask     (0xFFU << GNTPIN_devr_shift)
 
-#define NR_GRANT_FRAMES      (1U << ORDER_GRANT_FRAMES)
-#define NR_GRANT_ENTRIES     \
-    ((NR_GRANT_FRAMES << PAGE_SHIFT) / sizeof(grant_entry_t))
+/* Initial size of a grant table. */
+#define INITIAL_NR_GRANT_ENTRIES ((INITIAL_NR_GRANT_FRAMES << PAGE_SHIFT) / \
+                                     sizeof(grant_entry_t))
+
+/* Default maximum size of a grant table. [POLICY] */
+#define DEFAULT_MAX_NR_GRANT_FRAMES   32
+/* The maximum size of a grant table. */
+extern unsigned int max_nr_grant_frames;
 
 /*
  * Tracks a mapping of another domain's grant reference. Each domain has a
@@ -71,14 +76,15 @@ struct grant_mapping {
 
 /* Per-domain grant information. */
 struct grant_table {
+    /* Table size. Number of frames shared with guest */
+    unsigned int          nr_grant_frames;
     /* Shared grant table (see include/public/grant_table.h). */
-    struct grant_entry   *shared;
+    struct grant_entry  **shared;
     /* Active grant table. */
-    struct active_grant_entry *active;
+    struct active_grant_entry **active;
     /* Mapping tracking table. */
-    struct grant_mapping *maptrack;
+    struct grant_mapping **maptrack;
     unsigned int          maptrack_head;
-    unsigned int          maptrack_order;
     unsigned int          maptrack_limit;
     unsigned int          map_count;
     /* Lock protecting updates to active and shared grant tables. */
@@ -96,4 +102,22 @@ gnttab_release_mappings(
 gnttab_release_mappings(
     struct domain *d);
 
+/* Increase the size of a domain's grant table.
+ * Caller must hold d's grant table lock.
+ */
+int
+gnttab_grow_table(struct domain *d, unsigned int req_nr_frames);
+
+/* Number of grant table frames. Caller must hold d's grant table lock. */
+static inline unsigned int nr_grant_frames(struct grant_table *gt)
+{
+    return gt->nr_grant_frames;
+}
+
+/* Number of grant table entries. Caller must hold d's grant table lock. */
+static inline unsigned int nr_grant_entries(struct grant_table *gt)
+{
+    return (nr_grant_frames(gt) << PAGE_SHIFT) / sizeof(grant_entry_t);
+}
+
 #endif /* __XEN_GRANT_TABLE_H__ */

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#2 lore raw
Quoted 3 lines · Keir Fraser

This one is waiting for final review before being checked in. Some people want the functionality this enables (>3 VIFs per guest) so I'm posting the patch in advance of applying it to xen-unstable.

I've been working on the same issue so that I have some patches. I attached a part of them. Although they should be adjusted to your patch which is to be commited, I wanted to send these out before checking in.
If necessary, I'm willing to update them.

- acm code uses NR_GRANT_ENTRIES.
The attached patch is only compile-tested.

- IA64 code needs a slight domain initialization adjustment.

- PPC code depends on the fact that shared grant table is machine continguous. I'm not familiar with PPC code so that I'm not sure how to solve it. If arch specific code can override maximal grant table size, Xen/PPC can prohibit growing grant table.
After solving the issue, they can allow to grow grant table.

Signature · 1 line
yamahata

# HG changeset patch
# User [email protected]
# Date 1168063801 -32400
# Node ID 504addd06eeda65eb625b2969fb1d26cde7eb62b # Parent 9865145e53eb02ddc2e2792f80bb7d54bc754a65
PATCHNAME: grow_granttable_acm

xen/acm/acm_simple_type_enforcement_hooks.c +8 -3
diff -r 9865145e53eb -r 504addd06eed xen/acm/acm_simple_type_enforcement_hooks.c
@@ -234,18 +234,21 @@ ste_init_state(struct acm_ste_policy_buf
             }
         } 
         /* b) check for grant table conflicts on shared pages */
-        if ((*pd)->grant_table->shared == NULL) {
+        spin_lock(&(*pd)->grant_table->lock);
+        if ((*pd)->grant_table->nr_grant_entries == 0) {
+            spin_unlock(&(*pd)->grant_table->lock);
             printkd("%s: Grant ... sharing for domain %x not setup!\n", __func__, (*pd)->domain_id);
             continue;
         }
-        for ( i = 0; i < NR_GRANT_ENTRIES; i++ ) {
-            sha_copy =  (*pd)->grant_table->shared[i];
+        for ( i = 0; i < (*pd)->grant_table->nr_grant_entries; i++ ) {
+            sha_copy = *gnttab_get_entry((*pd)->grant_table, i);
             if ( sha_copy.flags ) {
                 printkd("%s: grant dom (%hu) SHARED (%d) flags:(%hx) dom:(%hu) frame:(%lx)\n",
                         __func__, (*pd)->domain_id, i, sha_copy.flags, sha_copy.domid, 
                         (unsigned long)sha_copy.frame);
                 rdomid = sha_copy.domid;
                 if ((rdom = find_domain_by_id(rdomid)) == NULL) {
+                    spin_unlock(&(*pd)->grant_table->lock);
                     printkd("%s: domain not found ERROR!\n", __func__);
                     goto out;
                 };
@@ -255,6 +258,7 @@ ste_init_state(struct acm_ste_policy_buf
                 ste_rssidref = ste_rssid->ste_ssidref;
                 put_domain(rdom);
                 if (!have_common_type(ste_ssidref, ste_rssidref)) {
+                    spin_unlock(&(*pd)->grant_table->lock);
                     printkd("%s: Policy violation in grant table sharing domain %x -> domain %x.\n",
                             __func__, (*pd)->domain_id, rdomid);
                     goto out;
@@ -262,6 +266,7 @@ ste_init_state(struct acm_ste_policy_buf
             }
         }
     }
+    spin_unlock(&(*pd)->grant_table->lock);
     violation = 0;
  out:
     read_unlock(&domlist_lock);

# HG changeset patch
# User [email protected]
# Date 1168063865 -32400
# Node ID 79fd3532f2e4c3c019cd535edf1068bc3cd96e93 # Parent 504addd06eeda65eb625b2969fb1d26cde7eb62b
PATCHNAME: grow_granttable_ia64

xen/arch/ia64/xen/domain.c +8 -0
diff -r 504addd06eed -r 79fd3532f2e4 xen/arch/ia64/xen/domain.c
@@ -491,6 +491,14 @@ int arch_domain_create(struct domain *d)
 	if ((d->arch.mm.pgd = pgd_alloc(&d->arch.mm)) == NULL)
 	    goto fail_nomem;
 
+	/*
+	 * grant_table_create() can't fully initialize grant table for domain
+	 * because it is called before arch_domain_create().
+	 * Here we complete the initialization which requires p2m table.
+	 */
+	for (i = 0; i < d->grant_table->nr_grant_frames; i++)
+		ia64_gnttab_create_shared_page(d, d->grant_table, i);
+
 	d->arch.ioport_caps = rangeset_new(d, "I/O Ports",
 	                                   RANGESETF_prettyprint_hex);
 
xen/include/asm-ia64/grant_table.h +25 -15
diff -r 504addd06eed -r 79fd3532f2e4 xen/include/asm-ia64/grant_table.h
@@ -13,15 +13,6 @@ int destroy_grant_host_mapping(unsigned 
 
 // for grant transfer
 void guest_physmap_add_page(struct domain *d, unsigned long gpfn, unsigned long mfn);
-
-// for grant table shared page
-#define gnttab_create_shared_page(d, t, i)                              \
-    do {                                                                \
-        share_xen_page_with_guest(                                      \
-            virt_to_page((char *)(t)->shared + ((i) << PAGE_SHIFT)),    \
-            (d), XENSHARE_writable);                                    \
-    } while (0)
-
 
 /* XXX
  * somewhere appropriate
@@ -40,13 +31,32 @@ void guest_physmap_add_page(struct domai
 #define IA64_GRANT_TABLE_PADDR  IA64_XMAPPEDREGS_PADDR(NR_CPUS)
 
 #define gnttab_shared_maddr(d, t, i)                        \
-    virt_to_maddr((char*)(t)->shared + ((i) << PAGE_SHIFT))
+    (virt_to_maddr((t)->shared_frames[(i)]))
 
-# define gnttab_shared_gmfn(d, t, i)                                    \
-    ({ assign_domain_page((d),                                          \
-                          IA64_GRANT_TABLE_PADDR + ((i) << PAGE_SHIFT), \
-                          gnttab_shared_maddr((d), (t), (i)));          \
-        (IA64_GRANT_TABLE_PADDR >> PAGE_SHIFT) + (i);})
+#define ia64_gnttab_create_shared_page(d, t, i)                         \
+    do {                                                                \
+        BUG_ON((d)->arch.mm.pgd == NULL);                               \
+        assign_domain_page((d),                                         \
+                           IA64_GRANT_TABLE_PADDR + ((i) << PAGE_SHIFT),\
+                           gnttab_shared_maddr((d), (t), (i)));         \
+    } while (0)
+
+/*
+ * for grant table shared page
+ * grant_table_create() might call this macro before allocating the p2m table.
+ * In such case, arch_domain_create() completes the initialization.
+ */
+#define gnttab_create_shared_page(d, t, i)                              \
+    do {                                                                \
+        share_xen_page_with_guest(                                      \
+            virt_to_page((t)->shared_frames[(i)]),                      \
+            (d), XENSHARE_writable);                                    \
+        if ((d)->arch.mm.pgd)                                           \
+            ia64_gnttab_create_shared_page(d, t, i);                    \
+    } while (0)
+
+#define gnttab_shared_gmfn(d, t, i)                                     \
+    (IA64_GRANT_TABLE_PADDR >> PAGE_SHIFT) + (i)
 
 #define gnttab_mark_dirty(d, f) ((void)f)
 

# HG changeset patch
# User [email protected]
# Date 1168064285 -32400
# Node ID c8d2889b25dab855003adcc836cd3acf4ddb6245 # Parent 79fd3532f2e4c3c019cd535edf1068bc3cd96e93
PATCHNAME: grow_granttable_ppc

xen/arch/powerpc/mm.c +14 -2
diff -r 79fd3532f2e4 -r c8d2889b25da xen/arch/powerpc/mm.c
@@ -411,7 +411,13 @@ ulong pfn2mfn(struct domain *d, ulong pf
     if (pfn & foreign_map_pfn) {
         t = PFN_TYPE_FOREIGN;
         mfn = foreign_to_mfn(d, pfn);
-    } else if (pfn >= max_page && pfn < (max_page + NR_GRANT_FRAMES)) {
+    } else if (pfn >= max_page && pfn <
+	       (max_page + d->grant_table->nr_grant_frames)) {
+        /* XXX access d->grant_table->nr_grant_frames without lock.
+         * Currently on powerpc dynamic expanding grant table is
+         * inhibited by setting NR_GRANT_FRAMES_MAX = NR_GRANT_FRAMES
+         * so that this access is safe.
+         */
         /* Its a grant table access */
         t = PFN_TYPE_GNTTAB;
         mfn = gnttab_shared_mfn(d, d->grant_table, (pfn - max_page));
@@ -489,9 +495,15 @@ unsigned long mfn_to_gmfn(struct domain 
     ulong gnttab_mfn;
     ulong rma_mfn;
 
+    /* XXX access d->grant_table->nr_grant_frames without lock.
+     * Currently on powerpc dynamic expanding grant table is
+     * inhibited by setting NR_GRANT_FRAMES_MAX = NR_GRANT_FRAMES
+     * so that this access is safe.
+     */
     /* grant? */
     gnttab_mfn = gnttab_shared_mfn(d, d->grant_table, 0);
-    if (mfn >= gnttab_mfn && mfn < (gnttab_mfn + NR_GRANT_FRAMES))
+    if (mfn >= gnttab_mfn && mfn <
+	(gnttab_mfn + d->grant_table->nr_grant_frames))
         return max_page + (mfn - gnttab_mfn);
 
     /* IO? */
xen/include/asm-powerpc/grant_table.h +2 -2
diff -r 79fd3532f2e4 -r c8d2889b25da xen/include/asm-powerpc/grant_table.h
@@ -41,11 +41,11 @@ int destroy_grant_host_mapping(
 #define gnttab_create_shared_page(d, t, i)                               \
     do {                                                                 \
         share_xen_page_with_guest(                                       \
-            virt_to_page((char *)(t)->shared + ((i) * PAGE_SIZE)),       \
+            virt_to_page((t)->shared_frames[(i)]),                       \
             (d), XENSHARE_writable);                                     \
     } while ( 0 )
 
-#define gnttab_shared_mfn(d, t, i) (((ulong)((t)->shared) >> PAGE_SHIFT) + (i))
+#define gnttab_shared_mfn(d, t, i) (virt_to_mfn((t)->shared_frames[(i)]))
 
 #define gnttab_shared_gmfn(d, t, i)                     \
     (mfn_to_gmfn(d, gnttab_shared_mfn(d, t, i)))

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#3 lore raw

Hi Isaku,

The dynamic grant table patch is now in xen-unstable (c/s 13952:4bd0ea9c499). Can you update and resend these patches?

Thanks,
Keir

Quoted 21 lines · Isaku Yamahata
Quoted 3 lines · Keir Fraser

This one is waiting for final review before being checked in. Some people want the functionality this enables (>3 VIFs per guest) so I'm posting the patch in advance of applying it to xen-unstable.

I've been working on the same issue so that I have some patches. I attached a part of them. Although they should be adjusted to your patch which is to be commited, I wanted to send these out before checking in.
If necessary, I'm willing to update them.

- acm code uses NR_GRANT_ENTRIES.
The attached patch is only compile-tested.

- IA64 code needs a slight domain initialization adjustment.

- PPC code depends on the fact that shared grant table is machine continguous. I'm not familiar with PPC code so that I'm not sure how to solve it. If arch specific code can override maximal grant table size, Xen/PPC can prohibit growing grant table.
After solving the issue, they can allow to grow grant table.

#4 lore raw
Quoted 2 lines · Keir Fraser

The dynamic grant table patch is now in xen-unstable (c/s 13952:4bd0ea9c499). Can you update and resend these patches?

Will do. Probably next week.

Signature · 1 line
yamahata
#5 lore raw
In reply to Keir Fraser

Attached the updated patches. Please find them

- 13985_4fa9b86d41b8_grow_granttable_acm.patch
acm part. I did only compile test.

- 13986_cca098fd122c_grow_granttable_ia64.patch

	ia64 part.

- 13987_f5f232aef7bd_grow_granttable_powerpc.patch powerpc part. This patch prohibits growing grant table. I didn't even compile test.
It needs review by xen/powerpc developers and they may have their own opinions.

Thanks,

Quoted 40 lines · Keir Fraser

Hi Isaku,

The dynamic grant table patch is now in xen-unstable (c/s 13952:4bd0ea9c499). Can you update and resend these patches?

Thanks,
Keir

Quoted 14 lines · Isaku Yamahata
Quoted 3 lines · Keir Fraser

This one is waiting for final review before being checked in. Some people want the functionality this enables (>3 VIFs per guest) so I'm posting the patch in advance of applying it to xen-unstable.

I've been working on the same issue so that I have some patches. I attached a part of them. Although they should be adjusted to your patch which is to be commited, I wanted to send these out before checking in.
If necessary, I'm willing to update them.

- acm code uses NR_GRANT_ENTRIES.
The attached patch is only compile-tested.

Quoted 7 lines

- IA64 code needs a slight domain initialization adjustment.

- PPC code depends on the fact that shared grant table is machine continguous. I'm not familiar with PPC code so that I'm not sure how to solve it. If arch specific code can override maximal grant table size, Xen/PPC can prohibit growing grant table.
After solving the issue, they can allow to grow grant table.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Signature · 1 line
yamahata

# HG changeset patch
# User [email protected]
# Date 1171867951 -32400
# Node ID 4fa9b86d41b8faeeb4338dabe62956ecb359b475 # Parent b5fc88aad1b0eb35d12e503982c70fdc27f0544a acm catch up for expandable grant table. only compile-test is done.
PATCHNAME: grow_granttable_acm

xen/acm/acm_simple_type_enforcement_hooks.c +8 -7
diff -r b5fc88aad1b0 -r 4fa9b86d41b8 xen/acm/acm_simple_type_enforcement_hooks.c
@@ -234,18 +234,16 @@ ste_init_state(struct acm_ste_policy_buf
             }
         } 
         /* b) check for grant table conflicts on shared pages */
-        if ((*pd)->grant_table->shared == NULL) {
-            printkd("%s: Grant ... sharing for domain %x not setup!\n", __func__, (*pd)->domain_id);
-            continue;
-        }
-        for ( i = 0; i < NR_GRANT_ENTRIES; i++ ) {
-            sha_copy =  (*pd)->grant_table->shared[i];
+        spin_lock(&(*pd)->grant_table->lock);
+        for ( i = 0; i < nr_grant_entries((*pd)->grant_table); i++ ) {
+            sha_copy = shared_entry((*pd)->grant_table, i);
             if ( sha_copy.flags ) {
                 printkd("%s: grant dom (%hu) SHARED (%d) flags:(%hx) dom:(%hu) frame:(%lx)\n",
                         __func__, (*pd)->domain_id, i, sha_copy.flags, sha_copy.domid, 
                         (unsigned long)sha_copy.frame);
                 rdomid = sha_copy.domid;
                 if ((rdom = get_domain_by_id(rdomid)) == NULL) {
+                    spin_unlock(&(*pd)->grant_table->lock);
                     printkd("%s: domain not found ERROR!\n", __func__);
                     goto out;
                 };
@@ -253,14 +251,17 @@ ste_init_state(struct acm_ste_policy_buf
                 ste_rssid = GET_SSIDP(ACM_SIMPLE_TYPE_ENFORCEMENT_POLICY, 
                                       (struct acm_ssid_domain *)(rdom->ssid));
                 ste_rssidref = ste_rssid->ste_ssidref;
-                put_domain(rdom);
                 if (!have_common_type(ste_ssidref, ste_rssidref)) {
+                    spin_unlock(&(*pd)->grant_table->lock);
                     printkd("%s: Policy violation in grant table sharing domain %x -> domain %x.\n",
                             __func__, (*pd)->domain_id, rdomid);
+                    put_domain(rdom);
                     goto out;
                 }
+                put_domain(rdom);
             }
         }
+        spin_unlock(&(*pd)->grant_table->lock);
     }
     violation = 0;
  out:
xen/common/grant_table.c +0 -3
diff -r b5fc88aad1b0 -r 4fa9b86d41b8 xen/common/grant_table.c
@@ -100,9 +100,6 @@ nr_active_grant_frames(struct grant_tabl
     return num_act_frames_from_sha_frames(nr_grant_frames(gt));
 }
 
-#define SHGNT_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_t))
-#define shared_entry(t, e) \
-    ((t)->shared[(e)/SHGNT_PER_PAGE][(e)%SHGNT_PER_PAGE])
 #define ACGNT_PER_PAGE (PAGE_SIZE / sizeof(struct active_grant_entry))
 #define active_entry(t, e) \
     ((t)->active[(e)/ACGNT_PER_PAGE][(e)%ACGNT_PER_PAGE])
xen/include/xen/grant_table.h +5 -0
diff -r b5fc88aad1b0 -r 4fa9b86d41b8 xen/include/xen/grant_table.h
@@ -120,4 +120,9 @@ static inline unsigned int nr_grant_entr
     return (nr_grant_frames(gt) << PAGE_SHIFT) / sizeof(grant_entry_t);
 }
 
+/* for acm_cimple_type_enforcement_hooks.c */
+#define SHGNT_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_t))
+#define shared_entry(t, e) \
+    ((t)->shared[(e)/SHGNT_PER_PAGE][(e)%SHGNT_PER_PAGE])
+
 #endif /* __XEN_GRANT_TABLE_H__ */

# HG changeset patch
# User [email protected]
# Date 1171869376 -32400
# Node ID cca098fd122c9568b67e4ffec5d0c992003e37b9 # Parent 4fa9b86d41b8faeeb4338dabe62956ecb359b475
GNTTAB: ia64 part expandable grant table
PATCHNAME: grow_granttable_ia64

xen/arch/ia64/xen/domain.c +10 -0
diff -r 4fa9b86d41b8 -r cca098fd122c xen/arch/ia64/xen/domain.c
@@ -521,6 +521,16 @@ int arch_domain_create(struct domain *d)
 	if ((d->arch.mm.pgd = pgd_alloc(&d->arch.mm)) == NULL)
 	    goto fail_nomem;
 
+	/*
+	 * grant_table_create() can't fully initialize grant table for domain
+	 * because it is called before arch_domain_create().
+	 * Here we complete the initialization which requires p2m table.
+	 */
+	spin_lock(&d->grant_table->lock);
+	for (i = 0; i < nr_grant_frames(d->grant_table); i++)
+		ia64_gnttab_create_shared_page(d, d->grant_table, i);
+	spin_unlock(&d->grant_table->lock);
+
 	d->arch.ioport_caps = rangeset_new(d, "I/O Ports",
 	                                   RANGESETF_prettyprint_hex);
 
xen/include/asm-ia64/grant_table.h +25 -16

54 lines not on this page · raw message

# HG changeset patch
# User [email protected]
# Date 1171871352 -32400
# Node ID f5f232aef7bdfabd4997f45d88620dc45b50c185 # Parent cca098fd122c9568b67e4ffec5d0c992003e37b9
GNTTAB: powerpc part expandable grant table. Currently it is inhibited to grow grant table on powerpc. Patch only. Even compilation isn't tested.
PATCHNAME: grow_granttable_powerpc

xen/arch/powerpc/mm.c +14 -2
diff -r cca098fd122c -r f5f232aef7bd xen/arch/powerpc/mm.c
@@ -412,7 +412,13 @@ ulong pfn2mfn(struct domain *d, ulong pf
     if (pfn & foreign_map_pfn) {
         t = PFN_TYPE_FOREIGN;
         mfn = foreign_to_mfn(d, pfn);
-    } else if (pfn >= max_page && pfn < (max_page + NR_GRANT_FRAMES)) {
+    } else if (pfn >= max_page && pfn <
+	       (max_page + nr_grant_frames(d->grant_table))) {
+        /* XXX access d->grant_table->nr_grant_frames without lock.
+         * Currently on powerpc dynamic expanding grant table is
+         * inhibited by setting max_nr_grant_frames = INITIAL_NR_GRANT_FRAMES
+         * so that this access is safe.
+         */
         /* Its a grant table access */
         t = PFN_TYPE_GNTTAB;
         mfn = gnttab_shared_mfn(d, d->grant_table, (pfn - max_page));
@@ -490,9 +496,15 @@ unsigned long mfn_to_gmfn(struct domain 
     ulong gnttab_mfn;
     ulong rma_mfn;
 
+    /* XXX access d->grant_table->nr_grant_frames without lock.
+     * Currently on powerpc dynamic expanding grant table is
+     * inhibited by setting max_nr_grant_frames = INITIAL_NR_GRANT_FRAMES
+     * so that this access is safe.
+     */
     /* grant? */
     gnttab_mfn = gnttab_shared_mfn(d, d->grant_table, 0);
-    if (mfn >= gnttab_mfn && mfn < (gnttab_mfn + NR_GRANT_FRAMES))
+    if (mfn >= gnttab_mfn && mfn <
+	(gnttab_mfn + nr_grant_frames(d->grant_table)))
         return max_page + (mfn - gnttab_mfn);
 
     /* IO? */
xen/common/grant_table.c +2 -0
diff -r cca098fd122c -r f5f232aef7bd xen/common/grant_table.c
@@ -35,8 +35,10 @@
 #include <xen/domain_page.h>
 #include <acm/acm_hooks.h>
 
+#ifndef max_nr_grant_frames
 unsigned int max_nr_grant_frames = DEFAULT_MAX_NR_GRANT_FRAMES;
 integer_param("gnttab_max_nr_frames", max_nr_grant_frames);
+#endif
 
 /* The maximum number of grant mappings is defined as a multiplier of the
  * maximum number of grant table entries. This defines the multiplier used.
xen/include/asm-powerpc/config.h +9 -0
diff -r cca098fd122c -r f5f232aef7bd xen/include/asm-powerpc/config.h
@@ -73,4 +73,13 @@ extern char __bss_start[];
 
 #include <asm/powerpc64/config.h>
 
+/*
+ * Disallow grant table growing tempralily because pfn2mfn() and
+ * mfn_to_gmfn() depends on the fact that grant table is machine-address
+ * contiguous. Grant table growing breaks the assumption.
+ */
+#ifndef max_nr_grant_frames
+#define max_nr_grant_frames INITIAL_NR_GRANT_FRAMES
 #endif
+
+#endif
xen/include/asm-powerpc/grant_table.h +2 -2
diff -r cca098fd122c -r f5f232aef7bd xen/include/asm-powerpc/grant_table.h
@@ -41,11 +41,11 @@ int destroy_grant_host_mapping(
 #define gnttab_create_shared_page(d, t, i)                               \
     do {                                                                 \
         share_xen_page_with_guest(                                       \
-            virt_to_page((char *)(t)->shared + ((i) * PAGE_SIZE)),       \
+            virt_to_page((t)->shared_frames[(i)]),                       \
             (d), XENSHARE_writable);                                     \
     } while ( 0 )
 
-#define gnttab_shared_mfn(d, t, i) (((ulong)((t)->shared) >> PAGE_SHIFT) + (i))
+#define gnttab_shared_mfn(d, t, i) (virt_to_mfn((t)->shared_frames[(i)]))
 
 #define gnttab_shared_gmfn(d, t, i)                     \
     (mfn_to_gmfn(d, gnttab_shared_mfn(d, t, i)))
xen/include/xen/grant_table.h +4 -0
diff -r cca098fd122c -r f5f232aef7bd xen/include/xen/grant_table.h
@@ -56,10 +56,14 @@ struct active_grant_entry {
 #define INITIAL_NR_GRANT_ENTRIES ((INITIAL_NR_GRANT_FRAMES << PAGE_SHIFT) / \
                                      sizeof(grant_entry_t))
 
+#ifndef DEFAULT_MAX_NR_GRANT_FRAMES /* to allow arch to override */
 /* Default maximum size of a grant table. [POLICY] */
 #define DEFAULT_MAX_NR_GRANT_FRAMES   32
+#endif
+#ifndef max_nr_grant_frames /* to allow arch to override */
 /* The maximum size of a grant table. */
 extern unsigned int max_nr_grant_frames;
+#endif
 
 /*
  * Tracks a mapping of another domain's grant reference. Each domain has a

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#6 lore raw
Quoted 5 lines · Isaku Yamahata

Attached the updated patches. Please find them

- 13985_4fa9b86d41b8_grow_granttable_acm.patch
acm part. I did only compile test.

Quoted 8 lines

- 13986_cca098fd122c_grow_granttable_ia64.patch

	ia64 part.

- 13987_f5f232aef7bd_grow_granttable_powerpc.patch powerpc part. This patch prohibits growing grant table. I didn't even compile test.
It needs review by xen/powerpc developers and they may have their own opinions.

Hi Yamahata-san, thanks very much for your patch!

I'm confused about one thing though: why do we need to take a lock to read d->grant_table->nr_grant_frames? It's a simple integer, so no lock is required or useful.

Signature · 2 lines
Hollis Blanchard
IBM Linux Technology Center
#7 lore raw
Quoted 6 lines · Hollis Blanchard

Hi Yamahata-san, thanks very much for your patch!

I'm confused about one thing though: why do we need to take a lock to read d->grant_table->nr_grant_frames? It's a simple integer, so no lock is required or useful.

I haven't got the ppc code immediately to hand but the x86 code will dynamically grow the grant table based on requests made via the add_to_physmap hypercall, hence it needs to take the lock. I see ia64 takes it also even though it only reads from nr_grant_frames (it won;t dynamically expand via the add_to_physmap hypercall). That's possibly overkill although there is some concern over reading nr_grant_frames versus looking up a grant-table page that appears within the range [0, nr_grant_frames-1] which taking the lock trivially sidesteps. If ia64 didn't take the lock it might be possible to see an increased value of nr_grant_frames that the expand function hadn't yet fully installed the new grant-table pages for yet.

-- Keir

#8 lore raw
Quoted 5 lines · Keir Fraser
Quoted 3 lines · Hollis Blanchard

Hi Yamahata-san, thanks very much for your patch!

Quoted 7 lines
Quoted 3 lines

I'm confused about one thing though: why do we need to take a lock to read d->grant_table->nr_grant_frames? It's a simple integer, so no lock is required or useful.

I haven't got the ppc code immediately to hand but the x86 code will dynamically grow the grant table based on requests made via the add_to_physmap hypercall, hence it needs to take the lock.

OK, I see x86's XENMAPSPACE_grant_table handler; the locking makes sense there.

Quoted 8 lines

I see ia64 takes
it also even though it only reads from nr_grant_frames (it won;t dynamically expand via the add_to_physmap hypercall). That's possibly overkill although there is some concern over reading nr_grant_frames versus looking up a grant-table page that appears within the range [0, nr_grant_frames-1] which taking the lock trivially sidesteps. If ia64 didn't take the lock it might be possible to see an increased value of nr_grant_frames that the expand function hadn't yet fully installed the new grant-table pages for yet.

I don't believe that's a concern, since updating grant_table->nr_grant_frames is the very last step in gnttab_grow_table(), and it will only grow.

The comments about locking around nr_grant_frames() and nr_grant_entries() are confusing at best, since you don't need a lock to read an integer...

Signature · 2 lines
Hollis Blanchard
IBM Linux Technology Center
#9 lore raw
Quoted 3 lines · Hollis Blanchard

I don't believe that's a concern, since updating grant_table->nr_grant_frames is the very last step in gnttab_grow_table(), and it will only grow.

If there's a memory barrier before the update of nr_grant_frames, explicitly or implied, then removing the locking from add_to_physmap is fine. Otherwise
not: reading an integer is atomic, but using that as a flag to indicate presence of other state updated under the same lock is just a little bit suspect (but might be okay).

-- Keir