GRU - GRU Updates - Production Driver

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
@@ -217,8 +217,6 @@ struct gru_instruction {
 #define EOP_ERR_XOR	0x03 /* Logical XOR with memory */
 #define EOP_ERR_ADD	0x04 /* Add value to memory */
 #define EOP_ERR_CSWAP	0x08 /* Compare with operand2, write operand1 if match*/
-#define EOP_ERR_EPOLL	0x09 /* Poll for equality */
-#define EOP_ERR_NPOLL	0x0a /* Poll for inequality */
 
 /* GAMXR - SGI Arithmetic unit */
 #define EOP_XR_CSWAP	0x0b /* Masked compare exchange */

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -57,11 +57,11 @@ static inline int is_gru_paddr(unsigned
 /*
  * Find the vma of a GRU segment. Caller must hold mmap_sem.
  */
-struct vm_area_struct *gru_find_vma(unsigned long vaddr)
+struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr)
 {
 	struct vm_area_struct *vma;
 
-	vma = find_vma(current->mm, vaddr);
+	vma = find_vma(mm, vaddr);
 	if (vma && vma->vm_start <= vaddr && vma->vm_ops == &gru_vm_ops)
 		return vma;
 	return NULL;

@@ -82,7 +82,7 @@ static struct gru_thread_state *gru_find
 	struct gru_thread_state *gts = NULL;
 
 	down_read(&mm->mmap_sem);
-	vma = gru_find_vma(vaddr);
+	vma = gru_find_vma(current->mm, vaddr);
 	if (vma)
 		gts = gru_find_thread_state(vma, TSID(vaddr, vma));
 	if (gts)

@@ -99,7 +99,7 @@ static struct gru_thread_state *gru_allo
 	struct gru_thread_state *gts = ERR_PTR(-EINVAL);
 
 	down_write(&mm->mmap_sem);
-	vma = gru_find_vma(vaddr);
+	vma = gru_find_vma(current->mm, vaddr);
 	if (!vma)
 		goto err;
 

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -145,7 +145,7 @@ static int gru_create_new_context(unsign
 		req.options |= GRU_OPT_MISS_FMM_INTR;
 
 	down_write(&current->mm->mmap_sem);
-	vma = gru_find_vma(req.gseg);
+	vma = gru_find_vma(current->mm, req.gseg);
 	if (vma) {
 		vdata = vma->vm_private_data;
 		vdata->vd_user_options = req.options;

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
@@ -87,6 +87,7 @@ static int statistics_show(struct seq_fi
 	printstat(s, tlb_dropin_fail_no_exception);
 	printstat(s, tfh_stale_on_fault);
 	printstat(s, mmu_invalidate_range);
+	printstat(s, mmu_invalidate_range_flush);
 	printstat(s, mmu_invalidate_page);
 	printstat(s, flush_tlb);
 	printstat(s, flush_tlb_gru);

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -213,6 +213,7 @@ struct gru_stats_s {
 	atomic_long_t tlb_dropin_fail_no_exception;
 	atomic_long_t tfh_stale_on_fault;
 	atomic_long_t mmu_invalidate_range;
+	atomic_long_t mmu_invalidate_range_flush;
 	atomic_long_t mmu_invalidate_page;
 	atomic_long_t flush_tlb;
 	atomic_long_t flush_tlb_gru;

@@ -653,7 +654,7 @@ extern int gru_get_exception_detail(unsi
 extern int gru_set_context_option(unsigned long address);
 extern void gru_check_context_placement(struct gru_thread_state *gts);
 extern int gru_cpu_fault_map_id(void);
-extern struct vm_area_struct *gru_find_vma(unsigned long vaddr);
+extern struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr);
 extern void gru_flush_all_tlb(struct gru_state *gru);
 extern int gru_proc_init(void);
 extern void gru_proc_exit(void);

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
@@ -226,11 +226,19 @@ static void gru_invalidate_range_start(s
 	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
 						 ms_notifier);
 
+	if (mm && atomic_read(&mm->mm_users) && unlikely(down_write_trylock(&mm->mmap_sem))) {
+		up_write(&mm->mmap_sem);
+	}
 	STAT(mmu_invalidate_range);
 	atomic_inc(&gms->ms_range_active);
 	gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx, act %d\n", gms,
 		start, end, atomic_read(&gms->ms_range_active));
-	gru_flush_tlb_range(gms, start, end - start);
+
+	/* No need to flush if unmapping a GRU context */
+	if (!gru_find_vma(mm, start)) {
+		STAT(mmu_invalidate_range_flush);
+		gru_flush_tlb_range(gms, start, end - start);
+	}
 }
 
 static void gru_invalidate_range_end(struct mmu_notifier *mn,

@@ -242,7 +250,6 @@ static void gru_invalidate_range_end(str
 
 	/* ..._and_test() provides needed barrier */
 	(void)atomic_dec_and_test(&gms->ms_range_active);
-
 	wake_up_all(&gms->ms_wait_queue);
 	gru_dbg(grudev, "gms %p, start 0x%lx, end 0x%lx\n", gms, start, end);
 }

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
@@ -61,7 +61,8 @@ struct gru_gseg_statistics {
 	unsigned long	upm_tlbmiss;
 	unsigned long	tlbdropin;
 	unsigned long	context_stolen;
-	unsigned long	reserved[10];
+	unsigned long	interrupts;
+	unsigned long	reserved[9];
 };
 
 /* Flags for GRU options on the gru_create_context() call */

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -289,8 +289,10 @@ static int gru_vtop(struct gru_thread_st
 	return VTOP_SUCCESS;
 
 inval:
+	STAT(tlb_dropin_fail_invalid);
 	return VTOP_INVALID;
 upm:
+	STAT(tlb_dropin_fail_retry);
 	return VTOP_RETRY;
 }
 

@@ -422,7 +424,7 @@ static int gru_try_dropin(struct gru_sta
 	if (ret == VTOP_INVALID)
 		goto failinval;
 	if (ret == VTOP_RETRY)
-		goto failupm;
+		goto failretry;
 
 	if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) {
 		gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift);

@@ -438,9 +440,9 @@ static int gru_try_dropin(struct gru_sta
 	}
 
 	gru_cb_set_istatus_active(cbk);
-	gts->ustats.tlbdropin++;
 	tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
 			  GRU_PAGESIZE(pageshift));
+	gts->ustats.tlbdropin++;
 	gru_dbg(grudev,
 		"%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
 		" rw %d, ps %d, gpa 0x%lx\n",

@@ -453,20 +455,33 @@ failnoasid:
 	/* No asid (delayed unload). */
 	STAT(tlb_dropin_fail_no_asid);
 	gru_dbg(grudev, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
-	if (!cbk)
+	if (atomic)
 		tfh_user_polling_mode(tfh);
 	else
 		gru_flush_cache(tfh);
 	gru_flush_cache_cbe(cbe);
 	return -EAGAIN;
 
+failretry:
+	/* vtop failed - retry */
+	if (atomic)
+		tfh_user_polling_mode(tfh);
+	else
+		gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
+	gru_dbg(grudev, "FAILED retry tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
+	return -EAGAIN;
+
 failupm:
 	/* Atomic failure switch CBR to UPM */
-	tfh_user_polling_mode(tfh);
+	if (atomic)
+		tfh_user_polling_mode(tfh);
+	else
+		gru_flush_cache(tfh);
 	gru_flush_cache_cbe(cbe);
 	STAT(tlb_dropin_fail_upm);
 	gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
-	return 1;
+	return -EAGAIN;
 
 failfmm:
 	/* FMM state on UPM call */

@@ -501,13 +516,12 @@ failinval:
 	/* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */
 	tfh_exception(tfh);
 	gru_flush_cache_cbe(cbe);
-	STAT(tlb_dropin_fail_invalid);
 	gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
 	return -EFAULT;
 
 failactive:
 	/* Range invalidate active. Switch to UPM iff atomic */
-	if (!cbk)
+	if (atomic)
 		tfh_user_polling_mode(tfh);
 	else
 		gru_flush_cache(tfh);

@@ -531,7 +545,7 @@ static irqreturn_t gru_intr(int chiplet,
 	struct gru_thread_state *gts;
 	struct gru_tlb_fault_handle *tfh = NULL;
 	struct completion *cmp;
-	int cbrnum, ctxnum;
+	int cbrnum, ctxnum, multi = 0;
 
 	STAT(intr);
 

@@ -581,7 +595,10 @@ static irqreturn_t gru_intr(int chiplet,
 		 * This is running in interrupt context. Trylock the mmap_sem.
 		 * If it fails, retry the fault in user context.
 		 */
+		if (!multi)
+			gts->ustats.interrupts++;
 		gts->ustats.fmm_tlbmiss++;
+		multi = 1;
 		if (!gts->ts_force_cch_reload &&
 					down_read_trylock(&gts->ts_mm->mmap_sem)) {
 			gru_try_dropin(gru, gts, tfh, NULL);

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
@@ -79,6 +79,7 @@ static int statistics_show(struct seq_fi
 	printstat(s, tlb_dropin);
 	printstat(s, tlb_preload_page);
 	printstat(s, tlb_dropin_fail_no_asid);
+	printstat(s, tlb_dropin_fail_retry);
 	printstat(s, tlb_dropin_fail_upm);
 	printstat(s, tlb_dropin_fail_invalid);
 	printstat(s, tlb_dropin_fail_range_active);

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -205,6 +205,7 @@ struct gru_stats_s {
 	atomic_long_t tlb_dropin;
 	atomic_long_t tlb_preload_page;
 	atomic_long_t tlb_dropin_fail_no_asid;
+	atomic_long_t tlb_dropin_fail_retry;
 	atomic_long_t tlb_dropin_fail_upm;
 	atomic_long_t tlb_dropin_fail_invalid;
 	atomic_long_t tlb_dropin_fail_range_active;

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -110,7 +110,7 @@ static int gru_file_mmap(struct file *fi
 
 	vma->vm_flags |=
 	    (VM_IO | VM_DONTCOPY | VM_LOCKED | VM_DONTEXPAND | VM_PFNMAP |
-			VM_RESERVED);
+			VM_RESERVED | VM_NONLINEAR);
 	vma->vm_page_prot = PAGE_SHARED;
 	vma->vm_ops = &gru_vm_ops;
 

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -306,7 +306,7 @@ upm:
 static void gru_flush_cache_cbe(struct gru_control_block_extended *cbe)
 {
 	if (unlikely(cbe)) {
-		cbe->cbrexecstatus = 0;         /* make CL dirty */
+		cbe->reserved6 = 0;         /* make CL dirty */
 		gru_flush_cache(cbe);
 	}
 }

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -564,7 +564,7 @@ static irqreturn_t gru_intr(int chiplet,
 
 	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
 		STAT(intr_cbr);
-		cmp = gru->gs_blade->bs_async_wq;
+		cmp = gru->gs_async_wq[cbrnum];
 		if (cmp)
 			complete(cmp);
 		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",

524 lines not on this page · raw message

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
@@ -52,9 +52,15 @@ static int statistics_show(struct seq_fi
 	printstat(s, assign_context_failed);
 	printstat(s, free_context);
 	printstat(s, load_user_context);
-	printstat(s, load_kernel_context);
-	printstat(s, lock_kernel_context);
-	printstat(s, unlock_kernel_context);
+	printstat(s, load_kcontext);
+	printstat(s, load_kcontext_assign);
+	printstat(s, load_kcontext_steal);
+	printstat(s, lock_kcontext);
+	printstat(s, unlock_kcontext);
+	printstat(s, get_kcontext_cbr);
+	printstat(s, get_kcontext_cbr_busy);
+	printstat(s, lock_async_resource);
+	printstat(s, unlock_async_resource);
 	printstat(s, steal_user_context);
 	printstat(s, steal_kernel_context);
 	printstat(s, steal_context_failed);

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -163,6 +163,8 @@ extern unsigned int gru_max_gids;
 #define GRU_DRIVER_ID_STR	"SGI GRU Device Driver"
 #define GRU_DRIVER_VERSION_STR	"0.85"
 
+#define gru_random()    get_cycles()
+
 /*
  * GRU statistics.
  */

@@ -178,9 +180,15 @@ struct gru_stats_s {
 	atomic_long_t assign_context_failed;
 	atomic_long_t free_context;
 	atomic_long_t load_user_context;
-	atomic_long_t load_kernel_context;
-	atomic_long_t lock_kernel_context;
-	atomic_long_t unlock_kernel_context;
+	atomic_long_t load_kcontext;
+	atomic_long_t load_kcontext_assign;
+	atomic_long_t load_kcontext_steal;
+	atomic_long_t lock_kcontext;
+	atomic_long_t unlock_kcontext;
+	atomic_long_t get_kcontext_cbr;
+	atomic_long_t get_kcontext_cbr_busy;
+	atomic_long_t lock_async_resource;
+	atomic_long_t unlock_async_resource;
 	atomic_long_t steal_user_context;
 	atomic_long_t steal_kernel_context;
 	atomic_long_t steal_context_failed;

@@ -443,14 +451,11 @@ struct gru_state {
 							   resources */
 	unsigned long		gs_dsr_map;		/* bitmap used to manage
 							   DATA resources */
-	unsigned int		gs_reserved_cbrs;	/* Number of kernel-
-							   reserved cbrs */
-	unsigned int		gs_reserved_dsr_bytes;	/* Bytes of kernel-
-							   reserved dsrs */
 	unsigned short		gs_active_contexts;	/* number of contexts
 							   in use */
 	struct gru_thread_state	*gs_gts[GRU_NUM_CCH];	/* GTS currently using
 							   the context */
+	struct completion	*gs_async_wq[GRU_NUM_CB];
 	int			gs_irq[GRU_NUM_TFM];	/* Interrupt irqs */
 };
 

@@ -466,8 +471,7 @@ struct gru_blade_state {
 	struct gru_thread_state *bs_kgts;		/* GTS for kernel use */
 
 	/* ---- the following are used for managing kernel async GRU CBRs --- */
-	int			bs_async_dsr_bytes;	/* DSRs for async */
-	int			bs_async_cbrs;		/* CBRs AU for async */
+	void			*bs_async_cbr;		/* CBR for async */
 	struct completion	*bs_async_wq;
 
 	/* ---- the following are protected by the bs_lock spinlock ---- */

@@ -558,11 +562,24 @@ struct gru_blade_state {
 
 /*-----------------------------------------------------------------------------
  * Lock / Unlock GRU handles
- * 	Use the "delresp" bit in the handle as a "lock" bit.
+ * 	Use the "delresp" bit in MCS handles as a "lock" bit.
+ * 	Use the "unmapped" bit in CBRs as a "lock" bit.
+ *
+ *		Return: 0 = lock failed, 1 = locked
  */
 
 /* Lock hierarchy checking enabled only in emulator */
 
+static inline int trylock_cbr_handle(void *h)
+{
+	return !test_and_set_bit(2, h);
+}
+
+static inline void unlock_cbr_handle(void *h)
+{
+	clear_bit(2, h);
+}
+
 /* 0 = lock failed, 1 = locked */
 static inline int __trylock_handle(void *h)
 {

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
@@ -40,8 +40,6 @@
 #include "grutables.h"
 #include <asm/uv/uv_hub.h>
 
-#define gru_random()	get_cycles()
-
 /* ---------------------------------- TLB Invalidation functions --------
  * get_tgh_handle
  *

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
@@ -190,21 +190,22 @@ static int cch_seq_show(struct seq_file
 	const char *mode[] = { "??", "UPM", "INTR", "OS_POLL" };
 
 	if (gid == 0)
-		seq_printf(file, "#%5s%5s%6s%7s%9s%6s%8s%8s\n", "gid", "bid",
-			   "ctx#", "asid", "pid", "cbrs", "dsbytes", "mode");
+		seq_printf(file, "#%5s%5s%6s%7s%9s%6s%8s%8s%7s\n", "gid", "bid",
+			   "ctx#", "asid", "pid", "cbrs", "dsbytes", "mode", "locked");
 	if (gru)
 		for (i = 0; i < GRU_NUM_CCH; i++) {
 			ts = gru->gs_gts[i];
 			if (!ts)
 				continue;
-			seq_printf(file, " %5d%5d%6d%7d%9d%6d%8d%8s\n",
+			seq_printf(file, " %5d%5d%6d%7d%9d%6d%8d%8s%7s\n",
 				   gru->gs_gid, gru->gs_blade_id, i,
 				   is_kernel_context(ts) ? 0 : ts->ts_gms->ms_asids[gid].mt_asid,
 				   is_kernel_context(ts) ? 0 : ts->ts_tgid_owner,
 				   ts->ts_cbr_au_count * GRU_CBR_AU_SIZE,
-				   ts->ts_cbr_au_count * GRU_DSR_AU_BYTES,
+				   ts->ts_dsr_au_count * GRU_DSR_AU_BYTES,
 				   mode[ts->ts_user_options &
-					GRU_OPT_MISS_MASK]);
+					GRU_OPT_MISS_MASK],
+				   mutex_is_locked(&ts->ts_ctxlock) ? "LOCKED" : "-");
 		}
 
 	return 0;

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
@@ -380,4 +380,6 @@ void gru_tgh_flush_init(struct gru_state
 	/* first starting TGH index to use for remote purges */
 	gru->gs_tgh_first_remote = (cpus + (1 << shift) - 1) >> shift;
 
+	/* flush the GRU TLB in case there are stale entries present */
+	gru_flush_all_tlb(gru);
 }

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
@@ -112,7 +112,8 @@ struct gru_instruction_bits {
     unsigned char		reserved3: 1;
     unsigned char		tlb_fault_color: 1;
     /* DW 1 */
-    unsigned long		idef4;		/* 42 bits: TRi1, BufSize */
+    unsigned int		idef4;		/* 32 bits: TRi1, BufSize */
+    unsigned int		sw_reserved;
     /* DW 2-6 */
     unsigned long		idef1;		/* BAddr0 */
     unsigned long		idef5;		/* Nelem */

@@ -136,7 +137,13 @@ struct gru_instruction {
 		unsigned int	tri0;
 	};
     };
-    unsigned long		tri1_bufsize;		/* DW 1 */
+    union {
+	unsigned long		tri1_bufsize_64;	/* DW 1 */
+	struct {
+    		unsigned int	tri1_bufsize_32;
+		unsigned int	sw_reserved;
+	};
+    };
     unsigned long		baddr0;			/* DW 2 */
     unsigned long		nelem;			/* DW 3 */
     unsigned long		op1_stride;		/* DW 4 */

@@ -411,7 +418,7 @@ static inline void gru_ivload(void *cb,
 
 	ins->baddr0 = (long)mem_addr;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = tri1;
+	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVLOAD, 0, xtype, IAA_RAM, 0,
 					tri0, CB_IMA(hints)));
 }

@@ -424,7 +431,7 @@ static inline void gru_ivstore(void *cb,
 
 	ins->baddr0 = (long)mem_addr;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = tri1;
+	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSTORE, 0, xtype, IAA_RAM, 0,
 					tri0, CB_IMA(hints)));
 }

@@ -452,7 +459,7 @@ static inline void gru_ivset(void *cb, u
 	ins->baddr0 = (long)mem_addr;
 	ins->op2_value_baddr1 = value;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = tri1;
+	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSET, 0, xtype, IAA_RAM, 0,
 					0, CB_IMA(hints)));
 }

@@ -488,7 +495,7 @@ static inline void gru_bcopy(void *cb, c
 	ins->baddr0 = (long)src;
 	ins->op2_value_baddr1 = (long)dest;
 	ins->nelem = nelem;
-	ins->tri1_bufsize = bufsize;
+	ins->tri1_bufsize_64 = bufsize;
 	gru_start_instruction(ins, __opdword(OP_BCOPY, 0, xtype, IAA_RAM,
 					IAA_RAM, tri0, CB_IMA(hints)));
 }

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -98,7 +98,7 @@ static struct gru_thread_state *gru_allo
 	struct vm_area_struct *vma;
 	struct gru_thread_state *gts = ERR_PTR(-EINVAL);
 
-	down_write(&mm->mmap_sem);
+	down_read(&mm->mmap_sem);
 	vma = gru_find_vma(current->mm, vaddr);
 	if (!vma)
 		goto err;

@@ -107,11 +107,10 @@ static struct gru_thread_state *gru_allo
 	if (IS_ERR(gts))
 		goto err;
 	mutex_lock(&gts->ts_ctxlock);
-	downgrade_write(&mm->mmap_sem);
 	return gts;
 
 err:
-	up_write(&mm->mmap_sem);
+	up_read(&mm->mmap_sem);
 	return gts;
 }
 

@@ -764,18 +763,20 @@ static int gru_unload_all_contexts(void)
 	struct gru_thread_state *gts;
 	struct gru_state *gru;
 	int gid, ctxnum;
+	struct gru_blade_state *blade;
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 	foreach_gid(gid) {
 		gru = GID_TO_GRU(gid);
+		blade = gru->gs_blade;
 		spin_lock(&gru->gs_lock);
 		for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
 			gts = gru->gs_gts[ctxnum];
-			if (gts && mutex_trylock(&gts->ts_ctxlock)) {
+			if (gts && gru_is_gts_stealable(gts, blade)) {
 				spin_unlock(&gru->gs_lock);
 				gru_unload_context(gts, 1);
-				mutex_unlock(&gts->ts_ctxlock);
+				gru_gts_stolen(gts, blade);
 				spin_lock(&gru->gs_lock);
 			}
 		}

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -88,6 +88,7 @@ static void gru_vma_close(struct vm_area
 		mutex_unlock(&gts->ts_ctxlock);
 		gts_drop(gts);
 	}
+	gru_drop_mmu_notifier(vdata->vd_gms);
 	kfree(vdata);
 	STAT(vdata_free);
 }

@@ -101,6 +102,8 @@ static void gru_vma_close(struct vm_area
  */
 static int gru_file_mmap(struct file *file, struct vm_area_struct *vma)
 {
+	struct gru_vma_data *vdata;
+
 	if ((vma->vm_flags & (VM_SHARED | VM_WRITE)) != (VM_SHARED | VM_WRITE))
 		return -EPERM;
 

@@ -114,9 +117,10 @@ static int gru_file_mmap(struct file *fi
 	vma->vm_page_prot = PAGE_SHARED;
 	vma->vm_ops = &gru_vm_ops;
 
-	vma->vm_private_data = gru_alloc_vma_data(vma, 0);
-	if (!vma->vm_private_data)
-		return -ENOMEM;
+	vdata = gru_alloc_vma_data(vma, 0);
+	if (IS_ERR(vdata))
+		return PTR_ERR(vdata);
+	vma->vm_private_data = vdata;
 
 	gru_dbg(grudev, "file %p, vaddr 0x%lx, vma %p, vdata %p\n",
 		file, vma->vm_start, vma, vma->vm_private_data);

@@ -144,18 +148,20 @@ static int gru_create_new_context(unsign
 	if (!(req.options & GRU_OPT_MISS_MASK))
 		req.options |= GRU_OPT_MISS_FMM_INTR;
 
-	down_write(&current->mm->mmap_sem);
+	down_read(&current->mm->mmap_sem);
 	vma = gru_find_vma(current->mm, req.gseg);
 	if (vma) {
 		vdata = vma->vm_private_data;
+		spin_lock(&vdata->vd_lock);
 		vdata->vd_user_options = req.options;
 		vdata->vd_dsr_au_count =
 		    GRU_DS_BYTES_TO_AU(req.data_segment_bytes);
 		vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks);
 		vdata->vd_tlb_preload_count = req.tlb_preload_count;
+		spin_unlock(&vdata->vd_lock);
 		ret = 0;
 	}
-	up_write(&current->mm->mmap_sem);
+	up_read(&current->mm->mmap_sem);
 
 	return ret;
 }

Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
@@ -132,11 +132,6 @@ int cch_deallocate(struct gru_context_co
 	start_instruction(cch);
 	ret = wait_instruction_complete(cch, cchop_deallocate);
 
-	/*
-	 * Stop speculation into the GSEG being unmapped by the previous
-	 * DEALLOCATE.
-	 */
-	sync_core();
 	return ret;
 }
 

124 lines not on this page · raw message

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -115,10 +115,14 @@
  *       task -->|
  *       task ---+---> mm ->------ (notifier) -------+-> gms
  *                     |                             |
+ *                     |              +------------->|
+ *                     |             /               |
  *                     |--> vma -> vdata ---> gts--->|		GSEG1 (thread1)
  *                     |                  |          |
  *                     |                  +-> gts--->|		GSEG1 (thread2)
  *                     |                             |
+ *                     |              +------------->|
+ *                     |             /               |
  *                     |--> vma -> vdata ---> gts--->|		GSEG2 (thread2)
  *                     .
  *                     .

@@ -336,6 +340,7 @@ struct gru_mm_struct {
 	wait_queue_head_t	ms_wait_queue;
 	DECLARE_BITMAP(ms_asidmap, GRU_MAX_GRUS);
 	struct gru_mm_tracker	ms_asids[GRU_MAX_GRUS];
+	struct mm_struct	*ms_mm;		/* for mmu_notifier_unreg */
 };
 
 /*

@@ -345,6 +350,7 @@ struct gru_mm_struct {
 struct gru_vma_data {
 	spinlock_t		vd_lock;	/* Serialize access to vma */
 	struct list_head	vd_head;	/* head of linked list of gts */
+	struct gru_mm_struct	*vd_gms;	/* asid & ioproc struct */
 	long			vd_user_options;/* misc user option flags */
 	int			vd_cbr_au_count;
 	int			vd_dsr_au_count;

@@ -469,7 +475,6 @@ struct gru_blade_state {
 							   reserved DSR */
 	struct rw_semaphore	bs_kgts_sema;		/* lock for kgts */
 	struct gru_thread_state *bs_kgts;		/* GTS for kernel use */
-
 	/* ---- the following are used for managing kernel async GRU CBRs --- */
 	void			*bs_async_cbr;		/* CBR for async */
 	struct completion	*bs_async_wq;

@@ -670,6 +675,10 @@ extern int gru_user_flush_tlb(unsigned l
 extern int gru_user_unload_context(unsigned long arg);
 extern int gru_get_exception_detail(unsigned long arg);
 extern int gru_set_context_option(unsigned long address);
+extern int gru_is_gts_stealable(struct gru_thread_state *gts,
+				struct gru_blade_state *blade);
+extern void gru_gts_stolen(struct gru_thread_state *gts,
+				struct gru_blade_state *blade);
 extern void gru_check_context_placement(struct gru_thread_state *gts);
 extern int gru_cpu_fault_map_id(void);
 extern struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr);

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
@@ -263,21 +263,10 @@ static void gru_invalidate_page(struct m
 	gru_dbg(grudev, "gms %p, address 0x%lx\n", gms, address);
 }
 
-static void gru_release(struct mmu_notifier *mn, struct mm_struct *mm)
-{
-	struct gru_mm_struct *gms = container_of(mn, struct gru_mm_struct,
-						 ms_notifier);
-
-	gms->ms_released = 1;
-	gru_dbg(grudev, "gms %p\n", gms);
-}
-
-
 static const struct mmu_notifier_ops gru_mmuops = {
 	.invalidate_page	= gru_invalidate_page,
 	.invalidate_range_start	= gru_invalidate_range_start,
 	.invalidate_range_end	= gru_invalidate_range_end,
-	.release		= gru_release,
 };
 
 /* Move this to the basic mmu_notifier file. But for now... */

@@ -316,6 +305,7 @@ struct gru_mm_struct *gru_register_mmu_n
 			STAT(gms_alloc);
 			spin_lock_init(&gms->ms_asid_lock);
 			gms->ms_notifier.ops = &gru_mmuops;
+			gms->ms_mm = current->mm;
 			atomic_set(&gms->ms_refcnt, 1);
 			init_waitqueue_head(&gms->ms_wait_queue);
 			err = __mmu_notifier_register(&gms->ms_notifier, current->mm);

@@ -328,6 +318,9 @@ struct gru_mm_struct *gru_register_mmu_n
 	return gms;
 error:
 	kfree(gms);
+	/* mmu_notifier_register EINTR is reported as EAGAIN */
+	if (err == -EINTR)
+		err = -EAGAIN;
 	return ERR_PTR(err);
 }
 

@@ -336,8 +329,7 @@ void gru_drop_mmu_notifier(struct gru_mm
 	gru_dbg(grudev, "gms %p, refcnt %d, released %d\n", gms,
 		atomic_read(&gms->ms_refcnt), gms->ms_released);
 	if (atomic_dec_return(&gms->ms_refcnt) == 0) {
-		if (!gms->ms_released)
-			mmu_notifier_unregister(&gms->ms_notifier, current->mm);
+		mmu_notifier_unregister(&gms->ms_notifier, gms->ms_mm);
 		kfree(gms);
 		STAT(gms_free);
 	}

Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
@@ -618,7 +618,7 @@ void gru_load_context(struct gru_thread_
 		cch->unmap_enable = 1;
 		cch->tfm_done_bit_enable = 1;
 		cch->cb_int_enable = 1;
-		cch->tlb_int_select = 0;	/* For now, ints go to cpu 0 */
+		cch->tlb_int_select = uv_cpu_core_number(0); /* For now, ints go to cpu 0 */
 	} else {
 		cch->unmap_enable = 0;
 		cch->tfm_done_bit_enable = 0;

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
@@ -217,18 +217,18 @@ static int gru_seq_show(struct seq_file
 	struct gru_state *gru = GID_TO_GRU(gid);
 
 	if (gid == 0) {
-		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "gid", "nid",
+		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "hub", "gid",
 			   "ctx", "cbr", "dsr", "ctx", "cbr", "dsr");
-		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "", "", "busy",
-			   "busy", "busy", "free", "free", "free");
+		seq_printf(file, "#%5s%5s%7s%6s%6s%8s%6s%6s\n", "", "", "total",
+			   "total", "total", "free", "free", "free");
 	}
 	if (gru) {
 		ctxfree = GRU_NUM_CCH - gru->gs_active_contexts;
 		cbrfree = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
 		dsrfree = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
-		seq_printf(file, " %5d%5d%7ld%6ld%6ld%8ld%6ld%6ld\n",
-			   gru->gs_gid, gru->gs_blade_id, GRU_NUM_CCH - ctxfree,
-			   GRU_NUM_CBE - cbrfree, GRU_NUM_DSR_BYTES - dsrfree,
+		seq_printf(file, " %5d%5d%7d%6d%6d%8ld%6ld%6ld\n",
+			   gru->gs_blade_id, gru->gs_gid,
+			   GRU_NUM_CCH, GRU_NUM_CBE, GRU_NUM_DSR_BYTES,
 			   ctxfree, cbrfree, dsrfree);
 	}
 

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -166,28 +166,6 @@ static int gru_create_new_context(unsign
 	return ret;
 }
 
-/*
- * Get GRU configuration info (temp - for emulator testing)
- */
-static long gru_get_config_info(unsigned long arg)
-{
-	struct gru_config_info info;
-	int nodesperblade;
-
-	if (num_online_nodes() > 1 &&
-			(uv_node_to_blade_id(1) == uv_node_to_blade_id(0)))
-		nodesperblade = 2;
-	else
-		nodesperblade = 1;
-	info.cpus = num_online_cpus();
-	info.nodes = num_online_nodes();
-	info.blades = info.nodes / nodesperblade;
-	info.chiplets = GRU_CHIPLETS_PER_BLADE * info.blades;
-
-	if (copy_to_user((void __user *)arg, &info, sizeof(info)))
-		return -EFAULT;
-	return 0;
-}
 
 /*
  * gru_file_unlocked_ioctl

@@ -226,9 +204,6 @@ static long gru_file_unlocked_ioctl(stru
 	case GRU_KTEST:
 		err = gru_ktest(arg);
 		break;
-	case GRU_GET_CONFIG_INFO:
-		err = gru_get_config_info(arg);
-		break;
 	case GRU_DUMP_CHIPLET_STATE:
 		err = gru_dump_chiplet_request(arg);
 		break;

Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
@@ -53,9 +53,6 @@
 /* For user TLB flushing (primarily for tests) */
 #define GRU_USER_FLUSH_TLB		_IOWR(GRU_IOCTL_NUM, 50, void *)
 
-/* Get some config options (primarily for tests & emulator) */
-#define GRU_GET_CONFIG_INFO		_IOWR(GRU_IOCTL_NUM, 51, void *)
-
 /* Various kernel self-tests */
 #define GRU_KTEST			_IOWR(GRU_IOCTL_NUM, 52, void *)
 

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
@@ -71,7 +71,6 @@ struct gru_gseg_statistics {
 #define GRU_OPT_MISS_USER_POLL	0x0001	/* User will poll CB for faults */
 #define GRU_OPT_MISS_FMM_INTR	0x0002	/* Send interrupt to cpu to
 					   handle fault */
-#define GRU_OPT_MISS_FMM_POLL	0x0003	/* Use system polling thread */
 #define GRU_OPT_MISS_MASK	0x0003	/* Mask for TLB MISS option */
 
 

80 lines not on this page · raw message

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -127,6 +127,23 @@ static int gru_file_mmap(struct file *fi
 	return 0;
 }
 
+int gru_enable_polling_mode(unsigned long arg)
+{
+	struct gru_fmm_polling_req req;
+	struct gru_state *gru;
+
+	if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
+		return -EFAULT;
+
+	if (req.gid >= gru_max_gids)
+		return -EINVAL;
+
+	gru = GID_TO_GRU(req.gid);
+	gru_intr_poll(gru->gs_chiplet_id, gru->gs_blade_id);
+	return 0;
+}
+
+
 /*
  * Create a new GRU context
  */

@@ -186,6 +203,9 @@ static long gru_file_unlocked_ioctl(stru
 	case GRU_SET_CONTEXT_OPTION:
 		err = gru_set_context_option(arg);
 		break;
+	case GRU_FMM_POLLING_MODE:
+		err = gru_enable_polling_mode(arg);
+		break;
 	case GRU_USER_GET_EXCEPTION_DETAIL:
 		err = gru_get_exception_detail(arg);
 		break;

Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
@@ -50,6 +50,9 @@
 /* For getting gseg statistics */
 #define GRU_GET_GSEG_STATISTICS		_IOWR(GRU_IOCTL_NUM, 12, void *)
 
+/* For switching a GRU to FMM polling mode */
+#define GRU_FMM_POLLING_MODE		_IOWR(GRU_IOCTL_NUM, 13, void *)
+
 /* For user TLB flushing (primarily for tests) */
 #define GRU_USER_FLUSH_TLB		_IOWR(GRU_IOCTL_NUM, 50, void *)
 

@@ -85,6 +88,13 @@ struct gru_unload_context_req {
 };
 
 /*
+ * Structure used to initiate GRU polling for TLB misses
+ */
+struct gru_fmm_polling_req {
+	unsigned int	gid;
+};
+
+/*
  * Structure used to set context options
  */
 enum {sco_gseg_owner, sco_cch_req_slice, sco_blade_chiplet};

Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
@@ -51,7 +51,7 @@ struct device *grudev = &gru_device;
  * multiple cpus may be using the same map.
  *	ZZZ should be inline but did not work on emulator
  */
-int gru_cpu_fault_map_id(void)
+int gru_cpu_fault_map_id(struct gru_state *gru)
 {
 #ifdef CONFIG_IA64
 	return uv_blade_processor_id() % GRU_NUM_TFM;

@@ -59,6 +59,8 @@ int gru_cpu_fault_map_id(void)
 	int cpu = smp_processor_id();
 	int id, core;
 
+	if (gru->gs_fmm_polling_mode)
+		return 0;
 	core = uv_cpu_core_number(cpu);
 	id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
 	return id;

@@ -596,14 +598,11 @@ void gru_load_context(struct gru_thread_
 
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 	lock_cch_handle(cch);
-	cch->tfm_fault_bit_enable =
-	    (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
-	     || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
-	cch->tlb_int_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
-	if (cch->tlb_int_enable) {
-		gts->ts_tlb_int_select = gru_cpu_fault_map_id();
-		cch->tlb_int_select = gts->ts_tlb_int_select;
-	}
+	cch->tfm_fault_bit_enable = gts->ts_user_options == GRU_OPT_MISS_FMM_INTR;
+	gts->ts_tlb_int_select = gru_cpu_fault_map_id(gru);
+	cch->tlb_int_select = gts->ts_tlb_int_select;
+	cch->tlb_int_enable = !gru->gs_fmm_polling_mode &&
+			gts->ts_user_options == GRU_OPT_MISS_FMM_INTR;
 	if (gts->ts_cch_req_slice >= 0) {
 		cch->req_slice_set_enable = 1;
 		cch->req_slice = gts->ts_cch_req_slice;

@@ -671,11 +670,9 @@ int gru_update_cch(struct gru_thread_sta
 			BUG();
 		for (i = 0; i < 8; i++)
 			cch->sizeavail[i] = gts->ts_sizeavail;
-		gts->ts_tlb_int_select = gru_cpu_fault_map_id();
-		cch->tlb_int_select = gru_cpu_fault_map_id();
-		cch->tfm_fault_bit_enable =
-		  (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
-		    || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
+		gts->ts_tlb_int_select = gru_cpu_fault_map_id(gru);
+		cch->tlb_int_select = gts->ts_tlb_int_select;
+		cch->tfm_fault_bit_enable = gts->ts_user_options == GRU_OPT_MISS_FMM_INTR;
 		if (cch_start(cch))
 			BUG();
 		ret = 1;

@@ -692,14 +689,14 @@ exit:
  * 	- task has migrated to a different cpu on the same blade where
  * 	  it was previously running.
  */
-static int gru_retarget_intr(struct gru_thread_state *gts)
+static int gru_retarget_intr(struct gru_state *gru, struct gru_thread_state *gts)
 {
 	if (gts->ts_tlb_int_select < 0
-	    || gts->ts_tlb_int_select == gru_cpu_fault_map_id())
+	    || gts->ts_tlb_int_select == gru_cpu_fault_map_id(gru))
 		return 0;
 
 	gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select,
-		gru_cpu_fault_map_id());
+		gru_cpu_fault_map_id(gru));
 	return gru_update_cch(gts);
 }
 

@@ -745,7 +742,7 @@ void gru_check_context_placement(struct
 	if (!gru_check_chiplet_assignment(gru, gts)) {
 		STAT(check_context_unload);
 		gru_unload_context(gts, 1);
-	} else if (gru_retarget_intr(gts)) {
+	} else if (gru_retarget_intr(gru, gts)) {
 		STAT(check_context_retarget_intr);
 	}
 }

Index: linux/drivers/misc/sgi-gru/gruprocfs.c
===================================================================
@@ -72,6 +72,8 @@ static int statistics_show(struct seq_fi
 	printstat(s, intr);
 	printstat(s, intr_cbr);
 	printstat(s, intr_tfh);
+	printstat(s, intr_poll);
+	printstat(s, intr_poll_found);
 	printstat(s, intr_spurious);
 	printstat(s, intr_mm_lock_failed);
 	printstat(s, call_os);

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -204,6 +204,8 @@ struct gru_stats_s {
 	atomic_long_t intr;
 	atomic_long_t intr_cbr;
 	atomic_long_t intr_tfh;
+	atomic_long_t intr_poll;
+	atomic_long_t intr_poll_found;
 	atomic_long_t intr_spurious;
 	atomic_long_t intr_mm_lock_failed;
 	atomic_long_t call_os;

@@ -438,6 +440,8 @@ struct gru_state {
 							   local flush */
 	unsigned char		gs_tgh_first_remote;	/* starting TGH# for
 							   remote flush */
+	unsigned char		gs_fmm_polling_mode;	/* Chiplet is in TFH polling mode
+							   for TLB misses */
 	spinlock_t		gs_asid_lock;		/* lock used for
 							   assigning asids */
 	spinlock_t		gs_lock;		/* lock used for

@@ -668,6 +672,7 @@ extern void gru_kservices_exit(void);
 extern irqreturn_t gru0_intr(int irq, void *dev_id);
 extern irqreturn_t gru1_intr(int irq, void *dev_id);
 extern irqreturn_t gru_intr_mblade(int irq, void *dev_id);
+extern void gru_intr_poll(int chiplet, int blade);
 extern int gru_dump_chiplet_request(unsigned long arg);
 extern long gru_get_gseg_statistics(unsigned long arg);
 extern int gru_handle_user_call_os(unsigned long address);

@@ -680,7 +685,7 @@ extern int gru_is_gts_stealable(struct g
 extern void gru_gts_stolen(struct gru_thread_state *gts,
 				struct gru_blade_state *blade);
 extern void gru_check_context_placement(struct gru_thread_state *gts);
-extern int gru_cpu_fault_map_id(void);
+extern int gru_cpu_fault_map_id(struct gru_state *gru);
 extern struct vm_area_struct *gru_find_vma(struct mm_struct *mm, unsigned long vaddr);
 extern void gru_flush_all_tlb(struct gru_state *gru);
 extern int gru_proc_init(void);

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -7,7 +7,7 @@
  * These misses are reported either via interrupts or user polling of
  * the user CB.
  *
- *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
+ *  Copyright (c) 2008-2010 Silicon Graphics, Inc.  All Rights Reserved.
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by

@@ -559,19 +559,10 @@ static irqreturn_t gru_intr(int chiplet,
 		imap.fault_bits[0], imap.fault_bits[1],
 		dmap.fault_bits[0], dmap.fault_bits[1]);
 
-	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
-		STAT(intr_cbr);
-		cmp = gru->gs_async_wq[cbrnum];
-		if (cmp)
-			complete(cmp);
-		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
-			gru->gs_gid, cbrnum, cmp ? cmp->done : -1);
-	}
-
 	for_each_cbr_in_tfm(cbrnum, imap.fault_bits) {
-		STAT(intr_tfh);
 		tfh = get_tfh_by_index(gru, cbrnum);
 		prefetchw(tfh);	/* Helps on hdw, required for emulator */
+		STAT(intr_tfh);
 
 		/*
 		 * When hardware sets a bit in the faultmap, it implicitly

@@ -596,6 +587,8 @@ static irqreturn_t gru_intr(int chiplet,
 			gts->ustats.interrupts++;
 		gts->ustats.fmm_tlbmiss++;
 		multi = 1;
+		if (!gts->ts_mm)
+			continue;
 		if (!gts->ts_force_cch_reload &&
 					down_read_trylock(&gts->ts_mm->mmap_sem)) {
 			gru_try_dropin(gru, gts, tfh, NULL);

@@ -605,6 +598,15 @@ static irqreturn_t gru_intr(int chiplet,
 			STAT(intr_mm_lock_failed);
 		}
 	}
+	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
+		STAT(intr_cbr);
+		cmp = gru->gs_async_wq[cbrnum];
+		if (cmp)
+			complete(cmp);
+		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
+			gru->gs_gid, cbrnum, cmp ? cmp->done : -1);
+	}
+
 	return IRQ_HANDLED;
 }
 

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -65,6 +65,19 @@ static struct miscdevice gru_miscdev;
  * Called when unmapping a device mapping. Frees all gru resources
  * and tables belonging to the vma.
  */
+static void gru_vma_open(struct vm_area_struct *vma)
+{
+	struct gru_vma_data *vdata;
+
+	if (!vma->vm_private_data)
+		return;
+	STAT(vdata_open);
+	vdata = vma->vm_private_data;
+	atomic_inc(&vdata->vd_refcnt);
+	gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
+				vdata);
+}
+
 static void gru_vma_close(struct vm_area_struct *vma)
 {
 	struct gru_vma_data *vdata;

@@ -75,9 +88,12 @@ static void gru_vma_close(struct vm_area
 		return;
 
 	vdata = vma->vm_private_data;
+	gru_dbg(grudev, "vma %p, file %p, vdata %p, refcnt %d\n", vma, vma->vm_file,
+				vdata, atomic_read(&vdata->vd_refcnt));
+	if (atomic_dec_return(&vdata->vd_refcnt) != 0)
+		return;
+
 	vma->vm_private_data = NULL;
-	gru_dbg(grudev, "vma %p, file %p, vdata %p\n", vma, vma->vm_file,
-				vdata);
 	list_for_each_safe(entry, next, &vdata->vd_head) {
 		gts =
 		    list_entry(entry, struct gru_thread_state, ts_next);

@@ -597,6 +613,7 @@ static struct miscdevice gru_miscdev = {
 };
 
 const struct vm_operations_struct gru_vm_ops = {
+	.open		= gru_vma_open,
 	.close		= gru_vma_close,
 	.fault		= gru_fault,
 };

Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
@@ -374,6 +374,7 @@ struct gru_vma_data *gru_alloc_vma_data(
 	if (IS_ERR(gms))
 		goto err;
 	vdata->vd_gms = gms;
+	atomic_set(&vdata->vd_refcnt, 1);
 
 	STAT(vdata_alloc);
 	INIT_LIST_HEAD(&vdata->vd_head);

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -174,6 +174,7 @@ extern unsigned int gru_max_gids;
  */
 struct gru_stats_s {
 	atomic_long_t vdata_alloc;
+	atomic_long_t vdata_open;
 	atomic_long_t vdata_free;
 	atomic_long_t gts_alloc;
 	atomic_long_t gts_free;

@@ -351,6 +352,7 @@ struct gru_mm_struct {
  */
 struct gru_vma_data {
 	spinlock_t		vd_lock;	/* Serialize access to vma */
+	atomic_t		vd_refcnt;
 	struct list_head	vd_head;	/* head of linked list of gts */
 	struct gru_mm_struct	*vd_gms;	/* asid & ioproc struct */
 	long			vd_user_options;/* misc user option flags */

Index: linux/drivers/misc/sgi-gru/gruhandles.c
===================================================================
@@ -71,7 +71,7 @@ static void report_instruction_timeout(v
 	else if (TYPE_IS(TFH, goff))
 		id = "TFH";
 
-	panic(KERN_ALERT "GRU %p (%s) is malfunctioning\n", h, id);
+	printk(KERN_ALERT "GRU:%d %p (%s) is malfunctioning\n", smp_processor_id(), h, id);
 }
 
 static int wait_instruction_complete(void *h, enum mcs_op opc)

@@ -85,6 +85,7 @@ static int wait_instruction_complete(voi
 		if (status != CCHSTATUS_ACTIVE)
 			break;
 		if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time)) {
+			gru_flush_cache(h);
 			report_instruction_timeout(h);
 			start_time = get_cycles();
 		}

Index: linux/drivers/misc/sgi-gru/grufile.c
===================================================================
@@ -175,7 +175,7 @@ static int gru_create_new_context(unsign
 
 	if (req.data_segment_bytes > max_user_dsr_bytes)
 		return -EINVAL;
-	if (req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
+	if (req.control_blocks == 0 || req.control_blocks > max_user_cbrs || !req.maximum_thread_count)
 		return -EINVAL;
 
 	if (!(req.options & GRU_OPT_MISS_MASK))

Index: linux/drivers/misc/sgi-gru/grufault.c
===================================================================
@@ -209,7 +209,6 @@ static int non_atomic_pte_lookup(struct
  *
  * Convert a user virtual address to a physical address
  * Only supports Intel large pages (2MB only) on x86_64.
- *	ZZZ - hugepage support is incomplete
  *
  * NOTE: mmap_sem is already held on entry to this function. This
  * guarantees existence of the page tables.

Index: linux/drivers/misc/sgi-gru/grukservices.h
===================================================================
@@ -36,9 +36,8 @@
  * 	- multiple senders
  *	- cross partition message
  *
- * Missing features ZZZ:
+ * Missing features:
  * 	- user options for dealing with timeouts, queue full, etc.
- * 	- gru_create_message_queue() needs interrupt vector info
  */
 
 struct gru_message_queue_desc {

Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
@@ -25,7 +25,6 @@
 
 /*
  * Maximum number of GRU segments that a user can have open
- * ZZZ temp - set high for testing. Revisit.
  */
 #define GRU_MAX_OPEN_CONTEXTS		32
 

Index: linux/drivers/misc/sgi-gru/grumain.c
===================================================================
@@ -265,7 +265,7 @@ static int gru_load_mm_tracker(struct gr
 	spin_unlock(&gms->ms_asid_lock);
 
 	gru_dbg(grudev,
-		"gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
+		"gid %d, gts 0x%p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
 		gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,
 		gms->ms_asidmap[0]);
 	return asid;

@@ -284,7 +284,7 @@ static void gru_unload_mm_tracker(struct
 	spin_lock(&gru->gs_asid_lock);
 	BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
 	asids->mt_ctxbitmap ^= ctxbitmap;
-	gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
+	gru_dbg(grudev, "gid %d, gts 0x%p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
 		gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
 	spin_unlock(&gru->gs_asid_lock);
 	spin_unlock(&gms->ms_asid_lock);

@@ -350,12 +350,12 @@ struct gru_thread_state *gru_alloc_gts(s
 	if (vma) {
 		struct gru_vma_data *vdata = vma->vm_private_data;
 
-		gts->ts_gms = vdata->vd_gms;;
+		gts->ts_gms = vdata->vd_gms;
 		gts->ts_mm = current->mm;
 		gts->ts_vma = vma;
 	}
 
-	gru_dbg(grudev, "alloc gts %p\n", gts);
+	gru_dbg(grudev, "alloc gts 0x%p\n", gts);
 	return gts;
 }
 

@@ -399,7 +399,7 @@ struct gru_thread_state *gru_find_thread
 	spin_lock(&vdata->vd_lock);
 	gts = gru_find_current_gts_nolock(vdata, tsid);
 	spin_unlock(&vdata->vd_lock);
-	gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
+	gru_dbg(grudev, "vma %p, gts 0x%p\n", vma, gts);
 	return gts;
 }
 

@@ -430,7 +430,7 @@ struct gru_thread_state *gru_alloc_threa
 		list_add(&gts->ts_next, &vdata->vd_head);
 	}
 	spin_unlock(&vdata->vd_lock);
-	gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
+	gru_dbg(grudev, "vma %p, gts 0x%p\n", vma, gts);
 	return gts;
 }
 

@@ -442,7 +442,7 @@ static void gru_free_gru_context(struct
 	struct gru_state *gru;
 
 	gru = gts->ts_gru;
-	gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);
+	gru_dbg(grudev, "gts 0x%p, gid %d\n", gts, gru->gs_gid);
 
 	spin_lock(&gru->gs_lock);
 	gru->gs_gts[gts->ts_ctxnum] = NULL;

@@ -565,7 +565,7 @@ void gru_unload_context(struct gru_threa
 		zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 
-	gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
+	gru_dbg(grudev, "gts 0x%p, cbrmap 0x%lx, dsrmap 0x%lx\n",
 		gts, gts->ts_cbr_map, gts->ts_dsr_map);
 	lock_cch_handle(cch);
 	if (cch_interrupt_sync(cch))

@@ -633,7 +633,7 @@ void gru_load_context(struct gru_thread_
 	err = cch_allocate(cch);
 	if (err) {
 		gru_dbg(grudev,
-			"err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
+			"err %d: cch %p, gts 0x%p, cbr 0x%lx, dsr 0x%lx\n",
 			err, cch, gts, gts->ts_cbr_map, gts->ts_dsr_map);
 		BUG();
 	}

@@ -645,7 +645,7 @@ void gru_load_context(struct gru_thread_
 		BUG();
 	unlock_cch_handle(cch);
 
-	gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
+	gru_dbg(grudev, "gid %d, gts 0x%p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
 		gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map,
 		(gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select);
 }

@@ -851,7 +851,7 @@ void gru_steal_context(struct gru_thread
 		STAT(steal_context_failed);
 	}
 	gru_dbg(grudev,
-		"stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
+		"stole gid %d, ctxnum %d from gts 0x%p. Need cb %d, ds %d;"
 		" avail cb %ld, ds %ld\n",
 		gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),
 		hweight64(gru->gs_dsr_map));

@@ -913,8 +913,8 @@ again:
 
 		STAT(assign_context);
 		gru_dbg(grudev,
-			"gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
-			gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts,
+			"gseg %p, pid %d, gts 0x%p, gid %d, ctx %d, cbr %d, dsr %d\n",
+			gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), current->pid, gts,
 			gts->ts_gru->gs_gid, gts->ts_ctxnum,
 			gts->ts_cbr_au_count, gts->ts_dsr_au_count);
 	} else {

Index: linux/drivers/misc/sgi-gru/grutlbpurge.c
===================================================================
@@ -162,7 +162,6 @@ void gru_flush_tlb_range(struct gru_mm_s
 	unsigned long num;
 	int grupagesize, pagesize, pageshift, gid, asid;
 
-	/* ZZZ TODO - handle huge pages */
 	pageshift = PAGE_SHIFT;
 	pagesize = (1UL << pageshift);
 	grupagesize = GRU_PAGESIZE(pageshift);

Index: linux/drivers/misc/sgi-gru/grukdump.c
===================================================================
@@ -54,10 +54,16 @@ static int gru_dump_context_data(void *g
 	cb = gseg + GRU_CB_BASE;
 	cbe = grubase + GRU_CBE_BASE;
 	tfh = grubase + GRU_TFH_BASE;
+	sync_core();
+	gru_flush_cache(cbe);
 
 	for_each_cbr_in_allocation_map(i, &cch->cbr_allocation_map, scr) {
-		if (flush_cbrs)
+		if (flush_cbrs) {
 			gru_flush_cache(cb);
+			gru_flush_cache(tfh + i * GRU_HANDLE_STRIDE);
+		}
+		gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
+		sync_core();
 		if (gru_user_copy_handle(&ubuf, cb))
 			goto fail;
 		if (gru_user_copy_handle(&ubuf, tfh + i * GRU_HANDLE_STRIDE))

@@ -130,6 +136,7 @@ static int gru_dump_context(struct gru_s
 	memset(&hdr, 0, sizeof(hdr));
 	grubase = gru->gs_gru_base_vaddr;
 	cch = get_cch(grubase, ctxnum);
+	gru_flush_cache(cch);
 	for (try = 0; try < CCH_LOCK_ATTEMPTS; try++) {
 		cch_locked =  trylock_cch_handle(cch);
 		if (cch_locked)

@@ -184,6 +191,12 @@ fail:
 	unlock_cch_handle(cch);
 	return -EFAULT;
 }
+static void dump_gsh_state(struct gru_state *gru,
+			   struct gru_dump_chiplet_state_req *req)
+{
+	memset(&req->gsh_cb_state, 0, sizeof(req->gsh_cb_state));
+	memset(&req->gsh_cch_state, 0, sizeof(req->gsh_cch_state));
+}
 
 int gru_dump_chiplet_request(unsigned long arg)
 {

@@ -214,6 +227,8 @@ int gru_dump_chiplet_request(unsigned lo
 		goto fail;
 	ubuf += ret;
 
+	dump_gsh_state(gru, &req);
+
 	for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
 		if (req.ctxnum == ctxnum || req.ctxnum < 0) {
 			ret = gru_dump_context(gru, ctxnum, ubuf, ubufend,

Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
@@ -130,6 +130,8 @@ struct gru_dump_chiplet_state_req {
 	size_t		buflen;
 	/* ---- output --- */
 	unsigned int	num_contexts;
+	unsigned long	gsh_cb_state[16];
+	unsigned long	gsh_cch_state[16];
 };
 
 #define GRU_DUMP_MAGIC	0x3474ab6c

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
@@ -39,21 +39,6 @@
 #endif
 
 /*
- * Structure for obtaining GRU resource information
- */
-struct gru_chiplet_info {
-	int	node;
-	int	chiplet;
-	int	blade;
-	int	total_dsr_bytes;
-	int	total_cbr;
-	int	total_user_dsr_bytes;
-	int	total_user_cbr;
-	int	free_user_dsr_bytes;
-	int	free_user_cbr;
-};
-
-/*
  * Statictics kept for each context.
  */
 struct gru_gseg_statistics {

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
@@ -71,21 +71,6 @@ extern void gru_wait_abort_proc(void *cb
 #define CBSS_PUT_NACKED			5
 
 /*
- * Structure used to fetch exception detail for CBs that terminate with
- * CBS_EXCEPTION
- */
-struct control_block_extended_exc_detail {
-	unsigned long	cb;
-	int		opc;
-	int		ecause;
-	int		exopc;
-	long		exceptdet0;
-	int		exceptdet1;
-	int		cbrstate;
-	int		cbrexecstatus;
-};
-
-/*
  * Instruction formats
  */
 

@@ -289,19 +274,6 @@ struct gru_instruction {
 #define CBR_EXS_EXCEPTION			(1 << CBR_EXS_EXCEPTION_BIT)
 #define CBR_EXS_CB_INT_PENDING			(1 << CBR_EXS_CB_INT_PENDING_BIT)
 
-/*
- * Exceptions are retried for the following cases. If any OTHER bits are set
- * in ecause, the exception is not retryable.
- */
-#define EXCEPTION_RETRY_BITS (CBE_CAUSE_EXECUTION_HW_ERROR |		\
-			      CBE_CAUSE_TLBHW_ERROR |			\
-			      CBE_CAUSE_RA_REQUEST_TIMEOUT |		\
-			      CBE_CAUSE_RA_RESPONSE_NON_FATAL |		\
-			      CBE_CAUSE_HA_RESPONSE_NON_FATAL |		\
-			      CBE_CAUSE_RA_RESPONSE_DATA_ERROR |	\
-			      CBE_CAUSE_HA_RESPONSE_DATA_ERROR		\
-			      )
-
 /* Message queue head structure */
 union gru_mesqhead {
 	unsigned long	val;

@@ -311,7 +283,6 @@ union gru_mesqhead {
 	};
 };
 
-
 /* Generate the low word of a GRU instruction */
 static inline unsigned long
 __opdword(unsigned char opcode, unsigned char exopc, unsigned char xtype,

@@ -324,6 +295,7 @@ __opdword(unsigned char opcode, unsigned
 	   (iaa0 << GRU_CB_IAA0_SHFT) |
 	   (iaa1 << GRU_CB_IAA1_SHFT) |
 	   (ima << GRU_CB_IMA_SHFT) |
+	   (IMA_UNMAPPED << GRU_CB_IMA_SHFT) |
 	   (xtype << GRU_CB_XTYPE_SHFT) |
 	   (opcode << GRU_CB_OPC_SHFT) |
 	   (exopc << GRU_CB_EXOPC_SHFT);

@@ -349,12 +321,6 @@ static inline void gru_start_instruction
 }
 
 
-/* Convert "hints" to IMA */
-#define CB_IMA(h)		((h) | IMA_UNMAPPED)
-
-/* Convert data segment cache line index into TRI0 / TRI1 value */
-#define GRU_DINDEX(i)		((i) * GRU_CACHE_LINE_BYTES)
-
 /* Inline functions for GRU instructions.
  *     Note:
  *     	- nelem and stride are in elements

@@ -369,7 +335,7 @@ static inline void gru_vload_phys(void *
 	ins->nelem = 1;
 	ins->op1_stride = 1;
 	gru_start_instruction(ins, __opdword(OP_VLOAD, 0, XTYPE_DW, iaa, 0,
-					(unsigned long)tri0, CB_IMA(hints)));
+					(unsigned long)tri0, hints));
 }
 
 static inline void gru_vstore_phys(void *cb, unsigned long gpa,

@@ -381,7 +347,7 @@ static inline void gru_vstore_phys(void
 	ins->nelem = 1;
 	ins->op1_stride = 1;
 	gru_start_instruction(ins, __opdword(OP_VSTORE, 0, XTYPE_DW, iaa, 0,
-					(unsigned long)tri0, CB_IMA(hints)));
+					(unsigned long)tri0, hints));
 }
 
 static inline void gru_vload(void *cb, unsigned long mem_addr,

@@ -394,7 +360,7 @@ static inline void gru_vload(void *cb, u
 	ins->nelem = nelem;
 	ins->op1_stride = stride;
 	gru_start_instruction(ins, __opdword(OP_VLOAD, 0, xtype, IAA_RAM, 0,
-					(unsigned long)tri0, CB_IMA(hints)));
+					(unsigned long)tri0, hints));
 }
 
 static inline void gru_vstore(void *cb, unsigned long mem_addr,

@@ -407,7 +373,7 @@ static inline void gru_vstore(void *cb,
 	ins->nelem = nelem;
 	ins->op1_stride = stride;
 	gru_start_instruction(ins, __opdword(OP_VSTORE, 0, xtype, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_ivload(void *cb, unsigned long mem_addr,

@@ -420,7 +386,7 @@ static inline void gru_ivload(void *cb,
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVLOAD, 0, xtype, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_ivstore(void *cb, unsigned long mem_addr,

@@ -433,7 +399,7 @@ static inline void gru_ivstore(void *cb,
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSTORE, 0, xtype, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_vset(void *cb, unsigned long mem_addr,

@@ -447,7 +413,7 @@ static inline void gru_vset(void *cb, un
 	ins->nelem = nelem;
 	ins->op1_stride = stride;
 	gru_start_instruction(ins, __opdword(OP_VSET, 0, xtype, IAA_RAM, 0,
-					 0, CB_IMA(hints)));
+					 0, hints));
 }
 
 static inline void gru_ivset(void *cb, unsigned long mem_addr,

@@ -461,7 +427,7 @@ static inline void gru_ivset(void *cb, u
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = tri1;
 	gru_start_instruction(ins, __opdword(OP_IVSET, 0, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_vflush(void *cb, unsigned long mem_addr,

@@ -474,14 +440,14 @@ static inline void gru_vflush(void *cb,
 	ins->op1_stride = stride;
 	ins->nelem = nelem;
 	gru_start_instruction(ins, __opdword(OP_VFLUSH, 0, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_nop(void *cb, int hints)
 {
 	struct gru_instruction *ins = (void *)cb;
 
-	gru_start_instruction(ins, __opdword(OP_NOP, 0, 0, 0, 0, 0, CB_IMA(hints)));
+	gru_start_instruction(ins, __opdword(OP_NOP, 0, 0, 0, 0, 0, hints));
 }
 
 

@@ -497,7 +463,7 @@ static inline void gru_bcopy(void *cb, c
 	ins->nelem = nelem;
 	ins->tri1_bufsize_64 = bufsize;
 	gru_start_instruction(ins, __opdword(OP_BCOPY, 0, xtype, IAA_RAM,
-					IAA_RAM, tri0, CB_IMA(hints)));
+					IAA_RAM, tri0, hints));
 }
 
 static inline void gru_bstore(void *cb, const unsigned long src,

@@ -510,7 +476,7 @@ static inline void gru_bstore(void *cb,
 	ins->op2_value_baddr1 = (long)dest;
 	ins->nelem = nelem;
 	gru_start_instruction(ins, __opdword(OP_BSTORE, 0, xtype, 0, IAA_RAM,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline void gru_gamir(void *cb, int exopc, unsigned long src,

@@ -520,7 +486,7 @@ static inline void gru_gamir(void *cb, i
 
 	ins->baddr0 = (long)src;
 	gru_start_instruction(ins, __opdword(OP_GAMIR, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamirr(void *cb, int exopc, unsigned long src,

@@ -530,7 +496,7 @@ static inline void gru_gamirr(void *cb,
 
 	ins->baddr0 = (long)src;
 	gru_start_instruction(ins, __opdword(OP_GAMIRR, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamer(void *cb, int exopc, unsigned long src,

@@ -544,7 +510,7 @@ static inline void gru_gamer(void *cb, i
 	ins->op1_stride = operand1;
 	ins->op2_value_baddr1 = operand2;
 	gru_start_instruction(ins, __opdword(OP_GAMER, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamerr(void *cb, int exopc, unsigned long src,

@@ -557,7 +523,7 @@ static inline void gru_gamerr(void *cb,
 	ins->op1_stride = operand1;
 	ins->op2_value_baddr1 = operand2;
 	gru_start_instruction(ins, __opdword(OP_GAMERR, exopc, xtype, IAA_RAM, 0,
-					0, CB_IMA(hints)));
+					0, hints));
 }
 
 static inline void gru_gamxr(void *cb, unsigned long src,

@@ -568,7 +534,7 @@ static inline void gru_gamxr(void *cb, u
 	ins->baddr0 = (long)src;
 	ins->nelem = 4;
 	gru_start_instruction(ins, __opdword(OP_GAMXR, EOP_XR_CSWAP, XTYPE_DW,
-				 IAA_RAM, 0, 0, CB_IMA(hints)));
+				 IAA_RAM, 0, 0, hints));
 }
 
 static inline void gru_mesq(void *cb, unsigned long queue,

@@ -580,7 +546,7 @@ static inline void gru_mesq(void *cb, un
 	ins->baddr0 = (long)queue;
 	ins->nelem = nelem;
 	gru_start_instruction(ins, __opdword(OP_MESQ, 0, XTYPE_CL, IAA_RAM, 0,
-					tri0, CB_IMA(hints)));
+					tri0, hints));
 }
 
 static inline unsigned long gru_get_amo_value(void *cb)

@@ -613,12 +579,6 @@ static inline union gru_mesqhead  gru_me
 	return mqh;
 }
 
-/*
- * Get struct control_block_extended_exc_detail for CB.
- */
-extern int gru_get_cb_exception_detail(void *cb,
-		       struct control_block_extended_exc_detail *excdet);
-
 #define GRU_EXC_STR_SIZE		256
 
 

@@ -671,6 +631,7 @@ static inline int gru_check_status(void
 	struct gru_control_block_status *cbs = (void *)cb;
 	int ret;
 
+	barrier();
 	ret = cbs->istatus;
 	if (ret != CBS_ACTIVE)
 		ret = gru_check_status_proc(cb);

@@ -700,15 +661,6 @@ static inline void gru_wait_abort(void *
 }
 
 /*
- * Get a pointer to the start of a gseg
- * 	p	- Any valid pointer within the gseg
- */
-static inline void *gru_get_gseg_pointer (void *p)
-{
-	return (void *)((unsigned long)p & ~(GRU_GSEG_PAGESIZE - 1));
-}
-
-/*
  * Get a pointer to a control block
  * 	gseg	- GSeg address returned from gru_get_thread_gru_segment()
  * 	index	- index of desired CB

Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
@@ -22,6 +22,7 @@
 
 #ifndef __GRUHANDLES_H__
 #define __GRUHANDLES_H__
+#include "grulib.h"
 #include "gru_instructions.h"
 
 /*

Index: linux/drivers/misc/sgi-gru/grulib.h
===================================================================
@@ -62,6 +62,25 @@
 #define THREAD_POINTER(p, th)		(p + GRU_GSEG_PAGESIZE * (th))
 #define GSEG_START(cb)			((void *)((unsigned long)(cb) & ~(GRU_GSEG_PAGESIZE - 1)))
 
+/*
+ * Size used to map GRU GSeg
+ */
+#define GRU_GSEG_PAGESIZE    (256 * 1024UL)
+
+/*
+ * Exceptions are retried for the following cases. If any OTHER bits are set
+ * in ecause, the exception is not retryable.
+ */
+#define EXCEPTION_RETRY_BITS (CBE_CAUSE_EXECUTION_HW_ERROR |            \
+			      CBE_CAUSE_TLBHW_ERROR |                   \
+			      CBE_CAUSE_RA_REQUEST_TIMEOUT |            \
+			      CBE_CAUSE_RA_RESPONSE_NON_FATAL |         \
+			      CBE_CAUSE_HA_RESPONSE_NON_FATAL |         \
+			      CBE_CAUSE_RA_RESPONSE_DATA_ERROR |        \
+			      CBE_CAUSE_HA_RESPONSE_DATA_ERROR          \
+			     )
+
+
 struct gru_get_gseg_statistics_req {
 	unsigned long			gseg;
 	struct gru_gseg_statistics	stats;

@@ -134,6 +153,21 @@ struct gru_dump_chiplet_state_req {
 	unsigned long	gsh_cch_state[16];
 };
 
+/*
+ * Structure used to fetch exception detail for CBs that terminate with
+ * CBS_EXCEPTION
+ */
+struct control_block_extended_exc_detail {
+	unsigned long	cb;
+	int		opc;
+	int		ecause;
+	int		exopc;
+	long		exceptdet0;
+	int		exceptdet1;
+	int		cbrstate;
+	int		cbrexecstatus;
+};
+
 #define GRU_DUMP_MAGIC	0x3474ab6c
 struct gru_dump_context_header {
 	unsigned int	magic;

Index: linux/drivers/misc/sgi-gru/grutables.h
===================================================================
@@ -165,7 +165,7 @@ extern unsigned int gru_max_gids;
 #define GRU_MAX_GRUS		(GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE)
 
 #define GRU_DRIVER_ID_STR	"SGI GRU Device Driver"
-#define GRU_DRIVER_VERSION_STR	"0.85"
+#define GRU_DRIVER_VERSION_STR	"1.00"
 
 #define gru_random()    get_cycles()
 

Index: linux/drivers/misc/sgi-gru/gru.h
===================================================================
@@ -28,15 +28,9 @@
 #define GRU_DS_BASE			0x20000
 
 /*
- * Size used to map GRU GSeg
+ * Minimum page size used to map a GRU context
  */
-#if defined(CONFIG_IA64)
-#define GRU_GSEG_PAGESIZE	(256 * 1024UL)
-#elif defined(CONFIG_X86_64)
-#define GRU_GSEG_PAGESIZE	(256 * 1024UL)		/* ZZZ 2MB ??? */
-#else
-#error "Unsupported architecture"
-#endif
+#define GRU_MIN_GSEG_PAGESIZE	(256 * 1024UL)
 
 /*
  * Statictics kept for each context.

Index: linux/drivers/misc/sgi-gru/gru_instructions.h
===================================================================
@@ -687,6 +687,6 @@ static inline void *gru_get_data_pointer
  */
 static inline int gru_get_tri(void *vaddr)
 {
-	return ((unsigned long)vaddr & (GRU_GSEG_PAGESIZE - 1)) - GRU_DS_BASE;
+	return ((unsigned long)vaddr & (GRU_MIN_GSEG_PAGESIZE - 1)) - GRU_DS_BASE;
 }
 #endif		/* __GRU_INSTRUCTIONS_H__ */

Index: linux/drivers/misc/sgi-gru/gruhandles.h
===================================================================
@@ -478,15 +478,19 @@ enum gru_cbr_state {
 	CBRSTATE_INACTIVE,
 	CBRSTATE_IDLE,
 	CBRSTATE_PE_CHECK,
+	CBRSTATE_DSR_PRE,
 	CBRSTATE_QUEUED,
+	CBRSTATE_ISSUED,
 	CBRSTATE_WAIT_RESPONSE,
-	CBRSTATE_INTERRUPTED,
-	CBRSTATE_INTERRUPTED_MISS_FMM,
+	CBRSTATE_DSR_POST,
 	CBRSTATE_BUSY_INTERRUPT_MISS_FMM,
+	CBRSTATE_INTERRUPTED_MISS_FMM,
+	CBRSTATE_BUSY_INTERRUPT_MISS_UPM,
 	CBRSTATE_INTERRUPTED_MISS_UPM,
-	CBRSTATE_BUSY_INTERRUPTED_MISS_UPM,
-	CBRSTATE_REQUEST_ISSUE,
 	CBRSTATE_BUSY_INTERRUPT,
+	CBRSTATE_INTERRUPTED,
+	CBRSTATE_BCPY_CHECK,
+	CBRSTATE_WAIT_DONE,
 };
 
 /* CBE cbrexecstatus bits  - defined in gru_instructions.h*/