use r14 for a per-cpu kernel register

index ab386af2904f..fbbd2ca18866 100644
@@ -143,7 +143,7 @@ struct kvmppc_host_state {
 
 struct kvmppc_book3s_shadow_vcpu {
 	bool in_use;
-	ulong gpr[14];
+	ulong gpr[15];
 	u32 cr;
 	ulong xer;
 	ulong ctr;

index ae94b3626b6c..109f4dce8a79 100644
@@ -10,6 +10,16 @@
 #include <asm/ppc-opcode.h>
 #include <asm/firmware.h>
 
+#ifdef __powerpc64__
+#ifdef CONFIG_PPC_BOOK3S
+#define FIRST_NVGPR		15
+#else
+#define FIRST_NVGPR		14
+#endif
+#else
+#define FIRST_NVGPR		13
+#endif
+
 #ifdef __ASSEMBLY__
 
 #define SZL			(BITS_PER_LONG/8)

@@ -75,16 +85,21 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
 #ifdef __powerpc64__
 #define SAVE_GPR(n, base)	std	n,GPR0+8*(n)(base)
 #define REST_GPR(n, base)	ld	n,GPR0+8*(n)(base)
+#ifdef CONFIG_PPC_BOOK3S
+#define SAVE_NVGPRS(base)	SAVE_GPR(15, base); SAVE_2GPRS(16, base); SAVE_4GPRS(18, base); SAVE_10GPRS(22, base)
+#define REST_NVGPRS(base)	REST_GPR(15, base); REST_2GPRS(16, base); REST_4GPRS(18, base); REST_10GPRS(22, base)
+#else /* CONFIG_PPC_BOOK3S */
 #define SAVE_NVGPRS(base)	SAVE_8GPRS(14, base); SAVE_10GPRS(22, base)
 #define REST_NVGPRS(base)	REST_8GPRS(14, base); REST_10GPRS(22, base)
-#else
+#endif /* CONFIG_PPC_BOOK3S */
+#else /* __powerpc64__ */
 #define SAVE_GPR(n, base)	stw	n,GPR0+4*(n)(base)
 #define REST_GPR(n, base)	lwz	n,GPR0+4*(n)(base)
 #define SAVE_NVGPRS(base)	SAVE_GPR(13, base); SAVE_8GPRS(14, base); \
 				SAVE_10GPRS(22, base)
 #define REST_NVGPRS(base)	REST_GPR(13, base); REST_8GPRS(14, base); \
 				REST_10GPRS(22, base)
-#endif
+#endif /* __powerpc64__ */
 
 #define SAVE_2GPRS(n, base)	SAVE_GPR(n, base); SAVE_GPR(n+1, base)
 #define SAVE_4GPRS(n, base)	SAVE_2GPRS(n, base); SAVE_2GPRS(n+2, base)

@@ -184,7 +199,7 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
 #ifdef CONFIG_PPC64
 
 #define STACKFRAMESIZE 256
-#define __STK_REG(i)   (112 + ((i)-14)*8)
+#define __STK_REG(i)   (112 + ((i)-15)*8)
 #define STK_REG(i)     __STK_REG(__REG_##i)
 
 #ifdef PPC64_ELF_ABI_v2

index 397681f43eed..db8407483c9e 100644
@@ -283,9 +283,9 @@ int main(void)
 	STACK_PT_REGS_OFFSET(GPR11, gpr[11]);
 	STACK_PT_REGS_OFFSET(GPR12, gpr[12]);
 	STACK_PT_REGS_OFFSET(GPR13, gpr[13]);
-#ifndef CONFIG_PPC64
+#ifndef CONFIG_PPC_BOOK3E_64
 	STACK_PT_REGS_OFFSET(GPR14, gpr[14]);
-#endif /* CONFIG_PPC64 */
+#endif
 	/*
 	 * Note: these symbols include _ because they overlap with special
 	 * register names

index 35e240a0a408..d593e381f34d 100644
@@ -230,11 +230,11 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p)
 	for (reg = 0; reg < 3; reg++)
 		PACK64(ptr, regs->gpr[reg]);
 
-	/* Regs GPR3-13 are caller saved, not in regs->gpr[] */
-	ptr += 11;
+	/* Regs GPR3-13/14 are caller saved, not in regs->gpr[] */
+	ptr += FIRST_NVGPR - 2;
 
-	/* Regs GPR14-31 */
-	for (reg = 14; reg < 32; reg++)
+	/* Regs GPR14/15-31 */
+	for (reg = FIRST_NVGPR; reg < 32; reg++)
 		PACK64(ptr, regs->gpr[reg]);
 
 #ifdef CONFIG_FSL_BOOKE

index 2a63cc78257d..f56e42f06f24 100644
@@ -1672,12 +1672,12 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
 		childregs->gpr[1] = sp + sizeof(struct pt_regs);
 		/* function */
 		if (usp)
-			childregs->gpr[14] = ppc_function_entry((void *)usp);
+			childregs->gpr[FIRST_NVGPR] = ppc_function_entry((void *)usp);
 #ifdef CONFIG_PPC64
 		clear_tsk_thread_flag(p, TIF_32BIT);
 		childregs->softe = IRQ_SOFT_MASK_NONE;
 #endif
-		childregs->gpr[15] = kthread_arg;
+		childregs->gpr[FIRST_NVGPR + 1] = kthread_arg;
 		p->thread.regs = NULL;	/* no user register state */
 		ti->flags |= _TIF_RESTOREALL;
 		f = ret_from_kernel_thread;

index d0dc8624198f..b149d4fa4a45 100644
@@ -159,6 +159,9 @@ void kvmppc_copy_to_svcpu(struct kvmppc_book3s_shadow_vcpu *svcpu,
 	svcpu->gpr[11] = vcpu->arch.gpr[11];
 	svcpu->gpr[12] = vcpu->arch.gpr[12];
 	svcpu->gpr[13] = vcpu->arch.gpr[13];
+#ifdef CONFIG_PPC_BOOK3S_64
+	svcpu->gpr[14] = vcpu->arch.gpr[14];
+#endif
 	svcpu->cr  = vcpu->arch.cr;
 	svcpu->xer = vcpu->arch.xer;
 	svcpu->ctr = vcpu->arch.ctr;

@@ -209,6 +212,9 @@ void kvmppc_copy_from_svcpu(struct kvm_vcpu *vcpu,
 	vcpu->arch.gpr[11] = svcpu->gpr[11];
 	vcpu->arch.gpr[12] = svcpu->gpr[12];
 	vcpu->arch.gpr[13] = svcpu->gpr[13];
+#ifdef CONFIG_PPC_BOOK3S_64
+	vcpu->arch.gpr[14] = svcpu->gpr[14];
+#endif
 	vcpu->arch.cr  = svcpu->cr;
 	vcpu->arch.xer = svcpu->xer;
 	vcpu->arch.ctr = svcpu->ctr;

index 54afd1f140a4..dadaa7471755 100644
@@ -42,16 +42,18 @@
 #define EX_R11		16
 #define EX_R12		24
 #define EX_R13		32
-#define EX_DAR		40
-#define EX_DSISR	48
-#define EX_CCR		52
-#define EX_CFAR		56
-#define EX_PPR		64
+#define EX_R14		40
+#define EX_DAR		48
+#define EX_DSISR	56
+#define EX_CCR		60
+#define EX_CFAR		64
+#define EX_PPR		72
+
 #if defined(CONFIG_RELOCATABLE)
-#define EX_CTR		72
-#define EX_SIZE		10	/* size in u64 units */
+#define EX_CTR		80
+#define EX_SIZE		11	/* size in u64 units */
 #else
-#define EX_SIZE		9	/* size in u64 units */
+#define EX_SIZE		10	/* size in u64 units */
 #endif
 
 /*

@@ -77,9 +79,8 @@
 #ifdef CONFIG_RELOCATABLE
 #define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
-	LOAD_HANDLER(r12,label);					\
-	mtctr	r12;							\
-	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
+	LOAD_HANDLER(r10,label);					\
+	mtctr	r10;							\
 	li	r10,MSR_RI;						\
 	mtmsrd 	r10,1;			/* Set RI (EE=0) */		\
 	bctr;

@@ -87,7 +88,6 @@
 /* If not relocatable, we can jump directly -- and save messing with LR */
 #define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)			\
 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
-	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
 	li	r10,MSR_RI;						\
 	mtmsrd 	r10,1;			/* Set RI (EE=0) */		\
 	b	label;

@@ -102,7 +102,7 @@
  */
 #define EXCEPTION_RELON_PROLOG_PSERIES(area, label, h, extra, vec)	\
 	EXCEPTION_PROLOG_0(area);					\
-	EXCEPTION_PROLOG_1(area, extra, vec);				\
+	EXCEPTION_PROLOG_1(area, h, extra, vec);				\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)
 
 /*

@@ -198,17 +198,21 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	std	r10,area+EX_R10(r13);	/* save r10 - r12 */		\
 	OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
 
-#define __EXCEPTION_PROLOG_1_PRE(area)					\
+#define __EXCEPTION_PROLOG_1(area, h)					\
+	std	r11,area+EX_R11(r13);					\
+	std	r12,area+EX_R12(r13);					\
+	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
+	GET_SCRATCH0(r11);						\
 	OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);		\
 	OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);		\
 	SAVE_CTR(r10, area);						\
-	mfcr	r9;
-
-#define __EXCEPTION_PROLOG_1_POST(area)					\
-	std	r11,area+EX_R11(r13);					\
-	std	r12,area+EX_R12(r13);					\
-	GET_SCRATCH0(r10);						\
-	std	r10,area+EX_R13(r13)
+	mfcr	r9;							\
+	std	r11,area+EX_R13(r13);					\
+	std	r14,area+EX_R14(r13);					\
+	andi.	r10,r12,MSR_PR;						\
+	beq	1f;							\
+	ld	r14,PACA_R14(r13);					\
+1:
 
 /*
  * This version of the EXCEPTION_PROLOG_1 will carry

@@ -216,30 +220,27 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
  * checking of the interrupt maskable level in the SOFTEN_TEST.
  * Intended to be used in MASKABLE_EXCPETION_* macros.
  */
-#define MASKABLE_EXCEPTION_PROLOG_1(area, extra, vec, bitmask)			\
-	__EXCEPTION_PROLOG_1_PRE(area);					\
-	extra(vec, bitmask);						\
-	__EXCEPTION_PROLOG_1_POST(area);
+#define MASKABLE_EXCEPTION_PROLOG_1(area, h, extra, vec, bitmask)	\
+	__EXCEPTION_PROLOG_1(area, h);					\
+	extra(vec, bitmask);
 
 /*
  * This version of the EXCEPTION_PROLOG_1 is intended
  * to be used in STD_EXCEPTION* macros
  */
-#define _EXCEPTION_PROLOG_1(area, extra, vec)				\
-	__EXCEPTION_PROLOG_1_PRE(area);					\
-	extra(vec);							\
-	__EXCEPTION_PROLOG_1_POST(area);
+#define _EXCEPTION_PROLOG_1(area, h, extra, vec)			\
+	__EXCEPTION_PROLOG_1(area, h);					\
+	extra(vec);
 
-#define EXCEPTION_PROLOG_1(area, extra, vec)				\
-	_EXCEPTION_PROLOG_1(area, extra, vec)
+#define EXCEPTION_PROLOG_1(area, h, extra, vec)				\
+	_EXCEPTION_PROLOG_1(area, h, extra, vec)
 
 #define __EXCEPTION_PROLOG_PSERIES_1(label, h)				\
-	ld	r10,PACAKMSR(r13);	/* get MSR value for kernel */	\
 	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
-	LOAD_HANDLER(r12,label)						\
-	mtspr	SPRN_##h##SRR0,r12;					\
-	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
+	ld	r10,PACAKMSR(r13);	/* get MSR value for kernel */	\
 	mtspr	SPRN_##h##SRR1,r10;					\
+	LOAD_HANDLER(r10,label)						\
+	mtspr	SPRN_##h##SRR0,r10;					\
 	h##rfid;							\
 	b	.	/* prevent speculative execution */
 #define EXCEPTION_PROLOG_PSERIES_1(label, h)				\

@@ -247,13 +248,12 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 
 /* _NORI variant keeps MSR_RI clear */
 #define __EXCEPTION_PROLOG_PSERIES_1_NORI(label, h)			\
+	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
 	ld	r10,PACAKMSR(r13);	/* get MSR value for kernel */	\
 	xori	r10,r10,MSR_RI;		/* Clear MSR_RI */		\
-	mfspr	r11,SPRN_##h##SRR0;	/* save SRR0 */			\
-	LOAD_HANDLER(r12,label)						\
-	mtspr	SPRN_##h##SRR0,r12;					\
-	mfspr	r12,SPRN_##h##SRR1;	/* and SRR1 */			\
 	mtspr	SPRN_##h##SRR1,r10;					\
+	LOAD_HANDLER(r10,label)						\
+	mtspr	SPRN_##h##SRR0,r10;					\
 	h##rfid;							\
 	b	.	/* prevent speculative execution */
 

@@ -262,7 +262,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 
 #define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec)		\
 	EXCEPTION_PROLOG_0(area);					\
-	EXCEPTION_PROLOG_1(area, extra, vec);				\
+	EXCEPTION_PROLOG_1(area, h, extra, vec);			\
 	EXCEPTION_PROLOG_PSERIES_1(label, h);
 
 #define __KVMTEST(h, n)							\

@@ -336,7 +336,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 /* Do not enable RI */
 #define EXCEPTION_PROLOG_PSERIES_NORI(area, label, h, extra, vec)	\
 	EXCEPTION_PROLOG_0(area);					\
-	EXCEPTION_PROLOG_1(area, extra, vec);				\
+	EXCEPTION_PROLOG_1(area, h, extra, vec);			\
 	EXCEPTION_PROLOG_PSERIES_1_NORI(label, h);
 
 

@@ -349,8 +349,9 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	ld	r10,area+EX_PPR(r13);					\
 	std	r10,HSTATE_PPR(r13);					\
 	END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948);	\
+	ld	r10,area+EX_R12(r13);					\
+	std	r10,HSTATE_SCRATCH0(r13);				\
 	ld	r10,area+EX_R10(r13);					\
-	std	r12,HSTATE_SCRATCH0(r13);				\
 	sldi	r12,r9,32;						\
 	ori	r12,r12,(n);						\
 	/* This reloads r9 before branching to kvmppc_interrupt */	\

@@ -363,8 +364,9 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	ld	r10,area+EX_PPR(r13);					\
 	std	r10,HSTATE_PPR(r13);					\
 	END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948);	\
+	ld	r10,area+EX_R12(r13);					\
+	std	r10,HSTATE_SCRATCH0(r13);				\
 	ld	r10,area+EX_R10(r13);					\
-	std	r12,HSTATE_SCRATCH0(r13);				\
 	sldi	r12,r9,32;						\
 	ori	r12,r12,(n);						\
 	/* This reloads r9 before branching to kvmppc_interrupt */	\

@@ -372,6 +374,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 89:	mtocrf	0x80,r9;						\
 	ld	r9,area+EX_R9(r13);					\
 	ld	r10,area+EX_R10(r13);					\
+	ld	r12,area+EX_R12(r13);					\
 	b	kvmppc_skip_##h##interrupt
 
 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER

@@ -393,7 +396,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	std	r12,_MSR(r1);		/* save SRR1 in stackframe	*/ \
 	std	r10,0(r1);		/* make stack chain pointer	*/ \
 	std	r0,GPR0(r1);		/* save r0 in stackframe	*/ \
-	std	r10,GPR1(r1);		/* save r1 in stackframe	*/ \
+	std	r10,GPR1(r1)		/* save r1 in stackframe	*/
 
 
 /*

@@ -430,16 +433,18 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 
 /* Save original regs values from save area to stack frame. */
 #define EXCEPTION_PROLOG_COMMON_2(area)					   \
-	ld	r9,area+EX_R9(r13);	/* move r9, r10 to stackframe	*/ \
+	ld	r9,area+EX_R9(r13);	/* move r9-r14 to stackframe	*/ \
 	ld	r10,area+EX_R10(r13);					   \
 	std	r9,GPR9(r1);						   \
 	std	r10,GPR10(r1);						   \
-	ld	r9,area+EX_R11(r13);	/* move r11 - r13 to stackframe	*/ \
+	ld	r9,area+EX_R11(r13);					   \
 	ld	r10,area+EX_R12(r13);					   \
-	ld	r11,area+EX_R13(r13);					   \
 	std	r9,GPR11(r1);						   \
 	std	r10,GPR12(r1);						   \
-	std	r11,GPR13(r1);						   \
+	ld	r9,area+EX_R13(r13);					   \
+	ld	r10,area+EX_R14(r13);					   \
+	std	r9,GPR13(r1);						   \
+	std	r10,GPR14(r1);						   \
 	BEGIN_FTR_SECTION_NESTED(66);					   \
 	ld	r10,area+EX_CFAR(r13);					   \
 	std	r10,ORIG_GPR3(r1);					   \

@@ -480,7 +485,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	b hdlr;
 
 #define STD_EXCEPTION_PSERIES_OOL(vec, label)			\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec);	\
+	EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_STD, KVMTEST_PR, vec);	\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
 
 #define STD_EXCEPTION_HV(loc, vec, label)			\

@@ -489,7 +494,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 				 EXC_HV, KVMTEST_HV, vec);
 
 #define STD_EXCEPTION_HV_OOL(vec, label)			\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec);	\
+	EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_HV, KVMTEST_HV, vec);	\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
 
 #define STD_RELON_EXCEPTION_PSERIES(loc, vec, label)	\

@@ -498,7 +503,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, EXC_STD, NOTEST, vec);
 
 #define STD_RELON_EXCEPTION_PSERIES_OOL(vec, label)		\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec);		\
+	EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_STD, NOTEST, vec);		\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_STD)
 
 #define STD_RELON_EXCEPTION_HV(loc, vec, label)		\

@@ -507,7 +512,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 				       EXC_HV, KVMTEST_HV, vec);
 
 #define STD_RELON_EXCEPTION_HV_OOL(vec, label)			\
-	EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec);	\
+	EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_HV, KVMTEST_HV, vec);	\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
 
 /* This associate vector numbers with bits in paca->irq_happened */

@@ -548,7 +553,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 #define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)	\
 	SET_SCRATCH0(r13);    /* save r13 */				\
 	EXCEPTION_PROLOG_0(PACA_EXGEN);					\
-	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask);	\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, h, extra, vec, bitmask);\
 	EXCEPTION_PROLOG_PSERIES_1(label, h);
 
 #define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)	\

@@ -559,7 +564,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 				    EXC_STD, SOFTEN_TEST_PR, bitmask)
 
 #define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label, bitmask)		\
-	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_STD, SOFTEN_TEST_PR, vec, bitmask);\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD)
 
 #define MASKABLE_EXCEPTION_HV(loc, vec, label, bitmask)			\

@@ -567,13 +572,13 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 				    EXC_HV, SOFTEN_TEST_HV, bitmask)
 
 #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask)			\
-	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_HV, SOFTEN_TEST_HV, vec, bitmask);\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV)
 
 #define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \
 	SET_SCRATCH0(r13);    /* save r13 */				\
 	EXCEPTION_PROLOG_0(PACA_EXGEN);					\
-	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask);	\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, h, extra, vec, bitmask);\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, h)
 
 #define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)\

@@ -584,7 +589,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 					  EXC_STD, SOFTEN_NOTEST_PR, bitmask)
 
 #define MASKABLE_RELON_EXCEPTION_PSERIES_OOL(vec, label, bitmask)	\
-	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_STD, SOFTEN_NOTEST_PR, vec, bitmask);\
 	EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD);
 
 #define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label, bitmask)		\

@@ -592,7 +597,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 					  EXC_HV, SOFTEN_TEST_HV, bitmask)
 
 #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask)		\
-	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec, bitmask);\
+	MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, EXC_HV, SOFTEN_NOTEST_HV, vec, bitmask);\
 	EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
 
 /*

index e2ee193eb24d..cd6a9a010895 100644
@@ -34,6 +34,9 @@
 #include <asm/cpuidle.h>
 
 register struct paca_struct *local_paca asm("r13");
+#ifdef CONFIG_PPC_BOOK3S
+register u64 local_r14 asm("r14");
+#endif
 
 #if defined(CONFIG_DEBUG_PREEMPT) && defined(CONFIG_SMP)
 extern unsigned int debug_smp_processor_id(void); /* from linux/smp.h */

@@ -65,7 +68,6 @@ struct paca_struct {
 	 * read-only (after boot) fields in the first cacheline to
 	 * avoid cacheline bouncing.
 	 */
-
 	struct lppaca *lppaca_ptr;	/* Pointer to LpPaca for PLIC */
 #endif /* CONFIG_PPC_BOOK3S */
 	/*

@@ -104,6 +106,7 @@ struct paca_struct {
 	 */
 	/* used for most interrupts/exceptions */
 	u64 exgen[EX_SIZE] __attribute__((aligned(0x80)));
+	u64 r14;
 	u64 exslb[EX_SIZE];	/* used for SLB/segment table misses
  				 * on the linear mapping */
 	/* SLB related definitions */

index db8407483c9e..32d393f55a96 100644
@@ -218,8 +218,9 @@ int main(void)
 	OFFSET(PACACONTEXTSLLP, paca_struct, mm_ctx_sllp);
 #endif /* CONFIG_PPC_MM_SLICES */
 	OFFSET(PACA_EXGEN, paca_struct, exgen);
-	OFFSET(PACA_EXMC, paca_struct, exmc);
+	OFFSET(PACA_R14, paca_struct, r14);
 	OFFSET(PACA_EXSLB, paca_struct, exslb);
+	OFFSET(PACA_EXMC, paca_struct, exmc);
 	OFFSET(PACA_EXNMI, paca_struct, exnmi);
 	OFFSET(PACALPPACAPTR, paca_struct, lppaca_ptr);
 	OFFSET(PACA_SLBSHADOWPTR, paca_struct, slb_shadow_ptr);

index d6597038931d..2491e9cd8d24 100644
@@ -198,7 +198,6 @@ void setup_paca(struct paca_struct *new_paca)
 		mtspr(SPRN_SPRG_HPACA, local_paca);
 #endif
 	mtspr(SPRN_SPRG_PACA, local_paca);
-
 }
 
 static int __initdata paca_size;

index a4408a7e6f14..9a4c5bf35d92 100644
@@ -192,6 +192,9 @@ static void __init fixup_boot_paca(void)
 	get_paca()->data_offset = 0;
 	/* Mark interrupts disabled in PACA */
 	irq_soft_mask_set(IRQ_SOFT_MASK_STD);
+	/* Set r14 and paca_r14 to debug value */
+	get_paca()->r14 = (0xdeadbeefULL << 32) | mfspr(SPRN_PIR);
+	local_r14 = get_paca()->r14;
 }
 
 static void __init configure_exceptions(void)

index 70274b7b4773..b1b8f9f88083 100644
@@ -3073,6 +3073,7 @@ int emulate_step(struct pt_regs *regs, unsigned int instr)
 		regs->gpr[11] = regs->nip + 4;
 		regs->gpr[12] = regs->msr & MSR_MASK;
 		regs->gpr[13] = (unsigned long) get_paca();
+		regs->gpr[14] = local_r14;
 		regs->nip = (unsigned long) &system_call_common;
 		regs->msr = MSR_KERNEL;
 		return 1;

index cd6a9a010895..4dd4ac69e84f 100644
@@ -35,6 +35,11 @@
 
 register struct paca_struct *local_paca asm("r13");
 #ifdef CONFIG_PPC_BOOK3S
+/*
+ * The top 32-bits of r14 is used as the per-cpu offset, shifted by PAGE_SHIFT.
+ * The per-cpu could be moved completely to vmalloc space if we had large
+ * vmalloc page mapping? (no, must access it in real mode).
+ */
 register u64 local_r14 asm("r14");
 #endif
 

index dce863a7635c..1e0d79d30eac 100644
@@ -12,7 +12,7 @@
 
 #include <asm/paca.h>
 
-#define __my_cpu_offset local_paca->data_offset
+#define __my_cpu_offset (local_r14 >> 16)
 
 #endif /* CONFIG_SMP */
 #endif /* __powerpc64__ */

index 9a4c5bf35d92..f4a96ebb523a 100644
@@ -192,8 +192,8 @@ static void __init fixup_boot_paca(void)
 	get_paca()->data_offset = 0;
 	/* Mark interrupts disabled in PACA */
 	irq_soft_mask_set(IRQ_SOFT_MASK_STD);
-	/* Set r14 and paca_r14 to debug value */
-	get_paca()->r14 = (0xdeadbeefULL << 32) | mfspr(SPRN_PIR);
+	/* Set r14 and paca_r14 to zero */
+	get_paca()->r14 = 0;
 	local_r14 = get_paca()->r14;
 }
 

@@ -761,7 +761,14 @@ void __init setup_per_cpu_areas(void)
 	for_each_possible_cpu(cpu) {
                 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
 		paca[cpu].data_offset = __per_cpu_offset[cpu];
+
+		BUG_ON(paca[cpu].data_offset & (PAGE_SIZE-1));
+		BUG_ON(paca[cpu].data_offset >= (1UL << (64 - 16)));
+
+		/* The top 48 bits are used for per-cpu data */
+		paca[cpu].r14 |= paca[cpu].data_offset << 16;
 	}
+	local_r14 = paca[smp_processor_id()].r14;
 }
 #endif
 

index 422f99cf9924..c817f3a83fcc 100644
@@ -104,8 +104,8 @@ extern bool isa_io_special;
  *
  */
 
-#ifdef CONFIG_PPC64
-#define IO_SET_SYNC_FLAG()	do { local_paca->io_sync = 1; } while(0)
+#if defined(CONFIG_PPC64) && defined(CONFIG_SMP)
+#define IO_SET_SYNC_FLAG()	do { r14_set_bits(R14_BIT_IO_SYNC); } while(0)
 #else
 #define IO_SET_SYNC_FLAG()
 #endif

@@ -673,11 +673,8 @@ static inline void name at					\
  */
 static inline void mmiowb(void)
 {
-	unsigned long tmp;
-
-	__asm__ __volatile__("sync; li %0,0; stb %0,%1(13)"
-	: "=&r" (tmp) : "i" (offsetof(struct paca_struct, io_sync))
-	: "memory");
+	__asm__ __volatile__("sync" : : : "memory");
+	r14_clear_bits(R14_BIT_IO_SYNC);
 }
 #endif /* !CONFIG_PPC32 */
 

62 lines not on this page · raw message

index b9ebc3085fb7..182bb9304c79 100644
@@ -40,16 +40,9 @@
 #endif
 
 #if defined(CONFIG_PPC64) && defined(CONFIG_SMP)
-#define CLEAR_IO_SYNC	(get_paca()->io_sync = 0)
-#define SYNC_IO		do {						\
-				if (unlikely(get_paca()->io_sync)) {	\
-					mb();				\
-					get_paca()->io_sync = 0;	\
-				}					\
-			} while (0)
+#define CLEAR_IO_SYNC	do { r14_clear_bits(R14_BIT_IO_SYNC); } while(0)
 #else
 #define CLEAR_IO_SYNC
-#define SYNC_IO
 #endif
 
 #ifdef CONFIG_PPC_PSERIES

@@ -165,9 +158,19 @@ void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags)
 
 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 {
-	SYNC_IO;
+#if defined(CONFIG_PPC64) && defined(CONFIG_SMP)
+	bool io_sync = local_r14 & R14_BIT_IO_SYNC;
+	if (unlikely(io_sync)) {
+		mb();
+		CLEAR_IO_SYNC;
+	} else {
+		__asm__ __volatile__("# arch_spin_unlock\n\t"
+				PPC_RELEASE_BARRIER: : :"memory");
+	}
+#else
 	__asm__ __volatile__("# arch_spin_unlock\n\t"
 				PPC_RELEASE_BARRIER: : :"memory");
+#endif
 	lock->slock = 0;
 }
 

index a53454f61d09..40f0d02ae92d 100644
@@ -2393,7 +2393,6 @@ static void dump_one_paca(int cpu)
 	DUMP(p, trap_save, "x");
 	DUMP(p, irq_soft_mask, "x");
 	DUMP(p, irq_happened, "x");
-	DUMP(p, io_sync, "x");
 	DUMP(p, irq_work_pending, "x");
 	DUMP(p, nap_state_lost, "x");
 	DUMP(p, sprg_vdso, "llx");

index 408fa079e00d..cd3637f4ee4e 100644
@@ -36,7 +36,8 @@
 register struct paca_struct *local_paca asm("r13");
 #ifdef CONFIG_PPC_BOOK3S
 
-#define R14_BIT_IO_SYNC	0x0001
+#define R14_BIT_IO_SYNC			0x0001
+#define R14_BIT_IRQ_WORK_PENDING	0x0002 /* IRQ_WORK interrupt while soft-disable */
 
 /*
  * The top 32-bits of r14 is used as the per-cpu offset, shifted by PAGE_SHIFT.

@@ -212,7 +213,6 @@ struct paca_struct {
 	u16 trap_save;			/* Used when bad stack is encountered */
 	u8 irq_soft_mask;		/* mask for irq soft masking */
 	u8 irq_happened;		/* irq happened while soft-disabled */
-	u8 irq_work_pending;		/* IRQ_WORK interrupt while soft-disable */
 	u8 nap_state_lost;		/* NV GPR values lost in power7_idle */
 	u64 sprg_vdso;			/* Saved user-visible sprg */
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM

index 8d32ce95ec88..fac30152723f 100644
@@ -488,26 +488,17 @@ EXPORT_SYMBOL(profile_pc);
 #ifdef CONFIG_PPC64
 static inline unsigned long test_irq_work_pending(void)
 {
-	unsigned long x;
-
-	asm volatile("lbz %0,%1(13)"
-		: "=r" (x)
-		: "i" (offsetof(struct paca_struct, irq_work_pending)));
-	return x;
+	return local_r14 & R14_BIT_IRQ_WORK_PENDING;
 }
 
 static inline void set_irq_work_pending_flag(void)
 {
-	asm volatile("stb %0,%1(13)" : :
-		"r" (1),
-		"i" (offsetof(struct paca_struct, irq_work_pending)));
+	r14_set_bits(R14_BIT_IRQ_WORK_PENDING);
 }
 
 static inline void clear_irq_work_pending(void)
 {
-	asm volatile("stb %0,%1(13)" : :
-		"r" (0),
-		"i" (offsetof(struct paca_struct, irq_work_pending)));
+	r14_clear_bits(R14_BIT_IRQ_WORK_PENDING);
 }
 
 #else /* 32-bit */

index 40f0d02ae92d..7d2bb26ff333 100644
@@ -2393,7 +2393,6 @@ static void dump_one_paca(int cpu)
 	DUMP(p, trap_save, "x");
 	DUMP(p, irq_soft_mask, "x");
 	DUMP(p, irq_happened, "x");
-	DUMP(p, irq_work_pending, "x");
 	DUMP(p, nap_state_lost, "x");
 	DUMP(p, sprg_vdso, "llx");
 

index dadaa7471755..5602454ae56f 100644
@@ -459,9 +459,8 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 	mflr	r9;			/* Get LR, later save to stack	*/ \
 	ld	r2,PACATOC(r13);	/* get kernel TOC into r2	*/ \
 	std	r9,_LINK(r1);						   \
-	lbz	r10,PACAIRQSOFTMASK(r13);				   \
 	mfspr	r11,SPRN_XER;		/* save XER in stackframe	*/ \
-	std	r10,SOFTE(r1);						   \
+	std	r14,SOFTE(r1);		/* full r14 not just softe XXX	*/ \
 	std	r11,_XER(r1);						   \
 	li	r9,(n)+1;						   \
 	std	r9,_TRAP(r1);		/* set trap number		*/ \

@@ -526,8 +525,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 #define SOFTEN_VALUE_0xf00	PACA_IRQ_PMI
 
 #define __SOFTEN_TEST(h, vec, bitmask)					\
-	lbz	r10,PACAIRQSOFTMASK(r13);				\
-	andi.	r10,r10,bitmask;					\
+	andi.	r10,r14,bitmask;					\
 	li	r10,SOFTEN_VALUE_##vec;					\
 	bne	masked_##h##interrupt
 

154 lines not on this page · raw message

index 492b0a9fa352..19a2752868f8 100644
@@ -47,14 +47,12 @@
  * be clobbered.
  */
 #define RECONCILE_IRQ_STATE(__rA, __rB)		\
-	lbz	__rA,PACAIRQSOFTMASK(r13);	\
 	lbz	__rB,PACAIRQHAPPENED(r13);	\
-	andi.	__rA,__rA,IRQ_SOFT_MASK_STD;	\
-	li	__rA,IRQ_SOFT_MASK_STD;		\
+	andi.	__rA,r14,IRQ_SOFT_MASK_STD;	\
+	ori	r14,r14,IRQ_SOFT_MASK_STD;	\
 	ori	__rB,__rB,PACA_IRQ_HARD_DIS;	\
 	stb	__rB,PACAIRQHAPPENED(r13);	\
 	bne	44f;				\
-	stb	__rA,PACAIRQSOFTMASK(r13);	\
 	TRACE_DISABLE_INTS;			\
 44:
 

@@ -64,9 +62,8 @@
 
 #define RECONCILE_IRQ_STATE(__rA, __rB)		\
 	lbz	__rA,PACAIRQHAPPENED(r13);	\
-	li	__rB,IRQ_SOFT_MASK_STD;		\
+	ori	r14,r14,IRQ_SOFT_MASK_STD;	\
 	ori	__rA,__rA,PACA_IRQ_HARD_DIS;	\
-	stb	__rB,PACAIRQSOFTMASK(r13);	\
 	stb	__rA,PACAIRQHAPPENED(r13)
 #endif
 #endif

index 08053e596753..028b7cefe089 100644
@@ -873,7 +873,7 @@ static inline void kvmppc_fix_ee_before_entry(void)
 
 	/* Only need to enable IRQs by hard enabling them after this */
 	local_paca->irq_happened = 0;
-	irq_soft_mask_set(IRQ_SOFT_MASK_NONE);
+	__irq_soft_mask_clear(IRQ_SOFT_MASK_ALL);
 #endif
 }
 

index cd3637f4ee4e..dbf80fff2f53 100644
@@ -36,8 +36,10 @@
 register struct paca_struct *local_paca asm("r13");
 #ifdef CONFIG_PPC_BOOK3S
 
-#define R14_BIT_IO_SYNC			0x0001
-#define R14_BIT_IRQ_WORK_PENDING	0x0002 /* IRQ_WORK interrupt while soft-disable */
+#define R14_BIT_IRQ_SOFT_MASK_SHIFT	0
+#define R14_BIT_IRQ_SOFT_MASK		(0x3 << R14_BIT_IRQ_SOFT_MASK_SHIFT)
+#define R14_BIT_IO_SYNC			0x0004
+#define R14_BIT_IRQ_WORK_PENDING	0x0008 /* IRQ_WORK interrupt while soft-disable */
 
 /*
  * The top 32-bits of r14 is used as the per-cpu offset, shifted by PAGE_SHIFT.

@@ -79,6 +81,17 @@ static inline void r14_flip_bits(unsigned long mask)
 				: "0" (local_r14), "r" (mask));
 }
 
+static inline void r14_insert_bits(unsigned long source, unsigned long mask)
+{
+	unsigned long first = ffs(mask) - 1;
+	unsigned long last = fls64(mask) - 1;
+
+	mask >>= first;
+	asm volatile("rldimi	%0,%2,%3,%4\n"
+			: "=r" (local_r14)
+			: "0" (local_r14), "r" (source), "i" (first), "i" (63 - last));
+}
+
 static inline void r14_clear_bits(unsigned long mask)
 {
 	asm volatile("andc	%0,%0,%2\n"

@@ -211,7 +224,6 @@ struct paca_struct {
 	u64 saved_r1;			/* r1 save for RTAS calls or PM */
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
 	u16 trap_save;			/* Used when bad stack is encountered */
-	u8 irq_soft_mask;		/* mask for irq soft masking */
 	u8 irq_happened;		/* irq happened while soft-disabled */
 	u8 nap_state_lost;		/* NV GPR values lost in power7_idle */
 	u64 sprg_vdso;			/* Saved user-visible sprg */

index 32d393f55a96..c5c005d354b0 100644
@@ -83,6 +83,12 @@ int main(void)
 #ifdef CONFIG_PPC64
 	DEFINE(SIGSEGV, SIGSEGV);
 	DEFINE(NMI_MASK, NMI_MASK);
+	DEFINE(R14_BIT_IRQ_SOFT_MASK_SHIFT, R14_BIT_IRQ_SOFT_MASK_SHIFT);
+	DEFINE(R14_BIT_IRQ_SOFT_MASK, R14_BIT_IRQ_SOFT_MASK);
+	DEFINE(IRQ_SOFT_MASK_STD, IRQ_SOFT_MASK_STD);
+	DEFINE(IRQ_SOFT_MASK_PMU, IRQ_SOFT_MASK_PMU);
+	DEFINE(IRQ_SOFT_MASK_ALL, IRQ_SOFT_MASK_ALL);
+
 	OFFSET(TASKTHREADPPR, task_struct, thread.ppr);
 #else
 	OFFSET(THREAD_INFO, task_struct, stack);

@@ -178,7 +184,6 @@ int main(void)
 	OFFSET(PACATOC, paca_struct, kernel_toc);
 	OFFSET(PACAKBASE, paca_struct, kernelbase);
 	OFFSET(PACAKMSR, paca_struct, kernel_msr);
-	OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
 	OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
 #ifdef CONFIG_PPC_BOOK3S
 	OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);

index e7619d144c15..2341029653e4 100644
@@ -230,10 +230,13 @@ notrace void arch_local_irq_restore(unsigned long mask)
 	unsigned int replay;
 
 	/* Write the new soft-enabled value */
-	irq_soft_mask_set(mask);
+	__irq_soft_mask_insert(mask);
+	/* any bits still disabled */
 	if (mask)
 		return;
 
+	barrier();
+
 	/*
 	 * From this point onward, we can take interrupts, preempt,
 	 * etc... unless we got hard-disabled. We check if an event

@@ -261,6 +264,7 @@ notrace void arch_local_irq_restore(unsigned long mask)
 	 * then we are already hard disabled (there are other less
 	 * common cases that we'll ignore for now), so we skip the
 	 * (expensive) mtmsrd.
+	 * XXX: why not test & IRQ_HARD_DIS?
 	 */
 	if (unlikely(irq_happened != PACA_IRQ_HARD_DIS))
 		__hard_irq_disable();

@@ -277,7 +281,7 @@ notrace void arch_local_irq_restore(unsigned long mask)
 	}
 #endif
 
-	irq_soft_mask_set(IRQ_SOFT_MASK_ALL);
+	__irq_soft_mask_set(IRQ_SOFT_MASK_ALL);
 	trace_hardirqs_off();
 
 	/*

@@ -289,7 +293,7 @@ notrace void arch_local_irq_restore(unsigned long mask)
 
 	/* We can soft-enable now */
 	trace_hardirqs_on();
-	irq_soft_mask_set(IRQ_SOFT_MASK_NONE);
+	__irq_soft_mask_clear(IRQ_SOFT_MASK_ALL);
 
 	/*
 	 * And replay if we have to. This will return with interrupts

@@ -364,7 +368,7 @@ bool prep_irq_for_idle(void)
 	 * of entering the low power state.
 	 */
 	local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
-	irq_soft_mask_set(IRQ_SOFT_MASK_NONE);
+	__irq_soft_mask_clear(IRQ_SOFT_MASK_ALL);
 
 	/* Tell the caller to enter the low power state */
 	return true;

index f56e42f06f24..5914da5db7d9 100644
@@ -1675,7 +1675,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
 			childregs->gpr[FIRST_NVGPR] = ppc_function_entry((void *)usp);
 #ifdef CONFIG_PPC64
 		clear_tsk_thread_flag(p, TIF_32BIT);
-		childregs->softe = IRQ_SOFT_MASK_NONE;
+		childregs->softe = 0;
 #endif
 		childregs->gpr[FIRST_NVGPR + 1] = kthread_arg;
 		p->thread.regs = NULL;	/* no user register state */

index f4a96ebb523a..6e4f4e46b76b 100644
@@ -190,11 +190,12 @@ static void __init fixup_boot_paca(void)
 	get_paca()->cpu_start = 1;
 	/* Allow percpu accesses to work until we setup percpu data */
 	get_paca()->data_offset = 0;
-	/* Mark interrupts disabled in PACA */
-	irq_soft_mask_set(IRQ_SOFT_MASK_STD);
 	/* Set r14 and paca_r14 to zero */
 	get_paca()->r14 = 0;
 	local_r14 = get_paca()->r14;
+
+	/* Mark interrupts disabled in PACA or r14 */
+	__irq_soft_mask_set(IRQ_SOFT_MASK_STD);
 }
 
 static void __init configure_exceptions(void)

@@ -356,8 +357,8 @@ void __init early_setup(unsigned long dt_ptr)
 #ifdef CONFIG_SMP
 void early_setup_secondary(void)
 {
-	/* Mark interrupts disabled in PACA */
-	irq_soft_mask_set(IRQ_SOFT_MASK_STD);
+	/* Mark interrupts disabled in r14 */
+	__irq_soft_mask_set(IRQ_SOFT_MASK_STD);
 
 	/* Initialize the hash table or TLB handling */
 	early_init_mmu_secondary();

@@ -768,7 +769,7 @@ void __init setup_per_cpu_areas(void)
 		/* The top 48 bits are used for per-cpu data */
 		paca[cpu].r14 |= paca[cpu].data_offset << 16;
 	}
-	local_r14 = paca[smp_processor_id()].r14;
+	local_r14 |= paca[smp_processor_id()].r14;
 }
 #endif
 

index 7d2bb26ff333..d7d3885035f2 100644
@@ -1622,8 +1622,8 @@ static void excprint(struct pt_regs *fp)
 
 	printf("  current = 0x%lx\n", current);
 #ifdef CONFIG_PPC64
-	printf("  paca    = 0x%lx\t softe: %d\t irq_happened: 0x%02x\n",
-	       local_paca, local_paca->irq_soft_mask, local_paca->irq_happened);
+	printf("  paca    = 0x%lx\t r14: 0x%lx\t irq_happened: 0x%02x\n",
+	       local_paca, local_r14, local_paca->irq_happened);
 #endif
 	if (current) {
 		printf("    pid   = %ld, comm = %s\n",

@@ -2391,7 +2391,6 @@ static void dump_one_paca(int cpu)
 	DUMP(p, stab_rr, "lx");
 	DUMP(p, saved_r1, "lx");
 	DUMP(p, trap_save, "x");
-	DUMP(p, irq_soft_mask, "x");
 	DUMP(p, irq_happened, "x");
 	DUMP(p, nap_state_lost, "x");
 	DUMP(p, sprg_vdso, "llx");

index 9ba445de989d..f492a7779ea3 100644
@@ -22,6 +22,7 @@
  * and allow a proper replay. Additionally, PACA_IRQ_HARD_DIS
  * is set whenever we manually hard disable.
  */
+#ifdef CONFIG_PPC_BOOK3E
 #define PACA_IRQ_HARD_DIS	0x01
 #define PACA_IRQ_DBELL		0x02
 #define PACA_IRQ_EE		0x04

@@ -30,14 +31,22 @@
 #define PACA_IRQ_HMI		0x20
 #define PACA_IRQ_PMI		0x40
 
+#else /* CONFIG_PPC_BOOK3E */
 /*
- * 64s uses r14 rather than paca for irq_soft_mask
+ * 64s uses r14 rather than paca for irq_soft_mask and irq_happened
  */
-#ifdef CONFIG_PPC_BOOK3S
+
+#define PACA_IRQ_HARD_DIS	(0x01 << R14_BIT_IRQ_HAPPENED_SHIFT)
+#define PACA_IRQ_DBELL		(0x02 << R14_BIT_IRQ_HAPPENED_SHIFT)
+#define PACA_IRQ_EE		(0x04 << R14_BIT_IRQ_HAPPENED_SHIFT)
+#define PACA_IRQ_DEC		(0x08 << R14_BIT_IRQ_HAPPENED_SHIFT)
+#define PACA_IRQ_HMI		(0x10 << R14_BIT_IRQ_HAPPENED_SHIFT)
+#define PACA_IRQ_PMI		(0x20 << R14_BIT_IRQ_HAPPENED_SHIFT)
+
 #define IRQ_SOFT_MASK_STD	(0x01 << R14_BIT_IRQ_SOFT_MASK_SHIFT)
 #define IRQ_SOFT_MASK_PMU	(0x02 << R14_BIT_IRQ_SOFT_MASK_SHIFT)
 #define IRQ_SOFT_MASK_ALL	(0x03 << R14_BIT_IRQ_SOFT_MASK_SHIFT)
-#endif /* CONFIG_PPC_BOOK3S */
+#endif /* CONFIG_PPC_BOOK3E */
 
 #endif /* CONFIG_PPC64 */
 

@@ -206,14 +215,14 @@ static inline bool arch_irqs_disabled(void)
 	unsigned long flags;						\
 	__hard_irq_disable();						\
 	flags = irq_soft_mask_set_return(IRQ_SOFT_MASK_ALL);		\
-	local_paca->irq_happened |= PACA_IRQ_HARD_DIS;			\
+	r14_set_bits(PACA_IRQ_HARD_DIS);				\
 	if (!arch_irqs_disabled_flags(flags))				\
 		trace_hardirqs_off();					\
 } while(0)
 
 static inline bool lazy_irq_pending(void)
 {
-	return !!(get_paca()->irq_happened & ~PACA_IRQ_HARD_DIS);
+	return !!(local_r14 & R14_BIT_IRQ_HAPPENED_MASK & ~PACA_IRQ_HARD_DIS);
 }
 
 /*

@@ -223,8 +232,8 @@ static inline bool lazy_irq_pending(void)
  */
 static inline void may_hard_irq_enable(void)
 {
-	get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
-	if (!(get_paca()->irq_happened & PACA_IRQ_EE))
+	r14_clear_bits(PACA_IRQ_HARD_DIS);
+	if (!(local_r14 & PACA_IRQ_EE))
 		__hard_irq_enable();
 }
 

index 19a2752868f8..140e51b9f436 100644
@@ -45,26 +45,21 @@
  *
  * NB: This may call C code, so the caller must be prepared for volatiles to
  * be clobbered.
+ * XXX: could make this single-register now
  */
-#define RECONCILE_IRQ_STATE(__rA, __rB)		\
-	lbz	__rB,PACAIRQHAPPENED(r13);	\
-	andi.	__rA,r14,IRQ_SOFT_MASK_STD;	\
-	ori	r14,r14,IRQ_SOFT_MASK_STD;	\
-	ori	__rB,__rB,PACA_IRQ_HARD_DIS;	\
-	stb	__rB,PACAIRQHAPPENED(r13);	\
-	bne	44f;				\
-	TRACE_DISABLE_INTS;			\
+#define RECONCILE_IRQ_STATE(__rA, __rB)					\
+	andi.	__rA,r14,IRQ_SOFT_MASK_STD;				\
+	ori	r14,r14,(PACA_IRQ_HARD_DIS | IRQ_SOFT_MASK_STD);	\
+	bne	44f;							\
+	TRACE_DISABLE_INTS;						\
 44:
 
 #else
 #define TRACE_ENABLE_INTS
 #define TRACE_DISABLE_INTS
 
-#define RECONCILE_IRQ_STATE(__rA, __rB)		\
-	lbz	__rA,PACAIRQHAPPENED(r13);	\
-	ori	r14,r14,IRQ_SOFT_MASK_STD;	\
-	ori	__rA,__rA,PACA_IRQ_HARD_DIS;	\
-	stb	__rA,PACAIRQHAPPENED(r13)
+#define RECONCILE_IRQ_STATE(__rA, __rB)					\
+	ori	r14,r14,(PACA_IRQ_HARD_DIS | IRQ_SOFT_MASK_STD)
 #endif
 #endif
 

index 028b7cefe089..45202e124003 100644
@@ -869,10 +869,10 @@ static inline void kvmppc_fix_ee_before_entry(void)
 	 * To avoid races, the caller must have gone directly from having
 	 * interrupts fully-enabled to hard-disabled.
 	 */
-	WARN_ON(local_paca->irq_happened != PACA_IRQ_HARD_DIS);
+	WARN_ON((local_r14 & R14_BIT_IRQ_HAPPENED_MASK) != PACA_IRQ_HARD_DIS);
 
 	/* Only need to enable IRQs by hard enabling them after this */
-	local_paca->irq_happened = 0;
+	r14_clear_bits(R14_BIT_IRQ_HAPPENED_MASK);
 	__irq_soft_mask_clear(IRQ_SOFT_MASK_ALL);
 #endif
 }

index dbf80fff2f53..4edfcdecb268 100644
@@ -38,8 +38,10 @@ register struct paca_struct *local_paca asm("r13");
 
 #define R14_BIT_IRQ_SOFT_MASK_SHIFT	0
 #define R14_BIT_IRQ_SOFT_MASK		(0x3 << R14_BIT_IRQ_SOFT_MASK_SHIFT)
-#define R14_BIT_IO_SYNC			0x0004
-#define R14_BIT_IRQ_WORK_PENDING	0x0008 /* IRQ_WORK interrupt while soft-disable */
+#define R14_BIT_IRQ_HAPPENED_SHIFT	2
+#define R14_BIT_IRQ_HAPPENED_MASK	(0x3f << R14_BIT_IRQ_HAPPENED_SHIFT) /* irq happened while soft-disabled */
+#define R14_BIT_IO_SYNC			0x0100
+#define R14_BIT_IRQ_WORK_PENDING	0x0200 /* IRQ_WORK interrupt while soft-disable */
 
 /*
  * The top 32-bits of r14 is used as the per-cpu offset, shifted by PAGE_SHIFT.

@@ -224,7 +226,6 @@ struct paca_struct {
 	u64 saved_r1;			/* r1 save for RTAS calls or PM */
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
 	u16 trap_save;			/* Used when bad stack is encountered */
-	u8 irq_happened;		/* irq happened while soft-disabled */
 	u8 nap_state_lost;		/* NV GPR values lost in power7_idle */
 	u64 sprg_vdso;			/* Saved user-visible sprg */
 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM

index c5c005d354b0..2f03f778baf2 100644
@@ -88,6 +88,14 @@ int main(void)
 	DEFINE(IRQ_SOFT_MASK_STD, IRQ_SOFT_MASK_STD);
 	DEFINE(IRQ_SOFT_MASK_PMU, IRQ_SOFT_MASK_PMU);
 	DEFINE(IRQ_SOFT_MASK_ALL, IRQ_SOFT_MASK_ALL);
+	DEFINE(R14_BIT_IRQ_HAPPENED_SHIFT, R14_BIT_IRQ_HAPPENED_SHIFT);
+	DEFINE(R14_BIT_IRQ_HAPPENED_MASK, R14_BIT_IRQ_HAPPENED_MASK);
+	DEFINE(PACA_IRQ_HARD_DIS, PACA_IRQ_HARD_DIS);
+	DEFINE(PACA_IRQ_EE, PACA_IRQ_EE);
+	DEFINE(PACA_IRQ_DBELL, PACA_IRQ_DBELL);
+	DEFINE(PACA_IRQ_DEC, PACA_IRQ_DEC);
+	DEFINE(PACA_IRQ_HMI, PACA_IRQ_HMI);
+	DEFINE(PACA_IRQ_PMI, PACA_IRQ_PMI);
 
 	OFFSET(TASKTHREADPPR, task_struct, thread.ppr);
 #else

@@ -184,7 +192,6 @@ int main(void)
 	OFFSET(PACATOC, paca_struct, kernel_toc);
 	OFFSET(PACAKBASE, paca_struct, kernelbase);
 	OFFSET(PACAKMSR, paca_struct, kernel_msr);
-	OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
 #ifdef CONFIG_PPC_BOOK3S
 	OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);
 #ifdef CONFIG_PPC_MM_SLICES

148 lines not on this page · raw message

index 2d46037ce936..870dd835c8b6 100644
@@ -2614,13 +2614,13 @@ static void set_irq_happened(int trap)
 {
 	switch (trap) {
 	case BOOK3S_INTERRUPT_EXTERNAL:
-		local_paca->irq_happened |= PACA_IRQ_EE;
+		r14_set_bits(PACA_IRQ_EE);
 		break;
 	case BOOK3S_INTERRUPT_H_DOORBELL:
-		local_paca->irq_happened |= PACA_IRQ_DBELL;
+		r14_set_bits(PACA_IRQ_DBELL);
 		break;
 	case BOOK3S_INTERRUPT_HMI:
-		local_paca->irq_happened |= PACA_IRQ_HMI;
+		r14_set_bits(PACA_IRQ_HMI);
 		break;
 	case BOOK3S_INTERRUPT_SYSTEM_RESET:
 		replay_system_reset();

index d7d3885035f2..df73f8a1f030 100644
@@ -1622,8 +1622,7 @@ static void excprint(struct pt_regs *fp)
 
 	printf("  current = 0x%lx\n", current);
 #ifdef CONFIG_PPC64
-	printf("  paca    = 0x%lx\t r14: 0x%lx\t irq_happened: 0x%02x\n",
-	       local_paca, local_r14, local_paca->irq_happened);
+	printf("  paca    = 0x%lx\t r14: %lx\n", local_paca, local_r14);
 #endif
 	if (current) {
 		printf("    pid   = %ld, comm = %s\n",

@@ -2391,7 +2390,6 @@ static void dump_one_paca(int cpu)
 	DUMP(p, stab_rr, "lx");
 	DUMP(p, saved_r1, "lx");
 	DUMP(p, trap_save, "x");
-	DUMP(p, irq_happened, "x");
 	DUMP(p, nap_state_lost, "x");
 	DUMP(p, sprg_vdso, "llx");
 

index f492a7779ea3..8690e0d5605d 100644
@@ -132,11 +132,22 @@ static inline void arch_local_irq_disable(void)
 	irq_soft_mask_set(IRQ_SOFT_MASK_STD);
 }
 
-extern void arch_local_irq_restore(unsigned long);
+extern void __arch_local_irq_enable(void);
 
 static inline void arch_local_irq_enable(void)
 {
-	arch_local_irq_restore(0);
+	__irq_soft_mask_clear(IRQ_SOFT_MASK_ALL);
+	if (unlikely(local_r14 & R14_BIT_IRQ_HAPPENED_MASK))
+		__arch_local_irq_enable();
+}
+
+static inline void arch_local_irq_restore(unsigned long flags)
+{
+	__irq_soft_mask_insert(flags);
+	if (!flags) {
+		if (unlikely(local_r14 & R14_BIT_IRQ_HAPPENED_MASK))
+			__arch_local_irq_enable();
+	}
 }
 
 static inline unsigned long arch_local_irq_save(void)

index ebaf210a7406..e2ff0210477e 100644
@@ -97,11 +97,6 @@ extern int tau_interrupts(int);
 
 int distribute_irqs = 1;
 
-static inline notrace unsigned long get_irq_happened(void)
-{
-	return local_r14 & R14_BIT_IRQ_HAPPENED_MASK;
-}
-
 static inline notrace int decrementer_check_overflow(void)
 {
  	u64 now = get_tb_or_rtc();

@@ -210,19 +205,10 @@ notrace unsigned int __check_irq_replay(void)
 	return 0;
 }
 
-notrace void arch_local_irq_restore(unsigned long mask)
+notrace void __arch_local_irq_enable(void)
 {
-	unsigned char irq_happened;
 	unsigned int replay;
 
-	/* Write the new soft-enabled value */
-	__irq_soft_mask_insert(mask);
-	/* any bits still disabled */
-	if (mask)
-		return;
-
-	barrier();
-
 	/*
 	 * From this point onward, we can take interrupts, preempt,
 	 * etc... unless we got hard-disabled. We check if an event

@@ -236,9 +222,6 @@ notrace void arch_local_irq_restore(unsigned long mask)
 	 * be hard-disabled, so there is no problem, we
 	 * cannot have preempted.
 	 */
-	irq_happened = get_irq_happened();
-	if (!irq_happened)
-		return;
 
 	/*
 	 * We need to hard disable to get a trusted value from

@@ -252,10 +235,11 @@ notrace void arch_local_irq_restore(unsigned long mask)
 	 * (expensive) mtmsrd.
 	 * XXX: why not test & IRQ_HARD_DIS?
 	 */
-	if (unlikely(irq_happened != PACA_IRQ_HARD_DIS))
+	if (unlikely((local_r14 & R14_BIT_IRQ_HAPPENED_MASK) !=
+						PACA_IRQ_HARD_DIS)) {
 		__hard_irq_disable();
 #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
-	else {
+	} else {
 		/*
 		 * We should already be hard disabled here. We had bugs
 		 * where that wasn't the case so let's dbl check it and

@@ -264,8 +248,8 @@ notrace void arch_local_irq_restore(unsigned long mask)
 		 */
 		if (WARN_ON(mfmsr() & MSR_EE))
 			__hard_irq_disable();
-	}
 #endif
+	}
 
 	__irq_soft_mask_set(IRQ_SOFT_MASK_ALL);
 	trace_hardirqs_off();

@@ -293,7 +277,7 @@ notrace void arch_local_irq_restore(unsigned long mask)
 	/* Finally, let's ensure we are hard enabled */
 	__hard_irq_enable();
 }
-EXPORT_SYMBOL(arch_local_irq_restore);
+EXPORT_SYMBOL(__arch_local_irq_enable);
 
 /*
  * This is specifically called by assembly code to re-enable interrupts