powerpc: Rework lazy-interrupt handling

index 8057f4f..b3f42e9 100644
@@ -232,23 +232,24 @@ label##_hv:						\
 	EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common,	\
 				 EXC_HV, KVMTEST, vec)
 
-#define __SOFTEN_TEST(h)						\
+#define __SOFTEN_TEST(h, vec)						\
 	lbz	r10,PACASOFTIRQEN(r13);					\
 	cmpwi	r10,0;							\
+	li	r10,(vec)>>8;						\
 	beq	masked_##h##interrupt
-#define _SOFTEN_TEST(h)	__SOFTEN_TEST(h)
+#define _SOFTEN_TEST(h, vec)	__SOFTEN_TEST(h, vec)
 
 #define SOFTEN_TEST_PR(vec)						\
 	KVMTEST_PR(vec);						\
-	_SOFTEN_TEST(EXC_STD)
+	_SOFTEN_TEST(EXC_STD, vec)
 
 #define SOFTEN_TEST_HV(vec)						\
 	KVMTEST(vec);							\
-	_SOFTEN_TEST(EXC_HV)
+	_SOFTEN_TEST(EXC_HV, vec)
 
 #define SOFTEN_TEST_HV_201(vec)						\
 	KVMTEST(vec);							\
-	_SOFTEN_TEST(EXC_STD)
+	_SOFTEN_TEST(EXC_STD, vec)
 
 #define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra)		\
 	HMT_MEDIUM;							\

@@ -276,9 +277,9 @@ label##_hv:								\
 #define DISABLE_INTS				\
 	li	r11,0;				\
 	stb	r11,PACASOFTIRQEN(r13);		\
-BEGIN_FW_FTR_SECTION;				\
-	stb	r11,PACAHARDIRQEN(r13);		\
-END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);	\
+	lbz	r11,PACAIRQHAPPENED(r13);	\
+	ori	r11,r11,PACA_HAPPENED;		\
+	stb	r11,PACAIRQHAPPENED(r13);	\
 	TRACE_DISABLE_INTS;			\
 BEGIN_FW_FTR_SECTION;				\
 	mfmsr	r10;				\

@@ -289,7 +290,9 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #define DISABLE_INTS				\
 	li	r11,0;				\
 	stb	r11,PACASOFTIRQEN(r13);		\
-	stb	r11,PACAHARDIRQEN(r13);		\
+	lbz	r11,PACAIRQHAPPENED(r13);	\
+	ori	r11,r11,PACA_HAPPENED;		\
+	stb	r11,PACAIRQHAPPENED(r13);	\
 	TRACE_DISABLE_INTS
 #endif /* CONFIG_PPC_ISERIES */
 

79 lines not on this page · raw message

index b0b06d8..4bfbf0a 100644
@@ -47,16 +47,15 @@
 	b	skip;					\
 95:	TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_on)	\
 	li	en,1;
-#define TRACE_AND_RESTORE_IRQ(en)		\
-	TRACE_AND_RESTORE_IRQ_PARTIAL(en,96f);	\
-	stb	en,PACASOFTIRQEN(r13);		\
-96:
 #else
 #define TRACE_ENABLE_INTS
 #define TRACE_DISABLE_INTS
-#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip)
-#define TRACE_AND_RESTORE_IRQ(en)		\
-	stb	en,PACASOFTIRQEN(r13)
+#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip)		\
+	cmpdi	en,0;					\
+	bne	95f;					\
+	stb	en,PACASOFTIRQEN(r13);			\
+	b	skip;					\
+95:
 #endif
 #endif
 

index 269c05a..daf813f 100644
@@ -132,7 +132,7 @@ struct paca_struct {
 	u64 saved_msr;			/* MSR saved here by enter_rtas */
 	u16 trap_save;			/* Used when bad stack is encountered */
 	u8 soft_enabled;		/* irq soft-enable flag */
-	u8 hard_enabled;		/* set if irqs are enabled in MSR */
+	u8 irq_happened;		/* irq happened while soft-disabled */
 	u8 io_sync;			/* writel() needs spin_unlock sync */
 	u8 irq_work_pending;		/* IRQ_WORK interrupt while soft-disable */
 	u8 nap_state_lost;		/* NV GPR values lost in power7_idle */

index 04caee7..cdd0d26 100644
@@ -147,7 +147,7 @@ int main(void)
 	DEFINE(PACAKBASE, offsetof(struct paca_struct, kernelbase));
 	DEFINE(PACAKMSR, offsetof(struct paca_struct, kernel_msr));
 	DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
-	DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
+	DEFINE(PACAIRQHAPPENED, offsetof(struct paca_struct, irq_happened));
 	DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
 #ifdef CONFIG_PPC_MM_SLICES
 	DEFINE(PACALOWSLICESPSIZE, offsetof(struct paca_struct,

index 2cc451a..16f0e5e 100644
@@ -37,6 +37,18 @@ void doorbell_exception(struct pt_regs *regs)
 
 	irq_enter();
 
+#ifdef CONFIG_PPC64
+	/* Let's hard enable interrupts now that we have reset
+	 * the DEC (or acked it on BookE)
+	 *
+	 * We skip that if there's a pending EE "level" interrupt
+	 * as an optimization
+	 */
+	get_paca()->irq_happened &= ~PACA_HAPPENED;
+	if (!(get_paca()->irq_happened & PACA_HAPPENED_EE))
+		__hard_irq_enable();
+#endif /* CONFIG_PPC64 */
+
 	smp_ipi_demux();
 
 	irq_exit();

242 lines not on this page · raw message

index 567dd7c..39f201f 100644
@@ -259,7 +259,6 @@ void accumulate_stolen_time(void)
 	u64 sst, ust;
 
 	u8 save_soft_enabled = local_paca->soft_enabled;
-	u8 save_hard_enabled = local_paca->hard_enabled;
 
 	/* We are called early in the exception entry, before
 	 * soft/hard_enabled are sync'ed to the expected state

@@ -268,7 +267,6 @@ void accumulate_stolen_time(void)
 	 * complain
 	 */
 	local_paca->soft_enabled = 0;
-	local_paca->hard_enabled = 0;
 
 	sst = scan_dispatch_log(local_paca->starttime_user);
 	ust = scan_dispatch_log(local_paca->starttime);

@@ -277,7 +275,6 @@ void accumulate_stolen_time(void)
 	local_paca->stolen_time += ust + sst;
 
 	local_paca->soft_enabled = save_soft_enabled;
-	local_paca->hard_enabled = save_hard_enabled;
 }
 
 static inline u64 calculate_stolen_time(u64 stop_tb)

@@ -589,6 +586,18 @@ void timer_interrupt(struct pt_regs * regs)
 		do_IRQ(regs);
 #endif
 
+#ifdef CONFIG_PPC64
+	/* Let's hard enable interrupts now that we have reset
+	 * the DEC (or acked it on BookE)
+	 *
+	 * We skip that if there's a pending EE "level" interrupt
+	 * as an optimization
+	 */
+	get_paca()->irq_happened &= ~PACA_HAPPENED;
+	if (!(get_paca()->irq_happened & PACA_HAPPENED_EE))
+		__hard_irq_enable();	
+#endif /* CONFIG_PPC64 */
+
 	old_regs = set_irq_regs(regs);
 	irq_enter();
 

index 085fd3f..019a529 100644
@@ -96,6 +96,25 @@ out:
 	return index;
 }
 
+static int check_and_cede_processor(void)
+{
+	/*
+	 * Interrupts are soft-disabled at this point,
+	 * but not hard disabled. So an interrupt might have
+	 * occurred before entering NAP, and would be potentially
+	 * lost (edge events, decrementer events, etc...) unless
+	 * we first hard disable then check.
+	 *
+	 * We must use the low level __hard_irq_disable() and not
+	 * hard_irq_disable() as the later will set a bit in
+	 * paca->irq_happened (to force re-enable later) which we
+	 * don't need nor want here.
+	 */
+	__hard_irq_disable();
+	if (get_paca()->irq_happened == 0)
+		cede_processor();
+}
+
 static int dedicated_cede_loop(struct cpuidle_device *dev,
 				struct cpuidle_driver *drv,
 				int index)

@@ -108,7 +127,8 @@ static int dedicated_cede_loop(struct cpuidle_device *dev,
 
 	ppc64_runlatch_off();
 	HMT_medium();
-	cede_processor();
+
+	check_and_cede_processor();
 
 	get_lppaca()->donate_dedicated_cpu = 0;
 	dev->last_residency =

@@ -132,7 +152,7 @@ static int shared_cede_loop(struct cpuidle_device *dev,
 	 * processor. When returning here, external interrupts
 	 * are enabled.
 	 */
-	cede_processor();
+	check_and_cede_processor();
 
 	dev->last_residency =
 		(int)idle_loop_epilog(in_purr, kt_before);