Skip to contentSkip to conversation

xen/x86: add diagnostic printout to xen_mc_flush() in case of error

PATCH In review Juergen Gross 2 participants 7 messages Nov 23, 2018 – Nov 27, 2018

Subsystems (2)
lore
Share & tools
lore

Juergen Gross

author maintainer
#0 lore raw

PATCH 1/1 xen/x86: add diagnostic printout to xen_mc_flush() in case of error

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +20 -15
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
+#else
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif
 			}
 		}
-#endif
 	}
 
 	b->mcidx = 0;
@@ -126,8 +133,6 @@ void xen_mc_flush(void)
 	b->cbidx = 0;
 
 	local_irq_restore(flags);
-
-	WARN_ON(ret);
 }
 
 struct multicall_space __xen_mc_entry(size_t args)
Signature · 1 line
2.16.4
#1 lore raw
Quoted 2 lines · Juergen Gross

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to

s/unless/if

Quoted 62 lines

know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +13 -12
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 120

Quoted 10 lines
arch/x86/xen/multicalls.c +7 -0
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
+#else
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif

Doesn't (non-debug) hypervisor corrupt op and args?

(Also, we don't really need to print anything when b->entries[i].result == 0)

-boris

Quoted 7 lines
arch/x86/xen/multicalls.c +0 -1
 			}
 		}
-#endif
 	}
 
 	b->mcidx = 0;

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 138

Quoted 8 lines
arch/x86/xen/multicalls.c +0 -2
@@ -126,8 +133,6 @@ void xen_mc_flush(void)
 	b->cbidx = 0;
 
 	local_irq_restore(flags);
-
-	WARN_ON(ret);
 }
 
 struct multicall_space __xen_mc_entry(size_t args)

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

#2 lore raw
In reply to Juergen Gross
Quoted 2 lines · Juergen Gross

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to

s/unless/if

Quoted 62 lines

know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +13 -12
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 120

Quoted 10 lines
arch/x86/xen/multicalls.c +7 -0
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
+#else
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif

Doesn't (non-debug) hypervisor corrupt op and args?

(Also, we don't really need to print anything when b->entries[i].result == 0)

-boris

Quoted 15 lines
arch/x86/xen/multicalls.c +0 -3
 			}
 		}
-#endif
 	}
 
 	b->mcidx = 0;
@@ -126,8 +133,6 @@ void xen_mc_flush(void)
 	b->cbidx = 0;
 
 	local_irq_restore(flags);
-
-	WARN_ON(ret);
 }
 
 struct multicall_space __xen_mc_entry(size_t args)
author maintainer
#3 lore raw
Quoted 73 lines · Boris Ostrovsky
Quoted 2 lines · Juergen Gross

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to

s/unless/if

Quoted 66 lines

know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +14 -12
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
+#else
Quoted 8 lines

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 120

Quoted 6 lines
arch/x86/xen/multicalls.c +6 -0
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif

Doesn't (non-debug) hypervisor corrupt op and args?

No. Only debug hypervisor does so.

See my patch (and rather lengthy discussion) on xen-devel:

https://lists.xen.org/archives/html/xen-devel/2018-11/msg02714.html

Quoted 3 lines

(Also, we don't really need to print anything when b->entries[i].result == 0)

Right. Did you miss the:

+ if (b->entries[i].result < 0) {

above?

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

author maintainer
#4 lore raw
In reply to Boris Ostrovsky
Quoted 73 lines · Boris Ostrovsky
Quoted 2 lines · Juergen Gross

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to

s/unless/if

Quoted 66 lines

know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +14 -12
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
+#else
Quoted 8 lines

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 120

Quoted 6 lines
arch/x86/xen/multicalls.c +6 -0
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif

Doesn't (non-debug) hypervisor corrupt op and args?

No. Only debug hypervisor does so.

See my patch (and rather lengthy discussion) on xen-devel:

https://lists.xen.org/archives/html/xen-devel/2018-11/msg02714.html

Quoted 3 lines

(Also, we don't really need to print anything when b->entries[i].result == 0)

Right. Did you miss the:

+ if (b->entries[i].result < 0) {

above?

Juergen

reviewer Reviewed-by
#5 lore raw
Quoted 80 lines · Juergen Gross
Quoted 71 lines · Boris Ostrovsky
Quoted 2 lines · Juergen Gross

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to

s/unless/if

Quoted 65 lines

know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +13 -12
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
Quoted 8 lines

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 120

Quoted 7 lines
arch/x86/xen/multicalls.c +7 -0
+#else
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif

Doesn't (non-debug) hypervisor corrupt op and args?

Quoted 5 lines

No. Only debug hypervisor does so.

See my patch (and rather lengthy discussion) on xen-devel:

https://lists.xen.org/archives/html/xen-devel/2018-11/msg02714.html

Yes, I saw that later, after I responded to the patch.

Quoted 4 lines
Quoted 3 lines

(Also, we don't really need to print anything when b->entries[i].result == 0)

Right. Did you miss the:

I did.

(with commit message fixed)

Quoted 7 lines
arch/x86/xen/multicalls.c +1 -0
+			if (b->entries[i].result < 0) {

above?

Juergen

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

reviewer Reviewed-by
#6 lore raw
In reply to Juergen Gross
Quoted 80 lines · Juergen Gross
Quoted 71 lines · Boris Ostrovsky
Quoted 2 lines · Juergen Gross

Failure of an element of a Xen multicall is signalled via a WARN() only unless the kernel is compiled with MC_DEBUG. It is impossible to

s/unless/if

Quoted 65 lines

know which element failed and why it did so.

Change that by printing the related information even without MC_DEBUG, even if maybe in some limited form (e.g. without information which caller produced the failing element).

Move the printing out of the switch statement in order to have the same information for a single call.

---
 arch/x86/xen/multicalls.c | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)
arch/x86/xen/multicalls.c +13 -12
index 2bce7958ce8b..0766a08bdf45 100644
@@ -69,6 +69,11 @@ void xen_mc_flush(void)
 
 	trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx);
 
+#if MC_DEBUG
+	memcpy(b->debug, b->entries,
+	       b->mcidx * sizeof(struct multicall_entry));
+#endif
+
 	switch (b->mcidx) {
 	case 0:
 		/* no-op */
@@ -87,32 +92,34 @@ void xen_mc_flush(void)
 		break;
 
 	default:
-#if MC_DEBUG
-		memcpy(b->debug, b->entries,
-		       b->mcidx * sizeof(struct multicall_entry));
-#endif
-
 		if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0)
 			BUG();
 		for (i = 0; i < b->mcidx; i++)
 			if (b->entries[i].result < 0)
 				ret++;
+	}
 
+	if (WARN_ON(ret)) {
+		pr_err("%d of %d multicall(s) failed: cpu %d\n",
+		       ret, b->mcidx, smp_processor_id());
+		for (i = 0; i < b->mcidx; i++) {
+			if (b->entries[i].result < 0) {
 #if MC_DEBUG
-		if (ret) {
-			printk(KERN_ERR "%d multicall(s) failed: cpu %d\n",
-			       ret, smp_processor_id());
-			dump_stack();
-			for (i = 0; i < b->mcidx; i++) {
-				printk(KERN_DEBUG "  call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n",
-				       i+1, b->mcidx,
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\t%pF\n",
+				       i + 1,
 				       b->debug[i].op,
 				       b->debug[i].args[0],
 				       b->entries[i].result,
 				       b->caller[i]);
Quoted 8 lines

arch/x86/xen/multicalls.c void xen_mc_flush(void) · line 120

Quoted 7 lines
arch/x86/xen/multicalls.c +7 -0
+#else
+				pr_err("  call %2d: op=%lu arg=[%lx] result=%ld\n",
+				       i + 1,
+				       b->entries[i].op,
+				       b->entries[i].args[0],
+				       b->entries[i].result);
+#endif

Doesn't (non-debug) hypervisor corrupt op and args?

Quoted 5 lines

No. Only debug hypervisor does so.

See my patch (and rather lengthy discussion) on xen-devel:

https://lists.xen.org/archives/html/xen-devel/2018-11/msg02714.html

Yes, I saw that later, after I responded to the patch.

Quoted 4 lines
Quoted 3 lines

(Also, we don't really need to print anything when b->entries[i].result == 0)

Right. Did you miss the:

I did.

(with commit message fixed)

Quoted 7 lines
arch/x86/xen/multicalls.c +1 -0
+			if (b->entries[i].result < 0) {

above?

Juergen