Skip to contentSkip to conversation

More fixes for BMC failure handling

PATCH In review Corey Minyard 2 participants 7 messages Feb 13

Subsystems (3)
lore
Share & tools
lore

Corey Minyard

author maintainer
#0 lore raw

cover More fixes for BMC failure handling

These are in addition to the previous patches for this.

author maintainer
#1 lore raw

PATCH 1/3 ipmi:si: Don't block module unload if the BMC is messed up

If the BMC is in a bad state, don't bother waiting for queues messages since there can't be any. Otherwise the unload is blocked until the BMC is back in a good state.

Fixes bc3a9d217755 ("ipmi:si: Gracefully handle if the BMC is non-functional")
---
 drivers/char/ipmi/ipmi_si_intf.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/char/ipmi/ipmi_si_intf.c +2 -1
index 0049e3792ba1..3667033fcc51 100644
@@ -2234,7 +2234,8 @@ static void wait_msg_processed(struct smi_info *smi_info)
 	unsigned long jiffies_now;
 	long time_diff;
 
-	while (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL)) {
+	while (smi_info->si_state != SI_HOSED &&
+		    (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL))) {
 		jiffies_now = jiffies;
 		time_diff = (((long)jiffies_now - (long)smi_info->last_timeout_jiffies)
 		     * SI_USEC_PER_JIFFY);
Signature · 1 line
2.43.0
maintainer Reviewed-by on v1 1/3
#2 lore raw

Re: [PATCH 1/3] ipmi:si: Don't block module unload if the BMC is messed up

Quoted 8 lines · Corey Minyard

If the BMC is in a bad state, don't bother waiting for queues messages since there can't be any. Otherwise the unload is blocked until the BMC is back in a good state.

Fixes bc3a9d217755 ("ipmi:si: Gracefully handle if the BMC is non-functional")

I have missed this case, but indeed it needs to be addressed.

Quoted 19 lines
---
 drivers/char/ipmi/ipmi_si_intf.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/char/ipmi/ipmi_si_intf.c +2 -2
index 0049e3792ba1..3667033fcc51 100644
@@ -2234,7 +2234,8 @@ static void wait_msg_processed(struct smi_info *smi_info)
        unsigned long jiffies_now;
        long time_diff;
 
-       while (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL)) {
+       while (smi_info->si_state != SI_HOSED &&
+                   (smi_info->curr_msg || (smi_info->si_state != SI_NORMAL))) {
                jiffies_now = jiffies;
                time_diff = (((long)jiffies_now - (long)smi_info->last_timeout_jiffies)
                     * SI_USEC_PER_JIFFY);
--
author maintainer
#3 lore raw

PATCH 2/3 ipmi:msghandler: Handle error returns from the SMI sender

In reply to Corey Minyard

It used to be, until recently, that the sender operation on the low level interfaces would not fail. That's not the case any more with recent changes.

So check the return value from the sender operation, and propagate it back up from there and handle the errors in all places.

Fixes bc3a9d217755 ("ipmi:si: Gracefully handle if the BMC is non-functional")
---
 drivers/char/ipmi/ipmi_msghandler.c | 97 +++++++++++++++++++----------
 1 file changed, 65 insertions(+), 32 deletions(-)
drivers/char/ipmi/ipmi_msghandler.c +65 -32
index a042b1596933..c209486bfee7 100644
@@ -1887,19 +1887,32 @@ static struct ipmi_smi_msg *smi_add_send_msg(struct ipmi_smi *intf,
 	return smi_msg;
 }
 
-static void smi_send(struct ipmi_smi *intf,
+static int smi_send(struct ipmi_smi *intf,
 		     const struct ipmi_smi_handlers *handlers,
 		     struct ipmi_smi_msg *smi_msg, int priority)
 {
 	int run_to_completion = READ_ONCE(intf->run_to_completion);
 	unsigned long flags = 0;
+	int rv = 0;
 
 	ipmi_lock_xmit_msgs(intf, run_to_completion, &flags);
 	smi_msg = smi_add_send_msg(intf, smi_msg, priority);
 	ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags);
 
-	if (smi_msg)
-		handlers->sender(intf->send_info, smi_msg);
+	if (smi_msg) {
+		rv = handlers->sender(intf->send_info, smi_msg);
+		if (rv) {
+			ipmi_lock_xmit_msgs(intf, run_to_completion, &flags);
+			intf->curr_msg = NULL;
+			ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags);
+			/*
+			 * Something may have been added to the transmit
+			 * queue, so schedule a check for that.
+			 */
+			queue_work(system_wq, &intf->smi_work);
+		}
+	}
+	return rv;
 }
 
 static bool is_maintenance_mode_cmd(struct kernel_ipmi_msg *msg)
@@ -2312,6 +2325,7 @@ static int i_ipmi_request(struct ipmi_user     *user,
 	struct ipmi_recv_msg *recv_msg;
 	int run_to_completion = READ_ONCE(intf->run_to_completion);
 	int rv = 0;
+	bool in_seq_table = false;
 
 	if (supplied_recv) {
 		recv_msg = supplied_recv;
@@ -2365,33 +2379,47 @@ static int i_ipmi_request(struct ipmi_user     *user,
 		rv = i_ipmi_req_ipmb(intf, addr, msgid, msg, smi_msg, recv_msg,
 				     source_address, source_lun,
 				     retries, retry_time_ms);
+		in_seq_table = true;
 	} else if (is_ipmb_direct_addr(addr)) {
 		rv = i_ipmi_req_ipmb_direct(intf, addr, msgid, msg, smi_msg,
 					    recv_msg, source_lun);
 	} else if (is_lan_addr(addr)) {
 		rv = i_ipmi_req_lan(intf, addr, msgid, msg, smi_msg, recv_msg,
 				    source_lun, retries, retry_time_ms);
+		in_seq_table = true;
 	} else {
-	    /* Unknown address type. */
+		/* Unknown address type. */
 		ipmi_inc_stat(intf, sent_invalid_commands);
 		rv = -EINVAL;
 	}
 
-	if (rv) {
-out_err:
-		if (!supplied_smi)
-			ipmi_free_smi_msg(smi_msg);
-		if (!supplied_recv)
-			ipmi_free_recv_msg(recv_msg);
-	} else {
+	if (!rv) {
 		dev_dbg(intf->si_dev, "Send: %*ph\n",
 			smi_msg->data_size, smi_msg->data);
 
-		smi_send(intf, intf->handlers, smi_msg, priority);
+		rv = smi_send(intf, intf->handlers, smi_msg, priority);
+		if (rv && in_seq_table) {
+			/*
+			 * If it's in the sequence table, it will be
+			 * retried later, so ignore errors.
+			 */
+			rv = 0;
+			/* But we need to fix the timeout. */
+			intf_start_seq_timer(intf, smi_msg->msgid);
+			ipmi_free_smi_msg(smi_msg);
+			smi_msg = NULL;
+		}
 	}
+out_err:
 	if (!run_to_completion)
 		mutex_unlock(&intf->users_mutex);
 
+	if (rv) {
+		if (!supplied_smi)
+			ipmi_free_smi_msg(smi_msg);
+		if (!supplied_recv)
+			ipmi_free_recv_msg(recv_msg);
+	}
 	return rv;
 }
 
@@ -3965,12 +3993,12 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf,
 		dev_dbg(intf->si_dev, "Invalid command: %*ph\n",
 			msg->data_size, msg->data);
 
-		smi_send(intf, intf->handlers, msg, 0);
-		/*
-		 * We used the message, so return the value that
-		 * causes it to not be freed or queued.
-		 */
-		rv = -1;
+		if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR)
+			/*
+			 * We used the message, so return the value that
+			 * causes it to not be freed or queued.
+			 */
+			rv = -1;
 	} else if (!IS_ERR(recv_msg)) {
 		/* Extract the source address from the data. */
 		ipmb_addr = (struct ipmi_ipmb_addr *) &recv_msg->addr;
@@ -4044,12 +4072,12 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf,
 		msg->data[4] = IPMI_INVALID_CMD_COMPLETION_CODE;
 		msg->data_size = 5;
 
-		smi_send(intf, intf->handlers, msg, 0);
-		/*
-		 * We used the message, so return the value that
-		 * causes it to not be freed or queued.
-		 */
-		rv = -1;
+		if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR)
+			/*
+			 * We used the message, so return the value that
+			 * causes it to not be freed or queued.
+			 */
+			rv = -1;
 	} else if (!IS_ERR(recv_msg)) {
 		/* Extract the source address from the data. */
 		daddr = (struct ipmi_ipmb_direct_addr *)&recv_msg->addr;
@@ -4189,7 +4217,7 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf,
 				  struct ipmi_smi_msg *msg)
 {
 	struct cmd_rcvr          *rcvr;
-	int                      rv = 0;
+	int                      rv = 0; /* Free by default */
 	unsigned char            netfn;
 	unsigned char            cmd;
 	unsigned char            chan;
@@ -4242,12 +4270,12 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf,
 		dev_dbg(intf->si_dev, "Invalid command: %*ph\n",
 			msg->data_size, msg->data);
 
-		smi_send(intf, intf->handlers, msg, 0);
-		/*
-		 * We used the message, so return the value that
-		 * causes it to not be freed or queued.
-		 */
-		rv = -1;
+		if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR)
+			/*
+			 * We used the message, so return the value that
+			 * causes it to not be freed or queued.
+			 */
+			rv = -1;
 	} else if (!IS_ERR(recv_msg)) {
 		/* Extract the source address from the data. */
 		lan_addr = (struct ipmi_lan_addr *) &recv_msg->addr;
@@ -5056,7 +5084,12 @@ static void check_msg_timeout(struct ipmi_smi *intf, struct seq_table *ent,
 				ipmi_inc_stat(intf,
 					      retransmitted_ipmb_commands);
 
-			smi_send(intf, intf->handlers, smi_msg, 0);
+			/* If this fails we'll retry later or timeout. */
+			if (smi_send(intf, intf->handlers, smi_msg, 0) != IPMI_CC_NO_ERROR) {
+				/* But fix the timeout. */
+				intf_start_seq_timer(intf, smi_msg->msgid);
+				ipmi_free_smi_msg(smi_msg);
+			}
 		} else
 			ipmi_free_smi_msg(smi_msg);
 
Signature · 1 line
2.43.0
maintainer Reviewed-by on v1 2/3
#4 lore raw

Re: [PATCH 2/3] ipmi:msghandler: Handle error returns from the SMI sender

Quoted 11 lines · Corey Minyard

It used to be, until recently, that the sender operation on the low level interfaces would not fail. That's not the case any more with recent changes.

So check the return value from the sender operation, and propagate it back up from there and handle the errors in all places.

Fixes bc3a9d217755 ("ipmi:si: Gracefully handle if the BMC is non-functional")

The changes in smi_send() are what I would do there, but then I wouldn't know how to take the return value properly into account in some cases.

Quoted 192 lines
---
 drivers/char/ipmi/ipmi_msghandler.c | 97 +++++++++++++++++++----------
 1 file changed, 65 insertions(+), 32 deletions(-)
drivers/char/ipmi/ipmi_msghandler.c +65 -33
index a042b1596933..c209486bfee7 100644
@@ -1887,19 +1887,32 @@ static struct ipmi_smi_msg *smi_add_send_msg(struct ipmi_smi *intf,
        return smi_msg;
 }
 
-static void smi_send(struct ipmi_smi *intf,
+static int smi_send(struct ipmi_smi *intf,
                     const struct ipmi_smi_handlers *handlers,
                     struct ipmi_smi_msg *smi_msg, int priority)
 {
        int run_to_completion = READ_ONCE(intf->run_to_completion);
        unsigned long flags = 0;
+       int rv = 0;
 
        ipmi_lock_xmit_msgs(intf, run_to_completion, &flags);
        smi_msg = smi_add_send_msg(intf, smi_msg, priority);
        ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags);
 
-       if (smi_msg)
-               handlers->sender(intf->send_info, smi_msg);
+       if (smi_msg) {
+               rv = handlers->sender(intf->send_info, smi_msg);
+               if (rv) {
+                       ipmi_lock_xmit_msgs(intf, run_to_completion, &flags);
+                       intf->curr_msg = NULL;
+                       ipmi_unlock_xmit_msgs(intf, run_to_completion, &flags);
+                       /*
+                        * Something may have been added to the transmit
+                        * queue, so schedule a check for that.
+                        */
+                       queue_work(system_wq, &intf->smi_work);
+               }
+       }
+       return rv;
 }
 
 static bool is_maintenance_mode_cmd(struct kernel_ipmi_msg *msg)
@@ -2312,6 +2325,7 @@ static int i_ipmi_request(struct ipmi_user     *user,
        struct ipmi_recv_msg *recv_msg;
        int run_to_completion = READ_ONCE(intf->run_to_completion);
        int rv = 0;
+       bool in_seq_table = false;
 
        if (supplied_recv) {
                recv_msg = supplied_recv;
@@ -2365,33 +2379,47 @@ static int i_ipmi_request(struct ipmi_user     *user,
                rv = i_ipmi_req_ipmb(intf, addr, msgid, msg, smi_msg, recv_msg,
                                     source_address, source_lun,
                                     retries, retry_time_ms);
+               in_seq_table = true;
        } else if (is_ipmb_direct_addr(addr)) {
                rv = i_ipmi_req_ipmb_direct(intf, addr, msgid, msg, smi_msg,
                                            recv_msg, source_lun);
        } else if (is_lan_addr(addr)) {
                rv = i_ipmi_req_lan(intf, addr, msgid, msg, smi_msg, recv_msg,
                                    source_lun, retries, retry_time_ms);
+               in_seq_table = true;
        } else {
-           /* Unknown address type. */
+               /* Unknown address type. */
                ipmi_inc_stat(intf, sent_invalid_commands);
                rv = -EINVAL;
        }
 
-       if (rv) {
-out_err:
-               if (!supplied_smi)
-                       ipmi_free_smi_msg(smi_msg);
-               if (!supplied_recv)
-                       ipmi_free_recv_msg(recv_msg);
-       } else {
+       if (!rv) {
                dev_dbg(intf->si_dev, "Send: %*ph\n",
                        smi_msg->data_size, smi_msg->data);
 
-               smi_send(intf, intf->handlers, smi_msg, priority);
+               rv = smi_send(intf, intf->handlers, smi_msg, priority);
+               if (rv && in_seq_table) {
+                       /*
+                        * If it's in the sequence table, it will be
+                        * retried later, so ignore errors.
+                        */
+                       rv = 0;
+                       /* But we need to fix the timeout. */
+                       intf_start_seq_timer(intf, smi_msg->msgid);
+                       ipmi_free_smi_msg(smi_msg);
+                       smi_msg = NULL;
+               }
        }
+out_err:
        if (!run_to_completion)
                mutex_unlock(&intf->users_mutex);
 
+       if (rv) {
+               if (!supplied_smi)
+                       ipmi_free_smi_msg(smi_msg);
+               if (!supplied_recv)
+                       ipmi_free_recv_msg(recv_msg);
+       }
        return rv;
 }
 
@@ -3965,12 +3993,12 @@ static int handle_ipmb_get_msg_cmd(struct ipmi_smi *intf,
                dev_dbg(intf->si_dev, "Invalid command: %*ph\n",
                        msg->data_size, msg->data);
 
-               smi_send(intf, intf->handlers, msg, 0);
-               /*
-                * We used the message, so return the value that
-                * causes it to not be freed or queued.
-                */
-               rv = -1;
+               if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR)
+                       /*
+                        * We used the message, so return the value that
+                        * causes it to not be freed or queued.
+                        */
+                       rv = -1;
        } else if (!IS_ERR(recv_msg)) {
                /* Extract the source address from the data. */
                ipmb_addr = (struct ipmi_ipmb_addr *) &recv_msg->addr;
@@ -4044,12 +4072,12 @@ static int handle_ipmb_direct_rcv_cmd(struct ipmi_smi *intf,
                msg->data[4] = IPMI_INVALID_CMD_COMPLETION_CODE;
                msg->data_size = 5;
 
-               smi_send(intf, intf->handlers, msg, 0);
-               /*
-                * We used the message, so return the value that
-                * causes it to not be freed or queued.
-                */
-               rv = -1;
+               if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR)
+                       /*
+                        * We used the message, so return the value that
+                        * causes it to not be freed or queued.
+                        */
+                       rv = -1;
        } else if (!IS_ERR(recv_msg)) {
                /* Extract the source address from the data. */
                daddr = (struct ipmi_ipmb_direct_addr *)&recv_msg->addr;
@@ -4189,7 +4217,7 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf,
                                  struct ipmi_smi_msg *msg)
 {
        struct cmd_rcvr          *rcvr;
-       int                      rv = 0;
+       int                      rv = 0; /* Free by default */
        unsigned char            netfn;
        unsigned char            cmd;
        unsigned char            chan;
@@ -4242,12 +4270,12 @@ static int handle_lan_get_msg_cmd(struct ipmi_smi *intf,
                dev_dbg(intf->si_dev, "Invalid command: %*ph\n",
                        msg->data_size, msg->data);
 
-               smi_send(intf, intf->handlers, msg, 0);
-               /*
-                * We used the message, so return the value that
-                * causes it to not be freed or queued.
-                */
-               rv = -1;
+               if (smi_send(intf, intf->handlers, msg, 0) == IPMI_CC_NO_ERROR)
+                       /*
+                        * We used the message, so return the value that
+                        * causes it to not be freed or queued.
+                        */
+                       rv = -1;
        } else if (!IS_ERR(recv_msg)) {
                /* Extract the source address from the data. */
                lan_addr = (struct ipmi_lan_addr *) &recv_msg->addr;
@@ -5056,7 +5084,12 @@ static void check_msg_timeout(struct ipmi_smi *intf, struct seq_table *ent,
                                ipmi_inc_stat(intf,
                                              retransmitted_ipmb_commands);
 
-                       smi_send(intf, intf->handlers, smi_msg, 0);
+                       /* If this fails we'll retry later or timeout. */
+                       if (smi_send(intf, intf->handlers, smi_msg, 0) != IPMI_CC_NO_ERROR) {
+                               /* But fix the timeout. */
+                               intf_start_seq_timer(intf, smi_msg->msgid);
+                               ipmi_free_smi_msg(smi_msg);
+                       }
                } else
                        ipmi_free_smi_msg(smi_msg);
 
--

2.43.0

author maintainer
#5 lore raw

PATCH 3/3 ipmi:si: Fix check for a misbehaving BMC

In reply to Corey Minyard

There is a race on checking the state in the sender, it needs to be checked under a lock. But you also need a check to avoid issues with a misbehaving BMC for run to completion mode. So leave the check at the beginning for run to completion, and add a check under the lock to avoid the race.

Fixes bc3a9d217755 ("ipmi:si: Gracefully handle if the BMC is non-functional")
---
 drivers/char/ipmi/ipmi_si_intf.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)
drivers/char/ipmi/ipmi_si_intf.c +13 -11
index 3667033fcc51..6eda61664aaa 100644
@@ -924,9 +924,14 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg)
 {
 	struct smi_info   *smi_info = send_info;
 	unsigned long     flags;
+	int rv = IPMI_CC_NO_ERROR;
 
 	debug_timestamp(smi_info, "Enqueue");
 
+	/*
+	 * Check here for run to completion mode.  A check under lock is
+	 * later.
+	 */
 	if (smi_info->si_state == SI_HOSED)
 		return IPMI_BUS_ERR;
 
@@ -940,18 +945,15 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg)
 	}
 
 	spin_lock_irqsave(&smi_info->si_lock, flags);
-	/*
-	 * The following two lines don't need to be under the lock for
-	 * the lock's sake, but they do need SMP memory barriers to
-	 * avoid getting things out of order.  We are already claiming
-	 * the lock, anyway, so just do it under the lock to avoid the
-	 * ordering problem.
-	 */
-	BUG_ON(smi_info->waiting_msg);
-	smi_info->waiting_msg = msg;
-	check_start_timer_thread(smi_info);
+	if (smi_info->si_state == SI_HOSED) {
+		rv = IPMI_BUS_ERR;
+	} else {
+		BUG_ON(smi_info->waiting_msg);
+		smi_info->waiting_msg = msg;
+		check_start_timer_thread(smi_info);
+	}
 	spin_unlock_irqrestore(&smi_info->si_lock, flags);
-	return IPMI_CC_NO_ERROR;
+	return rv;
 }
 
 static void set_run_to_completion(void *send_info, bool i_run_to_completion)
Signature · 1 line
2.43.0
maintainer Reviewed-by on v1 3/3
#6 lore raw

Re: [PATCH 3/3] ipmi:si: Fix check for a misbehaving BMC

Quoted 10 lines · Corey Minyard

There is a race on checking the state in the sender, it needs to be checked under a lock. But you also need a check to avoid issues with a misbehaving BMC for run to completion mode. So leave the check at the beginning for run to completion, and add a check under the lock to avoid the race.

Fixes bc3a9d217755 ("ipmi:si: Gracefully handle if the BMC is non-functional")

That's what I would do in sender(), more or less, so

Quoted 53 lines
---
 drivers/char/ipmi/ipmi_si_intf.c | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)
drivers/char/ipmi/ipmi_si_intf.c +13 -12
index 3667033fcc51..6eda61664aaa 100644
@@ -924,9 +924,14 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg)
 {
        struct smi_info   *smi_info = send_info;
        unsigned long     flags;
+       int rv = IPMI_CC_NO_ERROR;
 
        debug_timestamp(smi_info, "Enqueue");
 
+       /*
+        * Check here for run to completion mode.  A check under lock is
+        * later.
+        */
        if (smi_info->si_state == SI_HOSED)
                return IPMI_BUS_ERR;
 
@@ -940,18 +945,15 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg)
        }
 
        spin_lock_irqsave(&smi_info->si_lock, flags);
-       /*
-        * The following two lines don't need to be under the lock for
-        * the lock's sake, but they do need SMP memory barriers to
-        * avoid getting things out of order.  We are already claiming
-        * the lock, anyway, so just do it under the lock to avoid the
-        * ordering problem.
-        */
-       BUG_ON(smi_info->waiting_msg);
-       smi_info->waiting_msg = msg;
-       check_start_timer_thread(smi_info);
+       if (smi_info->si_state == SI_HOSED) {
+               rv = IPMI_BUS_ERR;
+       } else {
+               BUG_ON(smi_info->waiting_msg);
+               smi_info->waiting_msg = msg;
+               check_start_timer_thread(smi_info);
+       }
        spin_unlock_irqrestore(&smi_info->si_lock, flags);
-       return IPMI_CC_NO_ERROR;
+       return rv;
 }
 
 static void set_run_to_completion(void *send_info, bool i_run_to_completion)
--

2.43.0