bonding: check port and aggregator when select
Subsystems (3)
PATCH v2 1/1 bonding: check port and aggregator when select
When the network service is repeatedly restarted in 802.3ad, there is a low
probability that oops occurs.
Test commands:systemctl restart network
1.crash: __enable_port():port->slave is NULL
crash> bt
PID: 2508692 TASK: ffff803e72a7ec80 CPU: 29 COMMAND: "kworker/u192:0"
#0 [ffff0000b13cb5c0] machine_kexec at ffff0000800a3964
#1 [ffff0000b13cb620] __crash_kexec at ffff0000801bf054
#2 [ffff0000b13cb7b0] panic at ffff0000800f350c
#3 [ffff0000b13cb890] die at ffff00008008f940
#4 [ffff0000b13cb8e0] die_kernel_fault at ffff0000800abbc0 #5 [ffff0000b13cb910] __do_kernel_fault at ffff0000800ab8c4 #6 [ffff0000b13cb940] do_page_fault at ffff000080a3eb44
#7 [ffff0000b13cba40] do_translation_fault at ffff000080a3f064 #8 [ffff0000b13cba70] do_mem_abort at ffff0000800812cc
#9 [ffff0000b13cbc70] el1_ia at ffff00008008320c
PC: ffff000000e2fcd0 [ad_agg_selection_logic+328]
LR: ffff000000e2fcb0 [ad_agg_selection_logic+296]
SP: ffff0000b13cbc80 PSTATE: 40c00009
X29: ffff0000b13cbc90 X28: ffff803e71c31438 X27: ffff000000e41eb8
X26: ffff0000b13cbd97 X25: ffff000000e4c0b8 X24: ffff803e71c31400
X23: ffff000081229000 X22: 0000000000000000 X21: ffff803e71c31400
X20: ffff0000b13cbcf0 X19: ffff803f4c772ac0 X18: ffffffffffffffff
X17: 0000000000000000 X16: ffff0000808acc70 X15: ffff000081229708
X14: 7361772074756220 X13: 353335353620726f X12: 7461676572676761
X11: 206f742064657461 X10: 0000000000000000 X9: ffff00008122baf0
X8: 00000000000e97a8 X7: ffff000081408080 X6: ffff805f7fa27448
X5: ffff805f7fa27448 X4: 0000000000000000 X3: 0000000000000006
X2: 0000000000000004 X1: 0000000000000000 X0: ffff803e739bea38crash> struct port ffff803e739bea38
struct port {
actor_port_number = 2,
actor_port_priority = 255,
actor_system = {
mac_addr_value = "\254\215\064\037\016y"
},
actor_system_priority = 65535,
actor_port_aggregator_identifier = 2094,
ntt = false,
actor_admin_port_key = 0,
actor_oper_port_key = 0,
actor_admin_port_state = 5 '\005',
actor_oper_port_state = 3 '\003',
partner_admin = {
system = {
mac_addr_value = "\000\000\000\000\000"
},
system_priority = 65535,
key = 1,
port_number = 1,
port_priority = 255,
port_state = 1
},
partner_oper = {
system = {
mac_addr_value = "\254\263\265\367b!"
},
system_priority = 32768,
key = 1089,
port_number = 8,
port_priority = 32768,
port_state = 61 },
is_enabled = false,
sm_vars = 304,
sm_rx_state = AD_RX_PORT_DISABLED,
sm_rx_timer_counter = 26,
sm_periodic_state = AD_NO_PERIODIC,
sm_periodic_timer_counter = 0,
sm_mux_state = AD_MUX_COLLECTING_DISTRIBUTING,
sm_mux_timer_counter = 0,
sm_tx_state = AD_TX_DUMMY,
sm_tx_timer_counter = 1,
sm_churn_actor_timer_counter = 0,
sm_churn_partner_timer_counter = 0,
churn_actor_count = 0,
churn_partner_count = 0,
lacpdu_send_success_count = 10,
lacpdu_send_failure_count = 0,
lacpdu_recv_count = 150,
marker_info_recv_count = 0,
marker_resp_recv_count = 0,
marker_unkown_recv_count = 0,
sm_churn_actor_state = AD_NO_CHURN,
sm_churn_partner_state = AD_NO_CHURN,
slave = 0x0,
aggregator = 0xffff803e739bea00,
next_port_in_aggregator = 0x0,
transaction_id = 0,
-- MORE --
2.I also have another call stack, same as in another person's post:
https://lore.kernel.org/netdev/[email protected]/
--- drivers/net/bonding/bond_3ad.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
drivers/net/bonding/bond_3ad.c +2 -1
@@ -183,7 +183,7 @@ static inline void __enable_port(struct port *port) {struct slave *slave = port->slave; -if ((slave->link ==BOND_LINK_UP ) &&bond_slave_is_up (slave)) +if (slave && (slave->link ==BOND_LINK_UP ) &&bond_slave_is_up (slave))bond_set_slave_active_flags (slave,BOND_SLAVE_NOTIFY_LATER ); }
@@ -1516,6 +1516,7 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) port->actor_port_number, port->aggregator->aggregator_identifier); }else { + port->aggregator = &(SLAVE_AD_INFO (slave)->aggregator);slave_err (bond->dev, port->slave->dev,"Port %d did not find a suitable aggregator\n" , port->actor_port_number);
Signature · 1 line
2.19.1
Jakub Kicinski
maintainerQuoted 3 lines · On Thu, 28 Jan 2021 16:20:34 +0800 Aichun Li wrote:
When the network service is repeatedly restarted in 802.3ad, there is a low
probability that oops occurs.
Test commands:systemctl restart network
Jay, others, any thoughts?
Quoted 5 lines · Aichun Li
When the network service is repeatedly restarted in 802.3ad, there is a low probability that oops occurs.
Test commands:systemctl restart network1.crash: __enable_port():port->slave is NULL
[...]
Quoted 1 line
PC: ffff000000e2fcd0 [ad_agg_selection_logic+328]
[...]
Quoted 2 lines
2.I also have another call stack, same as in another person's post:
https://lore.kernel.org/netdev/[email protected]/
What hardware platform are you using here?
moyufeng <[email protected]> appears to be using the same
platform, and I've not had any success so far with the provided script to reproduce the issue. I'm using an x86_64 system, however, so I wonder if perhaps this platform needs a barrier somewhere that x86 does not, or there's something different in the timing of the device driver close logic.
Quoted 10 lines
Signed-off-by Aichun Li <[email protected]>---
drivers/net/bonding/bond_3ad.c| 3 ++-1 file changed, 2 insertions(+), 1 deletion(-)drivers/net/bonding/bond_3ad.c +0
-0
drivers/net/bonding/bond_3ad.c static inline void __enable_port(struct port *port) · line 188
Quoted 7 lines
drivers/net/bonding/bond_3ad.c +1
-1@@ -183,7 +183,7 @@ static inline void __enable_port(struct port *port) {struct slave *slave = port->slave; -if ((slave->link ==BOND_LINK_UP ) &&bond_slave_is_up (slave)) +if (slave && (slave->link ==BOND_LINK_UP ) &&bond_slave_is_up (slave))bond_set_slave_active_flags (slave,BOND_SLAVE_NOTIFY_LATER ); }
This change seems like a band aid to cover the real problem.
The caller of __enable_port is ad_agg_selection_logic, and it shouldn't be possible for port->slave to be NULL when assigned to an aggregator.
drivers/net/bonding/bond_3ad.c static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) · line 1522
Quoted 8 lines
drivers/net/bonding/bond_3ad.c +1
-0@@ -1516,6 +1516,7 @@ static void ad_port_selection_logic(struct port *port, bool *update_slave_arr) port->actor_port_number, port->aggregator->aggregator_identifier); }else { + port->aggregator = &(SLAVE_AD_INFO (slave)->aggregator);slave_err (bond->dev, port->slave->dev,"Port %d did not find a suitable aggregator\n" , port->actor_port_number);
This change isn't correct; it's assigning the port to a more or
less random aggregator. This would eliminate the panic, but isn't doing the right thing. At this point in the code, the selection logic has failed to find an aggregator that matches, and also failed to find a free aggregator.
I do need to fix up the failure handling here when it hits the
"did not find a suitable agg" case; the code here is simply wrong, and has been wrong since the beginning. I'll hack the driver to induce this situation rather than reproducing whatever problem is making it unable to find a suitable aggregator.
-J
---
-Jay Vosburgh, [email protected]