drbd: change the variable ov_left to atomic
PATCH 1/1 drbd: change the variable ov_left to atomic
The problem was, that when a node(node-1) is a verify source of another node(node-2), it will send P_OV_REQUEST and node-2 will process it. In normal scenarios, node-2 will reply P_OV_REPLY to node-1, but if an application IO is onging on corresponding sector on node-2, it will reply P_RS_CANCEL to node-1.
So w_e_end_ov_reply and got_NegRSDReply can be happened concurrently, it may lead to the self subtraction of the variable ov_left happened at the same time, finally, the variable ov_left may not be reduced to zero, which causes verify unable to end.
--- drbd/drbd_debugfs.c | 4 ++-- drbd/drbd_int.h | 2 +- drbd/drbd_nl.c | 4 ++-- drbd/drbd_receiver.c | 9 +++++---- drbd/drbd_sender.c | 12 ++++++------ drbd/drbd_state.c | 8 ++++---- 6 files changed, 20 insertions(+), 19 deletions(-)
drbd/drbd_debugfs.c +2 -2
@@ -1481,7 +1481,7 @@ static void drbd_get_syncer_progress(struct drbd_peer_device *pd,* for the percentage, we don't care. */ if (repl_state ==L_VERIFY_S || repl_state ==L_VERIFY_T ) - *bits_left = pd->ov_left; + *bits_left =atomic64_read (&pd->ov_left);else *bits_left =drbd_bm_total_weight (pd) - pd->rs_failed;/* >> 10 to prevent overflow,
@@ -1617,7 +1617,7 @@ static void drbd_syncer_progress(struct drbd_peer_device *pd, struct seq_file *sunsigned long long stop_sector =0 ;if (repl_state ==L_VERIFY_S || repl_state ==L_VERIFY_T ) { - bit_pos = bm_bits - pd->ov_left; + bit_pos = bm_bits - (unsigned long )atomic64_read (&pd->ov_left);if (verify_can_do_stop_sector (pd)) stop_sector = pd->ov_stop_sector; }else
drbd/drbd_int.h +1 -1
@@ -1176,7 +1176,7 @@ struct drbd_peer_device {* on the lower level device when we last looked. */ int rs_in_flight;/* resync sectors in flight (to proxy, in proxy and from proxy) */ ktime_t rs_last_mk_req_kt; -unsigned long ov_left;/* in bits */ +atomic64_t ov_left;/* in bits */ unsigned long ov_skipped;/* in bits */ u64 rs_start_uuid;
drbd/drbd_nl.c +2 -2
@@ -5355,11 +5355,11 @@ static void peer_device_to_statistics(struct peer_device_statistics *s, s->peer_dev_out_of_sync =BM_BIT_TO_SECT (drbd_bm_total_weight (pd));if (is_verify_state (pd,NOW )) { - rs_left =BM_BIT_TO_SECT (pd->ov_left); + rs_left =BM_BIT_TO_SECT (atomic64_read (&pd->ov_left)); s->peer_dev_ov_start_sector = pd->ov_start_sector; s->peer_dev_ov_stop_sector = pd->ov_stop_sector; s->peer_dev_ov_position = pd->ov_position; - s->peer_dev_ov_left =BM_BIT_TO_SECT (pd->ov_left); + s->peer_dev_ov_left =BM_BIT_TO_SECT (atomic64_read (&pd->ov_left)); s->peer_dev_ov_skipped =BM_BIT_TO_SECT (pd->ov_skipped); }else if (is_sync_state (pd,NOW )) { rs_left = s->peer_dev_out_of_sync -BM_BIT_TO_SECT (pd->rs_failed);
drbd/drbd_receiver.c +5 -4
@@ -3141,7 +3141,7 @@ bool drbd_rs_c_min_rate_throttle(struct drbd_peer_device *peer_device) i = (peer_device->rs_last_mark +DRBD_SYNC_MARKS -1 ) %DRBD_SYNC_MARKS ;if (peer_device->repl_state[NOW ] ==L_VERIFY_S || peer_device->repl_state[NOW ] ==L_VERIFY_T ) - rs_left = peer_device->ov_left; + rs_left =atomic64_read (&peer_device->ov_left);else rs_left =drbd_bm_total_weight (peer_device) - peer_device->rs_failed;
@@ -3352,13 +3352,14 @@ static int receive_DataRequest(struct drbd_connection *connection, struct packet connection->agreed_pro_version >= 90) {unsigned long now = jiffies;int i; +unsigned long ov_left =drbd_bm_bits (device) -BM_SECT_TO_BIT (sector); +atomic64_set (&peer_device->ov_left, ov_left); peer_device->ov_start_sector = sector; - peer_device->ov_left =drbd_bm_bits (device) -BM_SECT_TO_BIT (sector); peer_device->ov_skipped =0 ; - peer_device->rs_total = peer_device->ov_left; + peer_device->rs_total =atomic64_read (&peer_device->ov_left); peer_device->rs_last_writeout = now;for (i =0 ; i <DRBD_SYNC_MARKS ; i++) { - peer_device->rs_mark_left[i] = peer_device->ov_left; + peer_device->rs_mark_left[i] =atomic64_read (&peer_device->ov_left); peer_device->rs_mark_time[i] = now; } drbd_info(device, "Online Verify start sector: %llu\n",
drbd/drbd_sender.c +6 -6
@@ -1108,7 +1108,7 @@ int drbd_resync_finished(struct drbd_peer_device *peer_device, db = peer_device->rs_total;/* adjust for verify start and stop sectors, respective reached position */ if (repl_state[NOW ] ==L_VERIFY_S || repl_state[NOW ] ==L_VERIFY_T ) - db -= peer_device->ov_left; + db -=atomic64_read (&peer_device->ov_left); dbdt =Bit2KB (db/dt); peer_device->rs_paused /=HZ ;
@@ -1250,7 +1250,7 @@ out_unlock: out:/* reset start sector, if we reached end of device */ -if (verify_done && peer_device->ov_left ==0 ) +if (verify_done &&atomic64_read (&peer_device->ov_left) ==0 ) peer_device->ov_start_sector =0 ;drbd_md_sync_if_dirty (device);
@@ -1556,13 +1556,13 @@ void verify_progress(struct drbd_peer_device *peer_device,verify_can_do_stop_sector (peer_device) && (sector + (size>>9 )) >= peer_device->ov_stop_sector; - --peer_device->ov_left; +atomic64_dec (&peer_device->ov_left);/* let's advance progress step marks only for every other megabyte */ -if ((peer_device->ov_left &0x1ff ) ==0 ) -drbd_advance_rs_marks (peer_device, peer_device->ov_left); +if ((atomic64_read (&peer_device->ov_left) &0x1ff ) ==0 ) +drbd_advance_rs_marks (peer_device, (unsigned long )atomic64_read (&peer_device->ov_left)); -if (peer_device->ov_left ==0 || stop_sector_reached) +if (atomic64_read (&peer_device->ov_left) ==0 || stop_sector_reached)drbd_peer_device_post_work (peer_device,RS_DONE ); }
drbd/drbd_state.c +4 -4
@@ -2161,7 +2161,7 @@ static void set_ov_position(struct drbd_peer_device *peer_device, peer_device->rs_total -= bit; peer_device->ov_position = peer_device->ov_start_sector; } - peer_device->ov_left = peer_device->rs_total; +atomic64_set (&peer_device->ov_left, peer_device->rs_total); peer_device->ov_skipped =0 ; }
@@ -2347,8 +2347,8 @@ static void finish_state_change(struct drbd_resource *resource, struct completioif ((repl_state[OLD ] ==L_VERIFY_S || repl_state[OLD ] ==L_VERIFY_T ) && repl_state[NEW ] <=L_ESTABLISHED ) { peer_device->ov_start_sector = -BM_BIT_TO_SECT (drbd_bm_bits (device) - peer_device->ov_left); -if (peer_device->ov_left) +BM_BIT_TO_SECT (drbd_bm_bits (device) -atomic64_read (&peer_device->ov_left)); +if (atomic64_read (&peer_device->ov_left))drbd_info (peer_device,"Online Verify reached sector %llu\n" , (unsigned long long )peer_device->ov_start_sector); }
@@ -2391,7 +2391,7 @@ static void finish_state_change(struct drbd_resource *resource, struct completio peer_device->ov_last_skipped_start =0 ; peer_device->rs_last_writeout = now;for (i =0 ; i <DRBD_SYNC_MARKS ; i++) { - peer_device->rs_mark_left[i] = peer_device->ov_left; + peer_device->rs_mark_left[i] =atomic64_read (&peer_device->ov_left); peer_device->rs_mark_time[i] = now; }
Signature · 1 line
2.25.1
Hi Xu,
Your analysis of the problem seems correct to me. Some details of the fix could be improved.
In verify_progress() there are now 3 separate atomic operations. It would be better to use atomic64_dec_return() and then use the returned value instead of calling atomic64_read() separately. This could have minor effects on the behavior.
Similarly, in receive_DataRequest() there is no need to read ov_left again after setting it. This should have no effect on the behavior, but leads to cleaner code.
Also the repeated reads in finish_state_change() could be avoided for cleaner code. Take care of the read after set_ov_position(), since that changes the value of ov_left. You could use peer_device->rs_total instead of peer_device->ov_left there, since that is what it is set to in set_ov_position().
Best regards,
Joel
Hi Joel,
Thanks for your suggestion and i've sent a new patch for it.
Thanks,
Xu
From: Joel Colledge <[email protected]>
Date: 2022-05-03 16:37:28
To: Rui Xu <[email protected]>
Cc: Philipp Reisner <[email protected]>,[email protected]
Subject: Re: [Drbd-dev] [PATCH] drbd: change the variable ov_left to atomic>Hi Xu,
Quoted 21 lines
Your analysis of the problem seems correct to me. Some details of the fix could be improved.
In
verify_progress()there are now 3 separate atomic operations. It would be better to useatomic64_dec_return()and then use the returned value instead of callingatomic64_read()separately. This could have minor effects on the behavior.Similarly, in
receive_DataRequest()there is no need to readov_leftagain after setting it. This should have no effect on the behavior, but leads to cleaner code.Also the repeated reads in
finish_state_change()could be avoided for cleaner code. Take care of the read afterset_ov_position(), since that changes the value ofov_left. You could usepeer_device->rs_totalinstead ofpeer_device->ov_leftthere, since that is what it is set to inset_ov_position().Best regards,
Joel