Skip to contentSkip to conversation

drbd: retry the IO when connection lost

PATCH New Rui Xu 1 participant 1 message Aug 18, 2022 DRBD driver

lore
Share & tools
lore

Rui Xu

author
#0 lore raw

PATCH v2 1/1 drbd: retry the IO when connection lost

Commit 33600a4632f2 has introduced some problems, for example, the problem mentioned in commit 300bfb7ba.

My test also meet a problem introduced by commit 33600a4632f2. I have three nodes running with drbd9.1(node-1, node-2 and node-3), node-1 is primary and other nodes are secondary. Both quorum and quorum-minimum-redundancy are set to 2.

Cut down the connection between node-1 and node-3 first, then cut down the connection between node-1 and node-2. When the connection between node-1 and node-3 comes back, the node-3 whose disk status isn't UPTODATE prevents the quorum-minimum-redundancy coming back, the resync process--send bitmap , in turn, is blocked by the suspended IO. Here is a dead lock.

We can solve those problems by close the blocked IO with POSTPONE, so the IO will neither return back to upper caller nor block the resync process. As soon as the resync with peers completes, the quorum will come back and the IO will be retried.

And it's simpler than the current mechanism.

---
 drbd/drbd_main.c     | 59 ++++++++++++++++++-------------
 drbd/drbd_receiver.c | 19 +++-------
 drbd/drbd_req.c      | 82 ++++++--------------------------------------
 drbd/drbd_req.h      |  4 ---
 drbd/drbd_state.c    | 48 --------------------------
 5 files changed, 50 insertions(+), 162 deletions(-)
drbd/drbd_main.c +35 -24
index 24baa860..dbee64d4 100644
@@ -2926,43 +2926,32 @@ void drbd_req_destroy_lock(struct kref *kref)
 	read_unlock_irq(&resource->state_rwlock);
 }
 
+struct drbd_request_dummy {
+	struct list_head retry_list;
+	struct bio *master_bio;
+	struct drbd_device *device;
+	unsigned long start_jif;
+	ktime_t start_kt;
+};
+
 static void do_retry(struct work_struct *ws)
 {
 	struct retry_worker *retry = container_of(ws, struct retry_worker, worker);
 	LIST_HEAD(writes);
-	struct drbd_request *req, *tmp;
+	struct drbd_request_dummy *req, *tmp;
 
 	spin_lock_irq(&retry->lock);
 	list_splice_init(&retry->writes, &writes);
 	spin_unlock_irq(&retry->lock);
 
-	list_for_each_entry_safe(req, tmp, &writes, list) {
+	list_for_each_entry_safe(req, tmp, &writes, retry_list) {
 		struct drbd_device *device = req->device;
 		struct bio *bio = req->master_bio;
 		unsigned long start_jif = req->start_jif;
-		bool expected;
 		ktime_get_accounting_assign(ktime_t start_kt, req->start_kt);
 
-
-		/* No locking when accessing local_rq_state & net_rq_state, since
-		 * this request is not active at the moment. */
-		expected =
-			expect(device, atomic_read(&req->completion_ref) == 0) &&
-			expect(device, req->local_rq_state & RQ_POSTPONED) &&
-			expect(device, (req->local_rq_state & RQ_LOCAL_PENDING) == 0 ||
-			       (req->local_rq_state & RQ_LOCAL_ABORTED) != 0);
-
-		if (!expected)
-			drbd_err(device, "req=%p completion_ref=%d rq_state=%x\n",
-				req, atomic_read(&req->completion_ref),
-				req->local_rq_state);
-
-		/* We still need to put one kref associated with the
-		 * "completion_ref" going zero in the code path that queued it
-		 * here.  The request object may still be referenced by a
-		 * frozen local req->private_bio, in case we force-detached.
-		 */
-		kref_put(&req->kref, drbd_req_destroy_lock);
+		list_del_init(&req->retry_list);
+		kfree(req);
 
 		/* A single suspended or otherwise blocking device may stall
 		 * all others as well. This code path is to recover from a
@@ -2979,12 +2968,34 @@ static void do_retry(struct work_struct *ws)
 	}
 }
 
+void drbd_dummy_request_prepare(struct drbd_request *req,
+										struct drbd_request_dummy *dummy)
+{
+	INIT_LIST_HEAD(&dummy->retry_list);
+	dummy->device = req->device;
+	dummy->master_bio = req->master_bio;
+	dummy->start_jif = req->start_jif;
+	ktime_get_accounting_assign(dummy->start_kt, req->start_kt);
+}
+
 /* called via drbd_req_put_completion_ref() */
 void drbd_restart_request(struct drbd_request *req)
 {
 	unsigned long flags;
+	struct drbd_request_dummy *req_dummy;
+	int alloc_times = 0;
+
+retry_alloc:
+	req_dummy = kzalloc(sizeof(struct drbd_request_dummy), GFP_ATOMIC);
+	if (!req_dummy) {
+		alloc_times++;
+		BUG_ON(alloc_times >= 100);
+		goto retry_alloc;
+	}
+	drbd_dummy_request_prepare(req, req_dummy);
+
 	spin_lock_irqsave(&retry.lock, flags);
-	list_move_tail(&req->list, &retry.writes);
+	list_move_tail(&req_dummy->retry_list, &retry.writes);
 	spin_unlock_irqrestore(&retry.lock, flags);
 
 	/* Drop the extra reference that would otherwise
drbd/drbd_receiver.c +5 -14
index f077f2b1..716fa555 100644
@@ -3852,13 +3852,6 @@ static enum sync_strategy drbd_uuid_compare(struct drbd_peer_device *peer_device
 			return SYNC_TARGET_USE_BITMAP;
 		}
 
-		if (connection->agreed_pro_version >= 120) {
-			*rule = RULE_RECONNECTED;
-			if (peer_device->uuid_flags & UUID_FLAG_RECONNECT &&
-			    local_uuid_flags & UUID_FLAG_RECONNECT)
-				return NO_SYNC;
-		}
-
 		if (connection->agreed_pro_version >= 121 &&
 		    (wire_protocol == DRBD_PROT_A || wire_protocol == DRBD_PROT_B)) {
 			*rule = RULE_CRASHED_PRIMARY;
@@ -3886,12 +3879,11 @@ static enum sync_strategy drbd_uuid_compare(struct drbd_peer_device *peer_device
 				SYNC_SOURCE_IF_BOTH_FAILED :
 				SYNC_TARGET_IF_BOTH_FAILED;
 
-		if (connection->agreed_pro_version < 120) {
-			*rule = RULE_RECONNECTED;
-			if (peer_device->uuid_flags & UUID_FLAG_RECONNECT &&
-			    local_uuid_flags & UUID_FLAG_RECONNECT)
+		*rule = RULE_RECONNECTED;
+		/* This is a safety net for the following two clauses */
+		if (peer_device->uuid_flags & UUID_FLAG_RECONNECT &&
+			local_uuid_flags & UUID_FLAG_RECONNECT)
 				return NO_SYNC;
-		}
 
 		/* Peer crashed as primary, I survived, resync from me */
 		if (peer_device->uuid_flags & UUID_FLAG_CRASHED_PRIMARY &&
@@ -7897,8 +7889,7 @@ static void conn_disconnect(struct drbd_connection *connection)
 	/* Apply these changes after peer_device_disconnected() because that
 	 * may cause the loss of other connections to be detected, which can
 	 * change the suspended state. */
-	tl_walk(connection, &connection->req_not_net_done,
-			resource->cached_susp ? CONNECTION_LOST_WHILE_SUSPENDED : CONNECTION_LOST);
+	tl_walk(connection, NULL, CONNECTION_LOST);
 
 	i = drbd_free_peer_reqs(connection, &connection->read_ee, true);
 	if (i)
drbd/drbd_req.c +10 -72

103 lines not on this page · raw message

drbd/drbd_req.h +0 -4
index 3cc364a2..e7776332 100644
@@ -86,7 +86,6 @@ enum drbd_req_event {
 	HANDED_OVER_TO_NETWORK,
 	OOS_HANDED_TO_NETWORK,
 	CONNECTION_LOST,
-	CONNECTION_LOST_WHILE_SUSPENDED,
 	RECV_ACKED_BY_PEER,
 	WRITE_ACKED_BY_PEER,
 	WRITE_ACKED_BY_PEER_AND_SIS, /* and set_in_sync */
@@ -102,9 +101,6 @@ enum drbd_req_event {
 	DISCARD_COMPLETED_WITH_ERROR,
 
 	ABORT_DISK_IO,
-	RESEND,
-	CANCEL_SUSPENDED_IO,
-	COMPLETION_RESUMED,
 	NOTHING,
 };
 
drbd/drbd_state.c +0 -48

78 lines not on this page · raw message

Signature · 1 line
2.25.1