Skip to contentSkip to conversation

Lustre fixes

PATCH New Oleg Drokin 1 participant 3 messages Nov 17, 2016 Staging

lore
Share & tools
lore
#0 lore raw

cover Lustre fixes

With multiple metadata RPCs code in now, these two fixes become important.

Please consider.

Niu Yawei (2):
staging/lustre/ptlrpc: track unreplied requests staging/lustre/ptlrpc: update imp_known_replied_xid on resend-replay

 .../staging/lustre/lustre/include/lustre_import.h  |   5 +
 drivers/staging/lustre/lustre/include/lustre_net.h |   3 +
 drivers/staging/lustre/lustre/obdclass/genops.c    |   2 +
 drivers/staging/lustre/lustre/ptlrpc/client.c      | 112 +++++++++++++++------
 drivers/staging/lustre/lustre/ptlrpc/import.c      |  34 +++++++
 drivers/staging/lustre/lustre/ptlrpc/niobuf.c      |  29 +++++-
 .../staging/lustre/lustre/ptlrpc/ptlrpc_internal.h |  24 +++++
 drivers/staging/lustre/lustre/ptlrpc/recover.c     |  14 +++
 8 files changed, 187 insertions(+), 36 deletions(-)
Signature · 1 line
2.7.4
#1 lore raw

PATCH 1/2 staging/lustre/ptlrpc: track unreplied requests

From: Niu Yawei <[email protected]>

The request xid was used to make sure the ost object timestamps being updated by the out of order setattr/punch/write requests properly. However, this mechanism is broken by the multiple rcvd slot feature, where we deferred the xid assignment from request packing to request sending.

This patch moved back the xid assignment to request packing, and the manner of finding lowest unreplied xid is changed from scan sending & delay list to scan a unreplied requests list.

This patch also skipped packing the known replied XID in connect and disconnect request, so that we can make sure the known replied XID is increased only on both server & client side.

Reviewed-on: http://review.whamcloud.com/16759
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-5951

---
 .../staging/lustre/lustre/include/lustre_import.h  |   5 +
 drivers/staging/lustre/lustre/include/lustre_net.h |   3 +
 drivers/staging/lustre/lustre/obdclass/genops.c    |   2 +
 drivers/staging/lustre/lustre/ptlrpc/client.c      | 112 +++++++++++++++------
 drivers/staging/lustre/lustre/ptlrpc/import.c      |  34 +++++++
 drivers/staging/lustre/lustre/ptlrpc/niobuf.c      |  29 +++++-
 .../staging/lustre/lustre/ptlrpc/ptlrpc_internal.h |  24 +++++
 drivers/staging/lustre/lustre/ptlrpc/recover.c     |  12 +++
 8 files changed, 185 insertions(+), 36 deletions(-)
drivers/staging/lustre/lustre/include/lustre_import.h +5 -0
index 5461ba3..4499c69 100644
@@ -185,6 +185,11 @@ struct obd_import {
 	struct list_head	       *imp_replay_cursor;
 	/** @} */
 
+	/** List of not replied requests */
+	struct list_head	imp_unreplied_list;
+	/** Known maximal replied XID */
+	__u64			imp_known_replied_xid;
+
 	/** obd device for this import */
 	struct obd_device	*imp_obd;
 
drivers/staging/lustre/lustre/include/lustre_net.h +3 -0
index d2cbec3..2be135d 100644
@@ -596,6 +596,8 @@ struct ptlrpc_cli_req {
 	union ptlrpc_async_args		 cr_async_args;
 	/** Opaq data for replay and commit callbacks. */
 	void				*cr_cb_data;
+	/** Link to the imp->imp_unreplied_list */
+	struct list_head		 cr_unreplied_list;
 	/**
 	 * Commit callback, called when request is committed and about to be
 	 * freed.
@@ -635,6 +637,7 @@ struct ptlrpc_cli_req {
 #define rq_interpret_reply	rq_cli.cr_reply_interp
 #define rq_async_args		rq_cli.cr_async_args
 #define rq_cb_data		rq_cli.cr_cb_data
+#define rq_unreplied_list	rq_cli.cr_unreplied_list
 #define rq_commit_cb		rq_cli.cr_commit_cb
 #define rq_replay_cb		rq_cli.cr_replay_cb
 
drivers/staging/lustre/lustre/obdclass/genops.c +2 -0
index 438d619..fa0d38d 100644
@@ -907,6 +907,8 @@ struct obd_import *class_new_import(struct obd_device *obd)
 	INIT_LIST_HEAD(&imp->imp_sending_list);
 	INIT_LIST_HEAD(&imp->imp_delayed_list);
 	INIT_LIST_HEAD(&imp->imp_committed_list);
+	INIT_LIST_HEAD(&imp->imp_unreplied_list);
+	imp->imp_known_replied_xid = 0;
 	imp->imp_replay_cursor = &imp->imp_committed_list;
 	spin_lock_init(&imp->imp_lock);
 	imp->imp_last_success_conn = 0;
drivers/staging/lustre/lustre/ptlrpc/client.c +81 -31

174 lines not on this page · raw message

drivers/staging/lustre/lustre/ptlrpc/import.c +34 -0
index babb80d..66f5b49 100644
@@ -904,6 +904,39 @@ static int ptlrpc_connect_set_flags(struct obd_import *imp,
 }
 
 /**
+ * Add all replay requests back to unreplied list before start replay,
+ * so that we can make sure the known replied XID is always increased
+ * only even if when replaying requests.
+ */
+static void ptlrpc_prepare_replay(struct obd_import *imp)
+{
+	struct ptlrpc_request *req;
+
+	if (imp->imp_state != LUSTRE_IMP_REPLAY ||
+	    imp->imp_resend_replay)
+		return;
+
+	/*
+	 * If the server was restart during repaly, the requests may
+	 * have been added to the unreplied list in former replay.
+	 */
+	spin_lock(&imp->imp_lock);
+
+	list_for_each_entry(req, &imp->imp_committed_list, rq_replay_list) {
+		if (list_empty(&req->rq_unreplied_list))
+			ptlrpc_add_unreplied(req);
+	}
+
+	list_for_each_entry(req, &imp->imp_replay_list, rq_replay_list) {
+		if (list_empty(&req->rq_unreplied_list))
+			ptlrpc_add_unreplied(req);
+	}
+
+	imp->imp_known_replied_xid = ptlrpc_known_replied_xid(imp);
+	spin_unlock(&imp->imp_lock);
+}
+
+/**
  * interpret_reply callback for connect RPCs.
  * Looks into returned status of connect operation and decides
  * what to do with the import - i.e enter recovery, promote it to
@@ -1154,6 +1187,7 @@ static int ptlrpc_connect_interpret(const struct lu_env *env,
 	}
 
 finish:
+	ptlrpc_prepare_replay(imp);
 	rc = ptlrpc_import_recovery_state_machine(imp);
 	if (rc == -ENOTCONN) {
 		CDEBUG(D_HA, "evicted/aborted by %s@%s during recovery; invalidating and reconnecting\n",
drivers/staging/lustre/lustre/ptlrpc/niobuf.c +24 -5
index 5810565..da1209e 100644
@@ -509,20 +509,39 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply)
 	lustre_msg_set_conn_cnt(request->rq_reqmsg, imp->imp_conn_cnt);
 	lustre_msghdr_set_flags(request->rq_reqmsg, imp->imp_msghdr_flags);
 
-	if (request->rq_nr_resend) {
+	/*
+	 * If it's the first time to resend the request for EINPROGRESS,
+	 * we need to allocate a new XID (see after_reply()), it's different
+	 * from the resend for reply timeout.
+	 */
+	if (request->rq_nr_resend && list_empty(&request->rq_unreplied_list)) {
+		__u64 min_xid = 0;
 		/*
 		 * resend for EINPROGRESS, allocate new xid to avoid reply
 		 * reconstruction
 		 */
-		request->rq_xid = ptlrpc_next_xid();
-		DEBUG_REQ(D_RPCTRACE, request, "Allocating new xid for resend on EINPROGRESS");
-	}
+		spin_lock(&imp->imp_lock);
+		ptlrpc_assign_next_xid_nolock(request);
+		request->rq_mbits = request->rq_xid;
+		min_xid = ptlrpc_known_replied_xid(imp);
+		spin_unlock(&imp->imp_lock);
 
-	if (request->rq_bulk) {
+		lustre_msg_set_last_xid(request->rq_reqmsg, min_xid);
+		DEBUG_REQ(D_RPCTRACE, request, "Allocating new xid for resend on EINPROGRESS");
+	} else if (request->rq_bulk) {
 		ptlrpc_set_bulk_mbits(request);
 		lustre_msg_set_mbits(request->rq_reqmsg, request->rq_mbits);
 	}
 
+	if (list_empty(&request->rq_unreplied_list) ||
+	    request->rq_xid <= imp->imp_known_replied_xid) {
+		DEBUG_REQ(D_ERROR, request,
+			  "xid: %llu, replied: %llu, list_empty:%d\n",
+			  request->rq_xid, imp->imp_known_replied_xid,
+			  list_empty(&request->rq_unreplied_list));
+		LBUG();
+	}
+
 	/**
 	 * For enabled AT all request should have AT_SUPPORT in the
 	 * FULL import state when OBD_CONNECT_AT is set
drivers/staging/lustre/lustre/ptlrpc/ptlrpc_internal.h +24 -0
index f056c82..e0f859c 100644
@@ -71,6 +71,9 @@ int ptlrpc_expired_set(void *data);
 int ptlrpc_set_next_timeout(struct ptlrpc_request_set *);
 void ptlrpc_resend_req(struct ptlrpc_request *request);
 void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req);
+void ptlrpc_assign_next_xid_nolock(struct ptlrpc_request *req);
+__u64 ptlrpc_known_replied_xid(struct obd_import *imp);
+void ptlrpc_add_unreplied(struct ptlrpc_request *req);
 
 /* events.c */
 int ptlrpc_init_portals(void);
@@ -324,6 +327,7 @@ static inline void ptlrpc_cli_req_init(struct ptlrpc_request *req)
 
 	INIT_LIST_HEAD(&cr->cr_set_chain);
 	INIT_LIST_HEAD(&cr->cr_ctx_chain);
+	INIT_LIST_HEAD(&cr->cr_unreplied_list);
 	init_waitqueue_head(&cr->cr_reply_waitq);
 	init_waitqueue_head(&cr->cr_set_waitq);
 }
@@ -340,4 +344,24 @@ static inline void ptlrpc_srv_req_init(struct ptlrpc_request *req)
 	INIT_LIST_HEAD(&sr->sr_hist_list);
 }
 
+static inline bool ptlrpc_req_is_connect(struct ptlrpc_request *req)
+{
+	if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_CONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == OST_CONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == MGS_CONNECT)
+		return true;
+	else
+		return false;
+}
+
+static inline bool ptlrpc_req_is_disconnect(struct ptlrpc_request *req)
+{
+	if (lustre_msg_get_opc(req->rq_reqmsg) == MDS_DISCONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == OST_DISCONNECT ||
+	    lustre_msg_get_opc(req->rq_reqmsg) == MGS_DISCONNECT)
+		return true;
+	else
+		return false;
+}
+
 #endif /* PTLRPC_INTERNAL_H */
drivers/staging/lustre/lustre/ptlrpc/recover.c +12 -0
index 9144cd8..344aedd 100644
@@ -157,10 +157,22 @@ int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
 		lustre_msg_add_flags(req->rq_reqmsg, MSG_RESENT);
 
 	spin_lock(&imp->imp_lock);
+	/* The resend replay request may have been removed from the
+	 * unreplied list.
+	 */
+	if (req && imp->imp_resend_replay &&
+	    list_empty(&req->rq_unreplied_list))
+		ptlrpc_add_unreplied(req);
+
 	imp->imp_resend_replay = 0;
 	spin_unlock(&imp->imp_lock);
 
 	if (req) {
+		/* The request should have been added back in unreplied list
+		 * by ptlrpc_prepare_replay().
+		 */
+		LASSERT(!list_empty(&req->rq_unreplied_list));
+
 		rc = ptlrpc_replay_req(req);
 		if (rc) {
 			CERROR("recovery replay error %d for req %llu\n",
Signature · 1 line
2.7.4
#2 lore raw

PATCH 2/2 staging/lustre/ptlrpc: update imp_known_replied_xid on resend-replay

In reply to Oleg Drokin

From: Niu Yawei <[email protected]>

The imp_known_replied_xid should be updated when try to resend an already replied replay request, because the xid of this replay request could be less than current imp_known_replied_xid.

Reviewed-on: http://review.whamcloud.com/22776
Intel-bug-id: https://jira.hpdd.intel.com/browse/LU-8645

---
 drivers/staging/lustre/lustre/ptlrpc/recover.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
drivers/staging/lustre/lustre/ptlrpc/recover.c +3 -1
index 344aedd..c004490 100644
@@ -161,8 +161,10 @@ int ptlrpc_replay_next(struct obd_import *imp, int *inflight)
 	 * unreplied list.
 	 */
 	if (req && imp->imp_resend_replay &&
-	    list_empty(&req->rq_unreplied_list))
+	    list_empty(&req->rq_unreplied_list)) {
 		ptlrpc_add_unreplied(req);
+		imp->imp_known_replied_xid = ptlrpc_known_replied_xid(imp);
+	}
 
 	imp->imp_resend_replay = 0;
 	spin_unlock(&imp->imp_lock);
Signature · 1 line
2.7.4