Skip to contentSkip to conversation

ceph: implement new-style ENOSPC handling in kcephfs

PATCH Superseded Jeff Layton 3 participants 8 messages Feb 7, 2017

Subsystems (2)

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v3 cover ceph: implement new-style ENOSPC handling in kcephfs

v3: track "abort_on_full" behavior with a new bool in osd request instead of a protocol flag. Remove some extraneous arguments from various functions. Don't export have_pool_full, call it from the abort_on_full callback instead.

v2: teach libcephfs how to hold on to requests until the right map epoch appears, instead of delaying cap handling in the cephfs layer.

This patchset is an updated version of the patch series originally done by John Spray and posted here:

    http://www.spinics.net/lists/ceph-devel/msg21257.html

This version incorporates changes based on Ilya's cursory review yesterday. It also cleans up the handling of requests that should be aborted on a full condition. The patch that exports have_pool_full has also been dropped since it's no longer needed.

Jeff Layton (4):
libceph: allow requests to return immediately on full conditions if
caller wishes
libceph: add an epoch_barrier field to struct ceph_osd_client
ceph: register map callback to handle ENOSPC conditions
ceph: handle epoch barriers in cap messages

John Spray (1):
libceph: add ceph_osdc_abort_on_full

 fs/ceph/addr.c                  |   4 ++
 fs/ceph/caps.c                  |  17 +++++--
 fs/ceph/file.c                  |   4 ++
 fs/ceph/mds_client.c            |  33 +++++++++++++
 fs/ceph/mds_client.h            |   7 ++-
 include/linux/ceph/osd_client.h |   7 +++
 net/ceph/osd_client.c           | 100 ++++++++++++++++++++++++++++++++++++----
 7 files changed, 158 insertions(+), 14 deletions(-)
Signature · 1 line
2.9.3
#1 lore raw

PATCH v3 1/5 libceph: allow requests to return immediately on full conditions if caller wishes

Right now, cephfs will cancel any in-flight OSD write operations when a new map comes in that shows the OSD or pool as full, but nothing prevents new requests from stalling out after that point.

If the caller knows that it will want an immediate error return instead of blocking on a full or at-quota error condition then allow it to set a flag to request that behavior. Cephfs write requests will always set that flag.

---
 fs/ceph/addr.c                  | 4 ++++
 fs/ceph/file.c                  | 4 ++++
 include/linux/ceph/osd_client.h | 1 +
 net/ceph/osd_client.c           | 6 ++++++
 4 files changed, 15 insertions(+)
fs/ceph/addr.c +4 -0
index 4547bbf80e4f..308787eeee2c 100644
@@ -1040,6 +1040,7 @@ static int ceph_writepages_start(struct address_space *mapping,
 
 		req->r_callback = writepages_finish;
 		req->r_inode = inode;
+		req->r_abort_on_full = true;
 
 		/* Format the osd request message and submit the write */
 		len = 0;
@@ -1689,6 +1690,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1732,6 +1734,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1893,6 +1896,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
 	err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false);
 
 	wr_req->r_mtime = ci->vfs_inode.i_mtime;
+	wr_req->r_abort_on_full = true;
 	err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false);
 
 	if (!err)
fs/ceph/file.c +4 -0
index a91a4f1fc837..987dcb9b566f 100644
@@ -714,6 +714,7 @@ static void ceph_aio_retry_work(struct work_struct *work)
 	req->r_callback = ceph_aio_complete_req;
 	req->r_inode = inode;
 	req->r_priv = aio_req;
+	req->r_abort_on_full = true;
 
 	ret = ceph_osdc_start_request(req->r_osdc, req, false);
 out:
@@ -912,6 +913,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,
 
 			osd_req_op_init(req, 1, CEPH_OSD_OP_STARTSYNC, 0);
 			req->r_mtime = mtime;
+			req->r_abort_on_full = true;
 		}
 
 		osd_req_op_extent_osd_data_pages(req, 0, pages, len, start,
@@ -1105,6 +1107,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 						false, true);
 
 		req->r_mtime = mtime;
+		req->r_abort_on_full = true;
 		ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 		if (!ret)
 			ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1557,6 +1560,7 @@ static int ceph_zero_partial_object(struct inode *inode,
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!ret) {
 		ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
include/linux/ceph/osd_client.h +1 -0
index 03a6653d329a..5da666cc5891 100644
@@ -171,6 +171,7 @@ struct ceph_osd_request {
 
 	int               r_result;
 	bool              r_got_reply;
+	bool		  r_abort_on_full; /* return ENOSPC when full */
 
 	struct ceph_osd_client *r_osdc;
 	struct kref       r_kref;
net/ceph/osd_client.c +6 -0
index 3a2417bb6ff0..f68bb42da240 100644
@@ -49,6 +49,7 @@ static void link_linger(struct ceph_osd *osd,
 			struct ceph_osd_linger_request *lreq);
 static void unlink_linger(struct ceph_osd *osd,
 			  struct ceph_osd_linger_request *lreq);
+static void complete_request(struct ceph_osd_request *req, int err);
 
 #if 1
 static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
@@ -1636,6 +1637,7 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	enum calc_target_result ct_res;
 	bool need_send = false;
 	bool promoted = false;
+	int ret = 0;
 
 	WARN_ON(req->r_tid || req->r_got_reply);
 	dout("%s req %p wrlocked %d\n", __func__, req, wrlocked);
@@ -1670,6 +1672,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 		pr_warn_ratelimited("FULL or reached pool quota\n");
 		req->r_t.paused = true;
 		maybe_request_map(osdc);
+		if (req->r_abort_on_full)
+			ret = -ENOSPC;
 	} else if (!osd_homeless(osd)) {
 		need_send = true;
 	} else {
@@ -1686,6 +1690,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	link_request(osd, req);
 	if (need_send)
 		send_request(req);
+	else if (ret)
+		complete_request(req, ret);
 	mutex_unlock(&osd->lock);
 
 	if (ct_res == CALC_TARGET_POOL_DNE)
Signature · 1 line
2.9.3
#2 lore raw

PATCH v3 2/5 libceph: add ceph_osdc_abort_on_full

In reply to Jeff Layton

From: John Spray <[email protected]>

When a Ceph volume hits capacity, a flag is set in the OSD map to indicate that, and a new map is sprayed around the cluster. When the cephfs client sees that, we want it to shut down any OSD writes that are in-progress with an -ENOSPC error as they'll just hang otherwise.

Add a routine that will see if there is an out-of-space condition in the cluster. It will then walk the tree and abort any request that has r_abort_on_full set with an ENOSPC error.

Also, add a callback to the osdc that gets called on map updates and a way for upper layers to register that callback.

[ jlayton: code style cleanup and adaptation to new osd msg handling ]

---
 include/linux/ceph/osd_client.h |  4 ++++
 net/ceph/osd_client.c           | 52 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
include/linux/ceph/osd_client.h +4 -0
index 5da666cc5891..1aaf4851f180 100644
@@ -21,6 +21,7 @@ struct ceph_osd_client;
 /*
  * completion callback for async writepages
  */
+typedef void (*ceph_osdc_map_callback_t)(struct ceph_osd_client *);
 typedef void (*ceph_osdc_callback_t)(struct ceph_osd_request *);
 typedef void (*ceph_osdc_unsafe_callback_t)(struct ceph_osd_request *, bool);
 
@@ -290,6 +291,8 @@ struct ceph_osd_client {
 	struct ceph_msgpool	msgpool_op_reply;
 
 	struct workqueue_struct	*notify_wq;
+
+	ceph_osdc_map_callback_t	map_cb;
 };
 
 static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
@@ -392,6 +395,7 @@ extern void ceph_osdc_put_request(struct ceph_osd_request *req);
 extern int ceph_osdc_start_request(struct ceph_osd_client *osdc,
 				   struct ceph_osd_request *req,
 				   bool nofail);
+extern u32 ceph_osdc_abort_on_full(struct ceph_osd_client *osdc);
 extern void ceph_osdc_cancel_request(struct ceph_osd_request *req);
 extern int ceph_osdc_wait_request(struct ceph_osd_client *osdc,
 				  struct ceph_osd_request *req);
net/ceph/osd_client.c +52 -0

76 lines not on this page · raw message

Signature · 1 line
2.9.3
#3 lore raw

Re: [PATCH v3 2/5] libceph: add ceph_osdc_abort_on_full

Quoted 77 lines · Jeff Layton

From: John Spray <[email protected]>

When a Ceph volume hits capacity, a flag is set in the OSD map to indicate that, and a new map is sprayed around the cluster. When the cephfs client sees that, we want it to shut down any OSD writes that are in-progress with an -ENOSPC error as they'll just hang otherwise.

Add a routine that will see if there is an out-of-space condition in the cluster. It will then walk the tree and abort any request that has r_abort_on_full set with an ENOSPC error.

Also, add a callback to the osdc that gets called on map updates and a way for upper layers to register that callback.

[ jlayton: code style cleanup and adaptation to new osd msg handling ]

---
 include/linux/ceph/osd_client.h |  4 ++++
 net/ceph/osd_client.c           | 52 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)
include/linux/ceph/osd_client.h +4 -0
index 5da666cc5891..1aaf4851f180 100644
@@ -21,6 +21,7 @@ struct ceph_osd_client;
 /*
  * completion callback for async writepages
  */
+typedef void (*ceph_osdc_map_callback_t)(struct ceph_osd_client *);
 typedef void (*ceph_osdc_callback_t)(struct ceph_osd_request *);
 typedef void (*ceph_osdc_unsafe_callback_t)(struct ceph_osd_request *, bool);
 
@@ -290,6 +291,8 @@ struct ceph_osd_client {
 	struct ceph_msgpool	msgpool_op_reply;
 
 	struct workqueue_struct	*notify_wq;
+
+	ceph_osdc_map_callback_t	map_cb;
 };
 
 static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
@@ -392,6 +395,7 @@ extern void ceph_osdc_put_request(struct ceph_osd_request *req);
 extern int ceph_osdc_start_request(struct ceph_osd_client *osdc,
 				   struct ceph_osd_request *req,
 				   bool nofail);
+extern u32 ceph_osdc_abort_on_full(struct ceph_osd_client *osdc);
 extern void ceph_osdc_cancel_request(struct ceph_osd_request *req);
 extern int ceph_osdc_wait_request(struct ceph_osd_client *osdc,
 				  struct ceph_osd_request *req);
net/ceph/osd_client.c +10 -0
index f68bb42da240..5a4f60000a73 100644
@@ -18,6 +18,7 @@
 #include <linux/ceph/decode.h>
 #include <linux/ceph/auth.h>
 #include <linux/ceph/pagelist.h>
+#include <linux/lockdep.h>
 
 #define OSD_OPREPLY_FRONT_LEN	512
 
@@ -1777,6 +1778,54 @@ static void complete_request(struct ceph_osd_request *req, int err)
 	ceph_osdc_put_request(req);
 }
 
+/*
+ * Drop all pending requests that have and complete
+ * them with the `r` as return code.
+ *
+ * Returns the highest OSD map epoch of a request that was
+ * cancelled, or 0 if none were cancelled.
+ */
+u32 ceph_osdc_abort_on_full(struct ceph_osd_client *osdc)
+{

net/ceph/osd_client.c static void complete_request(struct ceph_osd_request *req, int err) · line 1799

Quoted 10 lines
net/ceph/osd_client.c +10 -0
+	struct ceph_osd_request *req;
+	struct ceph_osd *osd;
+	struct rb_node *m, *n;
+	u32 latest_epoch = 0;
+	bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL);
+
+	lockdep_assert_held(&osdc->lock);
+
+	dout("enter complete_writes r=%d\n", r);
+

Oof. I sent out an earlier set instead of regenerating this. The above fails to compile since "r" no longer exists in this version. Fixed in my tree.

Quoted 33 lines
net/ceph/osd_client.c +29 -0
+	if (!osdmap_full && !have_pool_full(osdc))
+		goto out;
+
+	for (n = rb_first(&osdc->osds); n; n = rb_next(n)) {
+		osd = rb_entry(n, struct ceph_osd, o_node);
+		m = rb_first(&osd->o_requests);
+		mutex_lock(&osd->lock);
+		while (m) {
+			req = rb_entry(m, struct ceph_osd_request, r_node);
+			m = rb_next(m);
+
+			if (req->r_abort_on_full &&
+			    (osdmap_full || pool_full(osdc, req->r_t.base_oloc.pool))) {
+				u32 cur_epoch = le32_to_cpu(req->r_replay_version.epoch);
+
+				dout("%s: abort tid=%llu flags 0x%x\n", __func__, req->r_tid, req->r_flags);
+				complete_request(req, -ENOSPC);
+				if (cur_epoch > latest_epoch)
+					latest_epoch = cur_epoch;
+			}
+		}
+		mutex_unlock(&osd->lock);
+	}
+out:
+	dout("return abort_on_full latest_epoch=%u\n", latest_epoch);
+	return latest_epoch;
+}
+EXPORT_SYMBOL(ceph_osdc_abort_on_full);
+
 static void cancel_map_check(struct ceph_osd_request *req)
 {
 	struct ceph_osd_client *osdc = req->r_osdc;

net/ceph/osd_client.c void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg) · line 3345

Quoted 5 lines
net/ceph/osd_client.c +2 -0
@@ -3292,6 +3341,8 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg)
 
 	ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP,
 			  osdc->osdmap->epoch);
+	if (osdc->map_cb)
+		osdc->map_cb(osdc);

I'm now wondering though whether we should eliminate the map_cb pointer, and just call ceph_osdc_abort_on_full directly from ceph_osdc_handle_map. That would simplify things quite a bit, with the only downside being that when using something like rbd that doesn't set r_abort_on_full, and you get a map update that shows it being full that you'll end up walking the whole tree for nothing (since it doesn't set r_abort_on_full).

I can make that change, but I'll hold off on reposting with that until others have had a chance to review.

Quoted 11 lines
net/ceph/osd_client.c +1 -0
 	up_write(&osdc->lock);
 	wake_up_all(&osdc->client->auth_wq);
 	return;
@@ -4096,6 +4147,7 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
 	osdc->linger_requests = RB_ROOT;
 	osdc->map_checks = RB_ROOT;
 	osdc->linger_map_checks = RB_ROOT;
+	osdc->map_cb = NULL;
 	INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout);
 	INIT_DELAYED_WORK(&osdc->osds_timeout_work, handle_osds_timeout);
 
Signature · 1 line
Jeff Layton <[email protected]>
#4 lore raw

PATCH v3 3/5 libceph: add an epoch_barrier field to struct ceph_osd_client

In reply to Jeff Layton

Cephfs can get cap update requests that contain a new epoch barrier in them. When that happens we want to pause all OSD traffic until the right map epoch arrives. Add a way for the upper layers to set the epoch_barrier in ceph_osd_client and fix libceph to pause requests until it has the right map epoch.

Add an epoch_barrier field to ceph_osd_client that is protected by the osdc->lock rwsem. When the barrier is set, and the current OSD map epoch is below that, pause the request target when submitting the request or when revisiting it.

If we get a new map, compare the new epoch against the barrier before kicking requests and request another map if the map epoch is still lower than the one we want.

---
 include/linux/ceph/osd_client.h |  2 ++
 net/ceph/osd_client.c           | 42 +++++++++++++++++++++++++++++++++--------
 2 files changed, 36 insertions(+), 8 deletions(-)
include/linux/ceph/osd_client.h +2 -0
index 1aaf4851f180..d8114a0df4dd 100644
@@ -271,6 +271,7 @@ struct ceph_osd_client {
 	struct rb_root         osds;          /* osds */
 	struct list_head       osd_lru;       /* idle osds */
 	spinlock_t             osd_lru_lock;
+	u32		       epoch_barrier;
 	struct ceph_osd        homeless_osd;
 	atomic64_t             last_tid;      /* tid of last request */
 	u64                    last_linger_id;
@@ -311,6 +312,7 @@ extern void ceph_osdc_handle_reply(struct ceph_osd_client *osdc,
 				   struct ceph_msg *msg);
 extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc,
 				 struct ceph_msg *msg);
+void ceph_osdc_update_epoch_barrier(struct ceph_osd_client *osdc, u32 eb);
 
 extern void osd_req_op_init(struct ceph_osd_request *osd_req,
 			    unsigned int which, u16 opcode, u32 flags);
net/ceph/osd_client.c +34 -8

84 lines not on this page · raw message

Signature · 1 line
2.9.3
#5 lore raw

PATCH v3 4/5 ceph: register map callback to handle ENOSPC conditions

In reply to Jeff Layton
---
 fs/ceph/mds_client.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)
fs/ceph/mds_client.c +13 -0
index 93fc893e1930..fcf1e5313c16 100644
@@ -433,6 +433,18 @@ static int __verify_registered_session(struct ceph_mds_client *mdsc,
 	return 0;
 }
 
+static void handle_osd_map(struct ceph_osd_client *osdc)
+{
+	u32 cancelled_epoch;
+
+	lockdep_assert_held(&osdc->lock);
+
+	cancelled_epoch = ceph_osdc_abort_on_full(osdc);
+	if (cancelled_epoch)
+		osdc->epoch_barrier = max(cancelled_epoch + 1,
+					  osdc->epoch_barrier);
+}
+
 /*
  * create+register a new session for given mds.
  * called under mdsc->mutex.
@@ -3485,6 +3497,7 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)
 
 	ceph_caps_init(mdsc);
 	ceph_adjust_min_caps(mdsc, fsc->min_caps);
+	fsc->client->osdc.map_cb = handle_osd_map;
 
 	init_rwsem(&mdsc->pool_perm_rwsem);
 	mdsc->pool_perm_tree = RB_ROOT;
Signature · 1 line
2.9.3
#6 lore raw

PATCH v3 5/5 ceph: handle epoch barriers in cap messages

In reply to Jeff Layton

Have the client store and update the osdc epoch_barrier when a cap message comes in with one. This allows clients to inform servers that their released caps may not be used until a particular OSD map epoch.

---
 fs/ceph/caps.c       | 17 +++++++++++++----
 fs/ceph/mds_client.c | 20 ++++++++++++++++++++
 fs/ceph/mds_client.h |  7 +++++--
 3 files changed, 38 insertions(+), 6 deletions(-)
fs/ceph/caps.c +13 -4
index 3c2dfd72e5b2..d91d3f32a5b6 100644
@@ -1015,6 +1015,7 @@ static int send_cap_msg(struct cap_msg_args *arg)
 	void *p;
 	size_t extra_len;
 	struct timespec zerotime = {0};
+	struct ceph_osd_client *osdc = &arg->session->s_mdsc->fsc->client->osdc;
 
 	dout("send_cap_msg %s %llx %llx caps %s wanted %s dirty %s"
 	     " seq %u/%u tid %llu/%llu mseq %u follows %lld size %llu/%llu"
@@ -1077,7 +1078,9 @@ static int send_cap_msg(struct cap_msg_args *arg)
 	/* inline data size */
 	ceph_encode_32(&p, 0);
 	/* osd_epoch_barrier (version 5) */
-	ceph_encode_32(&p, 0);
+	down_read(&osdc->lock);
+	ceph_encode_32(&p, osdc->epoch_barrier);
+	up_read(&osdc->lock);
 	/* oldest_flush_tid (version 6) */
 	ceph_encode_64(&p, arg->oldest_flush_tid);
 
@@ -3635,13 +3638,19 @@ void ceph_handle_caps(struct ceph_mds_session *session,
 		p += inline_len;
 	}
 
+	if (le16_to_cpu(msg->hdr.version) >= 5) {
+		struct ceph_osd_client	*osdc = &mdsc->fsc->client->osdc;
+		u32			epoch_barrier;
+
+		ceph_decode_32_safe(&p, end, epoch_barrier, bad);
+		ceph_osdc_update_epoch_barrier(osdc, epoch_barrier);
+	}
+
 	if (le16_to_cpu(msg->hdr.version) >= 8) {
 		u64 flush_tid;
 		u32 caller_uid, caller_gid;
-		u32 osd_epoch_barrier;
 		u32 pool_ns_len;
-		/* version >= 5 */
-		ceph_decode_32_safe(&p, end, osd_epoch_barrier, bad);
+
 		/* version >= 6 */
 		ceph_decode_64_safe(&p, end, flush_tid, bad);
 		/* version >= 7 */
fs/ceph/mds_client.c +20 -0
index fcf1e5313c16..9125ec83a9f4 100644
@@ -1562,9 +1562,15 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 	struct ceph_msg *msg = NULL;
 	struct ceph_mds_cap_release *head;
 	struct ceph_mds_cap_item *item;
+	struct ceph_osd_client *osdc = &mdsc->fsc->client->osdc;
 	struct ceph_cap *cap;
 	LIST_HEAD(tmp_list);
 	int num_cap_releases;
+	__le32	barrier, *cap_barrier;
+
+	down_read(&osdc->lock);
+	barrier = cpu_to_le32(osdc->epoch_barrier);
+	up_read(&osdc->lock);
 
 	spin_lock(&session->s_cap_lock);
 again:
@@ -1582,7 +1588,11 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 			head = msg->front.iov_base;
 			head->num = cpu_to_le32(0);
 			msg->front.iov_len = sizeof(*head);
+
+			msg->hdr.version = cpu_to_le16(2);
+			msg->hdr.compat_version = cpu_to_le16(1);
 		}
+
 		cap = list_first_entry(&tmp_list, struct ceph_cap,
 					session_caps);
 		list_del(&cap->session_caps);
@@ -1600,6 +1610,11 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 		ceph_put_cap(mdsc, cap);
 
 		if (le32_to_cpu(head->num) == CEPH_CAPS_PER_RELEASE) {
+			// Append cap_barrier field
+			cap_barrier = msg->front.iov_base + msg->front.iov_len;
+			*cap_barrier = barrier;
+			msg->front.iov_len += sizeof(*cap_barrier);
+
 			msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
 			dout("send_cap_releases mds%d %p\n", session->s_mds, msg);
 			ceph_con_send(&session->s_con, msg);
@@ -1615,6 +1630,11 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 	spin_unlock(&session->s_cap_lock);
 
 	if (msg) {
+		// Append cap_barrier field
+		cap_barrier = msg->front.iov_base + msg->front.iov_len;
+		*cap_barrier = barrier;
+		msg->front.iov_len += sizeof(*cap_barrier);
+
 		msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
 		dout("send_cap_releases mds%d %p\n", session->s_mds, msg);
 		ceph_con_send(&session->s_con, msg);
fs/ceph/mds_client.h +5 -2
index ac0475a2daa7..517684c7c5f0 100644
@@ -104,10 +104,13 @@ struct ceph_mds_reply_info_parsed {
 
 /*
  * cap releases are batched and sent to the MDS en masse.
+ *
+ * Account for per-message overhead of mds_cap_release header
+ * and __le32 for osd epoch barrier trailing field.
  */
-#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE -			\
+#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE - sizeof(u32) -		\
 				sizeof(struct ceph_mds_cap_release)) /	\
-			       sizeof(struct ceph_mds_cap_item))
+			        sizeof(struct ceph_mds_cap_item))
 
 
 /*
Signature · 1 line
2.9.3
#7 lore raw
In reply to Jeff Layton
Quoted 31 lines · Jeff Layton

v3: track "abort_on_full" behavior with a new bool in osd request instead of a protocol flag. Remove some extraneous arguments from various functions. Don't export have_pool_full, call it from the abort_on_full callback instead.

v2: teach libcephfs how to hold on to requests until the right map epoch appears, instead of delaying cap handling in the cephfs layer.

This patchset is an updated version of the patch series originally done by John Spray and posted here:

   http://www.spinics.net/lists/ceph-devel/msg21257.html

This version incorporates changes based on Ilya's cursory review yesterday. It also cleans up the handling of requests that should be aborted on a full condition. The patch that exports have_pool_full has also been dropped since it's no longer needed.

Jeff Layton (4):
libceph: allow requests to return immediately on full conditions if
caller wishes
libceph: add an epoch_barrier field to struct ceph_osd_client
ceph: register map callback to handle ENOSPC conditions
ceph: handle epoch barriers in cap messages

John Spray (1):
libceph: add ceph_osdc_abort_on_full

Quoted 8 lines
fs/ceph/addr.c                  |   4 ++
fs/ceph/caps.c                  |  17 +++++--
fs/ceph/file.c                  |   4 ++
fs/ceph/mds_client.c            |  33 +++++++++++++
fs/ceph/mds_client.h            |   7 ++-
include/linux/ceph/osd_client.h |   7 +++
net/ceph/osd_client.c           | 100 ++++++++++++++++++++++++++++++++++++----
7 files changed, 158 insertions(+), 14 deletions(-)

The cephfs part looks good for me

Regards
Yan, Zheng

Quoted 4 lines
Signature · 2 lines
-- 
2.9.3