Skip to contentSkip to conversation

ceph: implement new-style ENOSPC handling in kcephfs

PATCH Superseded Jeff Layton 3 participants 16 messages Feb 9, 2017 – Feb 10, 2017

Subsystems (2)

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v4 cover ceph: implement new-style ENOSPC handling in kcephfs

v4: eliminate map_cb and just call ceph_osdc_abort_on_full directly Revert earlier patch flagging individual pages with error on writeback failure. Add mechanism to force synchronous writes when writes start failing, and reallowing async writes when they succeed.

v3: track "abort_on_full" behavior with a new bool in osd request instead of a protocol flag. Remove some extraneous arguments from various functions. Don't export have_pool_full, call it from the abort_on_full callback instead.

v2: teach libcephfs how to hold on to requests until the right map epoch appears, instead of delaying cap handling in the cephfs layer.

Ok, with this set, I think we have proper -ENOSPC handling for all different write types (direct, sync, async buffered, etc...). -ENOSPC conditions.

First I trimmed down the original set to have less tendrils in ceph.ko when we hit a full condition. Most of the machinery for that is now in libceph.ko, and cephfs just sets the r_abort_on_full flag on the appropriate calls.

For buffered, synchronous writes, it also reverts a patch that flags individual pages with the PG_error bit on writeback failure. I don't think we want that for the reasons described in the patch.

Finally, this also adds a new mechanism that forces synchronous writes on an inode when write requests start failing.

This patchset is an updated version of the patch series originally done by John Spray and posted here:

    http://www.spinics.net/lists/ceph-devel/msg21257.html

This version incorporates changes based on Ilya's cursory review yesterday. It also cleans up the handling of requests that should be aborted on a full condition. The patch that exports have_pool_full has also been dropped since it's no longer needed.

Jeff Layton (6):
libceph: allow requests to return immediately on full conditions if
caller wishes
libceph: abort already submitted but abortable requests when map or
pool goes full
libceph: add an epoch_barrier field to struct ceph_osd_client
ceph: handle epoch barriers in cap messages
Revert "ceph: SetPageError() for writeback pages if writepages fails"
ceph: when seeing write errors on an inode, switch to sync writes

 fs/ceph/addr.c                  | 14 ++++--
 fs/ceph/caps.c                  | 17 ++++++--
 fs/ceph/file.c                  | 50 ++++++++++++++++------
 fs/ceph/mds_client.c            | 20 +++++++++
 fs/ceph/mds_client.h            |  7 ++-
 fs/ceph/super.h                 | 26 +++++++++++
 include/linux/ceph/osd_client.h |  3 ++
 net/ceph/osd_client.c           | 95 +++++++++++++++++++++++++++++++++++++----
 8 files changed, 201 insertions(+), 31 deletions(-)
Signature · 1 line
2.9.3
#1 lore raw

PATCH v4 1/6 libceph: allow requests to return immediately on full conditions if caller wishes

Usually, when the osd map is flagged as full or the pool is at quota, write requests just hang. This is not what we want for cephfs, where it would be better to simply report -ENOSPC back to userland instead of stalling.

If the caller knows that it will want an immediate error return instead of blocking on a full or at-quota error condition then allow it to set a flag to request that behavior. Cephfs write requests will always set that flag.

A later patch will deal with requests that were submitted before the new map showing the full condition came in.

---
 fs/ceph/addr.c                  | 4 ++++
 fs/ceph/file.c                  | 4 ++++
 include/linux/ceph/osd_client.h | 1 +
 net/ceph/osd_client.c           | 6 ++++++
 4 files changed, 15 insertions(+)
fs/ceph/addr.c +4 -0
index 4547bbf80e4f..308787eeee2c 100644
@@ -1040,6 +1040,7 @@ static int ceph_writepages_start(struct address_space *mapping,
 
 		req->r_callback = writepages_finish;
 		req->r_inode = inode;
+		req->r_abort_on_full = true;
 
 		/* Format the osd request message and submit the write */
 		len = 0;
@@ -1689,6 +1690,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1732,6 +1734,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1893,6 +1896,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
 	err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false);
 
 	wr_req->r_mtime = ci->vfs_inode.i_mtime;
+	wr_req->r_abort_on_full = true;
 	err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false);
 
 	if (!err)
fs/ceph/file.c +4 -0
index a91a4f1fc837..987dcb9b566f 100644
@@ -714,6 +714,7 @@ static void ceph_aio_retry_work(struct work_struct *work)
 	req->r_callback = ceph_aio_complete_req;
 	req->r_inode = inode;
 	req->r_priv = aio_req;
+	req->r_abort_on_full = true;
 
 	ret = ceph_osdc_start_request(req->r_osdc, req, false);
 out:
@@ -912,6 +913,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,
 
 			osd_req_op_init(req, 1, CEPH_OSD_OP_STARTSYNC, 0);
 			req->r_mtime = mtime;
+			req->r_abort_on_full = true;
 		}
 
 		osd_req_op_extent_osd_data_pages(req, 0, pages, len, start,
@@ -1105,6 +1107,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 						false, true);
 
 		req->r_mtime = mtime;
+		req->r_abort_on_full = true;
 		ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 		if (!ret)
 			ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1557,6 +1560,7 @@ static int ceph_zero_partial_object(struct inode *inode,
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!ret) {
 		ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
include/linux/ceph/osd_client.h +1 -0
index 03a6653d329a..5da666cc5891 100644
@@ -171,6 +171,7 @@ struct ceph_osd_request {
 
 	int               r_result;
 	bool              r_got_reply;
+	bool		  r_abort_on_full; /* return ENOSPC when full */
 
 	struct ceph_osd_client *r_osdc;
 	struct kref       r_kref;
net/ceph/osd_client.c +6 -0
index 3a2417bb6ff0..f68bb42da240 100644
@@ -49,6 +49,7 @@ static void link_linger(struct ceph_osd *osd,
 			struct ceph_osd_linger_request *lreq);
 static void unlink_linger(struct ceph_osd *osd,
 			  struct ceph_osd_linger_request *lreq);
+static void complete_request(struct ceph_osd_request *req, int err);
 
 #if 1
 static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
@@ -1636,6 +1637,7 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	enum calc_target_result ct_res;
 	bool need_send = false;
 	bool promoted = false;
+	int ret = 0;
 
 	WARN_ON(req->r_tid || req->r_got_reply);
 	dout("%s req %p wrlocked %d\n", __func__, req, wrlocked);
@@ -1670,6 +1672,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 		pr_warn_ratelimited("FULL or reached pool quota\n");
 		req->r_t.paused = true;
 		maybe_request_map(osdc);
+		if (req->r_abort_on_full)
+			ret = -ENOSPC;
 	} else if (!osd_homeless(osd)) {
 		need_send = true;
 	} else {
@@ -1686,6 +1690,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	link_request(osd, req);
 	if (need_send)
 		send_request(req);
+	else if (ret)
+		complete_request(req, ret);
 	mutex_unlock(&osd->lock);
 
 	if (ct_res == CALC_TARGET_POOL_DNE)
Signature · 1 line
2.9.3
#2 lore raw

Re: [PATCH v4 1/6] libceph: allow requests to return immediately on full conditions if caller wishes

Quoted 53 lines · Jeff Layton

Usually, when the osd map is flagged as full or the pool is at quota, write requests just hang. This is not what we want for cephfs, where it would be better to simply report -ENOSPC back to userland instead of stalling.

If the caller knows that it will want an immediate error return instead of blocking on a full or at-quota error condition then allow it to set a flag to request that behavior. Cephfs write requests will always set that flag.

A later patch will deal with requests that were submitted before the new map showing the full condition came in.

---

fs/ceph/addr.c                  | 4 ++++
fs/ceph/file.c                  | 4 ++++
include/linux/ceph/osd_client.h | 1 +
net/ceph/osd_client.c           | 6 ++++++
4 files changed, 15 insertions(+)
fs/ceph/addr.c +3 -0
index 4547bbf80e4f..308787eeee2c 100644
@@ -1040,6 +1040,7 @@ static int ceph_writepages_start(struct address_space *mapping,
 
 		req->r_callback = writepages_finish;
 		req->r_inode = inode;
+		req->r_abort_on_full = true;
 
 		/* Format the osd request message and submit the write */
 		len = 0;
@@ -1689,6 +1690,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1732,6 +1734,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);

fs/ceph/addr.c static int __ceph_pool_perm_get(struct ceph_inode_info *ci, · line 1902

Quoted 7 lines
fs/ceph/addr.c +1 -0
@@ -1893,6 +1896,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
 	err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false);
 
 	wr_req->r_mtime = ci->vfs_inode.i_mtime;
+	wr_req->r_abort_on_full = true;
 	err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false);
 
 	if (!err)

do you ignore writepage_nounlock() case intentionally?

Regards
Yan, Zheng

Quoted 89 lines
fs/ceph/file.c +1 -0
index a91a4f1fc837..987dcb9b566f 100644
@@ -714,6 +714,7 @@ static void ceph_aio_retry_work(struct work_struct *work)
 	req->r_callback = ceph_aio_complete_req;
 	req->r_inode = inode;
 	req->r_priv = aio_req;
+	req->r_abort_on_full = true;
 
 	ret = ceph_osdc_start_request(req->r_osdc, req, false);

out:

fs/ceph/file.c +3 -0
@@ -912,6 +913,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,
 
 			osd_req_op_init(req, 1, CEPH_OSD_OP_STARTSYNC, 0);
 			req->r_mtime = mtime;
+			req->r_abort_on_full = true;
 		}
 
 		osd_req_op_extent_osd_data_pages(req, 0, pages, len, start,
@@ -1105,6 +1107,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 						false, true);
 
 		req->r_mtime = mtime;
+		req->r_abort_on_full = true;
 		ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 		if (!ret)
 			ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1557,6 +1560,7 @@ static int ceph_zero_partial_object(struct inode *inode,
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!ret) {
 		ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
include/linux/ceph/osd_client.h +1 -0
index 03a6653d329a..5da666cc5891 100644
@@ -171,6 +171,7 @@ struct ceph_osd_request {
 
 	int               r_result;
 	bool              r_got_reply;
+	bool		  r_abort_on_full; /* return ENOSPC when full */
 
 	struct ceph_osd_client *r_osdc;
 	struct kref       r_kref;
net/ceph/osd_client.c +6 -0
index 3a2417bb6ff0..f68bb42da240 100644
@@ -49,6 +49,7 @@ static void link_linger(struct ceph_osd *osd,
 			struct ceph_osd_linger_request *lreq);
 static void unlink_linger(struct ceph_osd *osd,
 			  struct ceph_osd_linger_request *lreq);
+static void complete_request(struct ceph_osd_request *req, int err);
 
 #if 1
 static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
@@ -1636,6 +1637,7 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	enum calc_target_result ct_res;
 	bool need_send = false;
 	bool promoted = false;
+	int ret = 0;
 
 	WARN_ON(req->r_tid || req->r_got_reply);
 	dout("%s req %p wrlocked %d\n", __func__, req, wrlocked);
@@ -1670,6 +1672,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 		pr_warn_ratelimited("FULL or reached pool quota\n");
 		req->r_t.paused = true;
 		maybe_request_map(osdc);
+		if (req->r_abort_on_full)
+			ret = -ENOSPC;
 	} else if (!osd_homeless(osd)) {
 		need_send = true;
 	} else {
@@ -1686,6 +1690,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	link_request(osd, req);
 	if (need_send)
 		send_request(req);
+	else if (ret)
+		complete_request(req, ret);
 	mutex_unlock(&osd->lock);
 
 	if (ct_res == CALC_TARGET_POOL_DNE)
Signature · 1 line
2.9.3
#3 lore raw

Re: [PATCH v4 1/6] libceph: allow requests to return immediately on full conditions if caller wishes

Quoted 54 lines · Zheng
Quoted 54 lines · Jeff Layton

Usually, when the osd map is flagged as full or the pool is at quota, write requests just hang. This is not what we want for cephfs, where it would be better to simply report -ENOSPC back to userland instead of stalling.

If the caller knows that it will want an immediate error return instead of blocking on a full or at-quota error condition then allow it to set a flag to request that behavior. Cephfs write requests will always set that flag.

A later patch will deal with requests that were submitted before the new map showing the full condition came in.

---

fs/ceph/addr.c                  | 4 ++++
fs/ceph/file.c                  | 4 ++++
include/linux/ceph/osd_client.h | 1 +
net/ceph/osd_client.c           | 6 ++++++
4 files changed, 15 insertions(+)
fs/ceph/addr.c +3 -0
index 4547bbf80e4f..308787eeee2c 100644
@@ -1040,6 +1040,7 @@ static int ceph_writepages_start(struct address_space *mapping,
 
 		req->r_callback = writepages_finish;
 		req->r_inode = inode;
+		req->r_abort_on_full = true;
 
 		/* Format the osd request message and submit the write */
 		len = 0;
@@ -1689,6 +1690,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1732,6 +1734,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!err)
 		err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1893,6 +1896,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
 	err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false);
 
Quoted 10 lines

fs/ceph/addr.c static int __ceph_pool_perm_get(struct ceph_inode_info *ci, · line 1902

Quoted 5 lines
fs/ceph/addr.c +1 -0
 	wr_req->r_mtime = ci->vfs_inode.i_mtime;
+	wr_req->r_abort_on_full = true;
 	err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false);
 
 	if (!err)

do you ignore writepage_nounlock() case intentionally?

No. Hmmm...writepage_nounlock calls ceph_osdc_writepages, and it's the only caller so I guess we'll need to set this there. Maybe we should just lift ceph_osdc_writepages into ceph.ko since there are no callers in libceph?

Quoted 91 lines
Quoted 89 lines
fs/ceph/file.c +1 -0
index a91a4f1fc837..987dcb9b566f 100644
@@ -714,6 +714,7 @@ static void ceph_aio_retry_work(struct work_struct *work)
 	req->r_callback = ceph_aio_complete_req;
 	req->r_inode = inode;
 	req->r_priv = aio_req;
+	req->r_abort_on_full = true;
 
 	ret = ceph_osdc_start_request(req->r_osdc, req, false);

out:

fs/ceph/file.c +3 -0
@@ -912,6 +913,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,
 
 			osd_req_op_init(req, 1, CEPH_OSD_OP_STARTSYNC, 0);
 			req->r_mtime = mtime;
+			req->r_abort_on_full = true;
 		}
 
 		osd_req_op_extent_osd_data_pages(req, 0, pages, len, start,
@@ -1105,6 +1107,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 						false, true);
 
 		req->r_mtime = mtime;
+		req->r_abort_on_full = true;
 		ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 		if (!ret)
 			ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1557,6 +1560,7 @@ static int ceph_zero_partial_object(struct inode *inode,
 	}
 
 	req->r_mtime = inode->i_mtime;
+	req->r_abort_on_full = true;
 	ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
 	if (!ret) {
 		ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
include/linux/ceph/osd_client.h +1 -0
index 03a6653d329a..5da666cc5891 100644
@@ -171,6 +171,7 @@ struct ceph_osd_request {
 
 	int               r_result;
 	bool              r_got_reply;
+	bool		  r_abort_on_full; /* return ENOSPC when full */
 
 	struct ceph_osd_client *r_osdc;
 	struct kref       r_kref;
net/ceph/osd_client.c +6 -0
index 3a2417bb6ff0..f68bb42da240 100644
@@ -49,6 +49,7 @@ static void link_linger(struct ceph_osd *osd,
 			struct ceph_osd_linger_request *lreq);
 static void unlink_linger(struct ceph_osd *osd,
 			  struct ceph_osd_linger_request *lreq);
+static void complete_request(struct ceph_osd_request *req, int err);
 
 #if 1
 static inline bool rwsem_is_wrlocked(struct rw_semaphore *sem)
@@ -1636,6 +1637,7 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	enum calc_target_result ct_res;
 	bool need_send = false;
 	bool promoted = false;
+	int ret = 0;
 
 	WARN_ON(req->r_tid || req->r_got_reply);
 	dout("%s req %p wrlocked %d\n", __func__, req, wrlocked);
@@ -1670,6 +1672,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 		pr_warn_ratelimited("FULL or reached pool quota\n");
 		req->r_t.paused = true;
 		maybe_request_map(osdc);
+		if (req->r_abort_on_full)
+			ret = -ENOSPC;
 	} else if (!osd_homeless(osd)) {
 		need_send = true;
 	} else {
@@ -1686,6 +1690,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 	link_request(osd, req);
 	if (need_send)
 		send_request(req);
+	else if (ret)
+		complete_request(req, ret);
 	mutex_unlock(&osd->lock);
 
 	if (ct_res == CALC_TARGET_POOL_DNE)
Signature · 1 line
2.9.3
Signature · 1 line
Jeff Layton <[email protected]>
maintainer
#4 lore raw

Re: [PATCH v4 1/6] libceph: allow requests to return immediately on full conditions if caller wishes

Quoted 65 lines · Jeff Layton
Quoted 54 lines · Zheng
Quoted 54 lines · Jeff Layton

Usually, when the osd map is flagged as full or the pool is at quota, write requests just hang. This is not what we want for cephfs, where it would be better to simply report -ENOSPC back to userland instead of stalling.

If the caller knows that it will want an immediate error return instead of blocking on a full or at-quota error condition then allow it to set a flag to request that behavior. Cephfs write requests will always set that flag.

A later patch will deal with requests that were submitted before the new map showing the full condition came in.

---

fs/ceph/addr.c                  | 4 ++++
fs/ceph/file.c                  | 4 ++++
include/linux/ceph/osd_client.h | 1 +
net/ceph/osd_client.c           | 6 ++++++
4 files changed, 15 insertions(+)
fs/ceph/addr.c +3 -0
index 4547bbf80e4f..308787eeee2c 100644
@@ -1040,6 +1040,7 @@ static int ceph_writepages_start(struct address_space *mapping,
 
            req->r_callback = writepages_finish;
            req->r_inode = inode;
+           req->r_abort_on_full = true;
 
            /* Format the osd request message and submit the write */
            len = 0;
@@ -1689,6 +1690,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
    }
 
    req->r_mtime = inode->i_mtime;
+   req->r_abort_on_full = true;
    err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
    if (!err)
            err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1732,6 +1734,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
    }
 
    req->r_mtime = inode->i_mtime;
+   req->r_abort_on_full = true;
    err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
    if (!err)
            err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1893,6 +1896,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
    err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false);
 
Quoted 10 lines

fs/ceph/addr.c static int __ceph_pool_perm_get(struct ceph_inode_info *ci, · line 1902

Quoted 5 lines
fs/ceph/addr.c +1 -0
    wr_req->r_mtime = ci->vfs_inode.i_mtime;
+   wr_req->r_abort_on_full = true;
    err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false);
 
    if (!err)

do you ignore writepage_nounlock() case intentionally?

Quoted 4 lines

No. Hmmm...writepage_nounlock calls ceph_osdc_writepages, and it's the only caller so I guess we'll need to set this there. Maybe we should just lift ceph_osdc_writepages into ceph.ko since there are no callers in libceph?

Set it in ceph_osdc_new_request() -- it's only user is ceph.ko. It should cover all filesystem OSD requests, except for pool check and ceph_aio_retry_work().

Thanks,

Ilya

#5 lore raw

Re: [PATCH v4 1/6] libceph: allow requests to return immediately on full conditions if caller wishes

Quoted 67 lines · Ilya Dryomov
Quoted 66 lines · Jeff Layton
Quoted 64 lines · Zheng
Quoted 52 lines · Jeff Layton

Usually, when the osd map is flagged as full or the pool is at quota, write requests just hang. This is not what we want for cephfs, where it would be better to simply report -ENOSPC back to userland instead of stalling.

If the caller knows that it will want an immediate error return instead of blocking on a full or at-quota error condition then allow it to set a flag to request that behavior. Cephfs write requests will always set that flag.

A later patch will deal with requests that were submitted before the new map showing the full condition came in.

---

fs/ceph/addr.c                  | 4 ++++
fs/ceph/file.c                  | 4 ++++
include/linux/ceph/osd_client.h | 1 +
net/ceph/osd_client.c           | 6 ++++++
4 files changed, 15 insertions(+)
fs/ceph/addr.c +3 -0
index 4547bbf80e4f..308787eeee2c 100644
@@ -1040,6 +1040,7 @@ static int ceph_writepages_start(struct address_space *mapping,
 
            req->r_callback = writepages_finish;
            req->r_inode = inode;
+           req->r_abort_on_full = true;
 
            /* Format the osd request message and submit the write */
            len = 0;
@@ -1689,6 +1690,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
    }
 
    req->r_mtime = inode->i_mtime;
+   req->r_abort_on_full = true;
    err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
    if (!err)
            err = ceph_osdc_wait_request(&fsc->client->osdc, req);
@@ -1732,6 +1734,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)
    }
 
    req->r_mtime = inode->i_mtime;
+   req->r_abort_on_full = true;
    err = ceph_osdc_start_request(&fsc->client->osdc, req, false);
    if (!err)
            err = ceph_osdc_wait_request(&fsc->client->osdc, req);

fs/ceph/addr.c static int __ceph_pool_perm_get(struct ceph_inode_info *ci, · line 1902

Quoted 7 lines
fs/ceph/addr.c +1 -0
@@ -1893,6 +1896,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
    err = ceph_osdc_start_request(&fsc->client->osdc, rd_req, false);
 
    wr_req->r_mtime = ci->vfs_inode.i_mtime;
+   wr_req->r_abort_on_full = true;
    err2 = ceph_osdc_start_request(&fsc->client->osdc, wr_req, false);
 
    if (!err)

do you ignore writepage_nounlock() case intentionally?

Quoted 9 lines
Quoted 4 lines

No. Hmmm...writepage_nounlock calls ceph_osdc_writepages, and it's the only caller so I guess we'll need to set this there. Maybe we should just lift ceph_osdc_writepages into ceph.ko since there are no callers in libceph?

Set it in ceph_osdc_new_request() -- it's only user is ceph.ko. It should cover all filesystem OSD requests, except for pool check and ceph_aio_retry_work().

Ok. The flag will end up being set on reads as well, but we don't actually do anything with the flag on a read request so that should be fine. I'll do that.

Thanks,

Signature · 1 line
Jeff Layton <[email protected]>
#6 lore raw

PATCH v4 2/6 libceph: abort already submitted but abortable requests when map or pool goes full

In reply to Jeff Layton

When a Ceph volume hits capacity, a flag is set in the OSD map to indicate that, and a new map is sprayed around the cluster. With cephfs we want it to shut down any abortable requests that are in progress with an -ENOSPC error as they'd just hang otherwise.

Add a new ceph_osdc_abort_on_full helper function to handle this. It will first check whether there is an out-of-space condition in the cluster. It will then walk the tree and abort any request that has r_abort_on_full set with an ENOSPC error. Call this new function directly whenever we get a new OSD map.

---
 net/ceph/osd_client.c | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
net/ceph/osd_client.c +42 -0
index f68bb42da240..cdb0b58c4c99 100644
@@ -1777,6 +1777,47 @@ static void complete_request(struct ceph_osd_request *req, int err)
 	ceph_osdc_put_request(req);
 }
 
+/*
+ * Drop all pending requests that are stalled waiting on a full condition to
+ * clear, and complete them with ENOSPC as the return code.
+ */
+static void ceph_osdc_abort_on_full(struct ceph_osd_client *osdc)
+{
+	struct ceph_osd_request *req;
+	struct ceph_osd *osd;
+	struct rb_node *m, *n;
+	u32 latest_epoch = 0;
+	bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL);
+
+	dout("enter abort_on_full\n");
+
+	if (!osdmap_full && !have_pool_full(osdc))
+		goto out;
+
+	for (n = rb_first(&osdc->osds); n; n = rb_next(n)) {
+		osd = rb_entry(n, struct ceph_osd, o_node);
+		mutex_lock(&osd->lock);
+		m = rb_first(&osd->o_requests);
+		while (m) {
+			req = rb_entry(m, struct ceph_osd_request, r_node);
+			m = rb_next(m);
+
+			if (req->r_abort_on_full &&
+			    (osdmap_full || pool_full(osdc, req->r_t.base_oloc.pool))) {
+				u32 cur_epoch = le32_to_cpu(req->r_replay_version.epoch);
+
+				dout("%s: abort tid=%llu flags 0x%x\n", __func__, req->r_tid, req->r_flags);
+				complete_request(req, -ENOSPC);
+				if (cur_epoch > latest_epoch)
+					latest_epoch = cur_epoch;
+			}
+		}
+		mutex_unlock(&osd->lock);
+	}
+out:
+	dout("return abort_on_full latest_epoch=%u\n", latest_epoch);
+}
+
 static void cancel_map_check(struct ceph_osd_request *req)
 {
 	struct ceph_osd_client *osdc = req->r_osdc;
@@ -3292,6 +3333,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg)
 
 	ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP,
 			  osdc->osdmap->epoch);
+	ceph_osdc_abort_on_full(osdc);
 	up_write(&osdc->lock);
 	wake_up_all(&osdc->client->auth_wq);
 	return;
Signature · 1 line
2.9.3
#7 lore raw

Re: [PATCH v4 2/6] libceph: abort already submitted but abortable requests when map or pool goes full

Quoted 42 lines · Jeff Layton

When a Ceph volume hits capacity, a flag is set in the OSD map to indicate that, and a new map is sprayed around the cluster. With cephfs we want it to shut down any abortable requests that are in progress with an -ENOSPC error as they'd just hang otherwise.

Add a new ceph_osdc_abort_on_full helper function to handle this. It will first check whether there is an out-of-space condition in the cluster. It will then walk the tree and abort any request that has r_abort_on_full set with an ENOSPC error. Call this new function directly whenever we get a new OSD map.

---
net/ceph/osd_client.c | 42 ++++++++++++++++++++++++++++++++++++++++++

1 file changed, 42 insertions(+)
net/ceph/osd_client.c +15 -0
index f68bb42da240..cdb0b58c4c99 100644
@@ -1777,6 +1777,47 @@ static void complete_request(struct ceph_osd_request *req, int err)
 	ceph_osdc_put_request(req);
 }
 
+/*
+ * Drop all pending requests that are stalled waiting on a full condition to
+ * clear, and complete them with ENOSPC as the return code.
+ */
+static void ceph_osdc_abort_on_full(struct ceph_osd_client *osdc)
+{
+	struct ceph_osd_request *req;
+	struct ceph_osd *osd;
+	struct rb_node *m, *n;
+	u32 latest_epoch = 0;
+	bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL);
+
+	dout("enter abort_on_full\n");
+
+	if (!osdmap_full && !have_pool_full(osdc))

net/ceph/osd_client.c static void complete_request(struct ceph_osd_request *req, int err) · line 1804

Quoted 10 lines
net/ceph/osd_client.c +10 -0
+		goto out;
+
+	for (n = rb_first(&osdc->osds); n; n = rb_next(n)) {
+		osd = rb_entry(n, struct ceph_osd, o_node);
+		mutex_lock(&osd->lock);
+		m = rb_first(&osd->o_requests);
+		while (m) {
+			req = rb_entry(m, struct ceph_osd_request, r_node);
+			m = rb_next(m);
+

For requests that have already got unsafe reply, we should ignore them or call req->r_unsafe_callback() to clean them up

Regards
Yan, Zheng

Quoted 30 lines
net/ceph/osd_client.c +17 -0
+			if (req->r_abort_on_full &&
+			    (osdmap_full || pool_full(osdc, req->r_t.base_oloc.pool))) {
+				u32 cur_epoch = le32_to_cpu(req->r_replay_version.epoch);
+
+				dout("%s: abort tid=%llu flags 0x%x\n", __func__, req->r_tid, req->r_flags);
+				complete_request(req, -ENOSPC);
+				if (cur_epoch > latest_epoch)
+					latest_epoch = cur_epoch;
+			}
+		}
+		mutex_unlock(&osd->lock);
+	}
+out:
+	dout("return abort_on_full latest_epoch=%u\n", latest_epoch);
+}
+
 static void cancel_map_check(struct ceph_osd_request *req)
 {
 	struct ceph_osd_client *osdc = req->r_osdc;
@@ -3292,6 +3333,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg)
 
 	ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP,
 			  osdc->osdmap->epoch);
+	ceph_osdc_abort_on_full(osdc);
 	up_write(&osdc->lock);
 	wake_up_all(&osdc->client->auth_wq);
 	return;
Signature · 1 line
2.9.3
#8 lore raw

Re: [PATCH v4 2/6] libceph: abort already submitted but abortable requests when map or pool goes full

Quoted 55 lines · Zheng
Quoted 41 lines · Jeff Layton

When a Ceph volume hits capacity, a flag is set in the OSD map to indicate that, and a new map is sprayed around the cluster. With cephfs we want it to shut down any abortable requests that are in progress with an -ENOSPC error as they'd just hang otherwise.

Add a new ceph_osdc_abort_on_full helper function to handle this. It will first check whether there is an out-of-space condition in the cluster. It will then walk the tree and abort any request that has r_abort_on_full set with an ENOSPC error. Call this new function directly whenever we get a new OSD map.

---
net/ceph/osd_client.c | 42 ++++++++++++++++++++++++++++++++++++++++++

1 file changed, 42 insertions(+)
net/ceph/osd_client.c +15 -0
index f68bb42da240..cdb0b58c4c99 100644
@@ -1777,6 +1777,47 @@ static void complete_request(struct ceph_osd_request *req, int err)
 	ceph_osdc_put_request(req);
 }
 
+/*
+ * Drop all pending requests that are stalled waiting on a full condition to
+ * clear, and complete them with ENOSPC as the return code.
+ */
+static void ceph_osdc_abort_on_full(struct ceph_osd_client *osdc)
+{
+	struct ceph_osd_request *req;
+	struct ceph_osd *osd;
+	struct rb_node *m, *n;
+	u32 latest_epoch = 0;
+	bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL);
+
+	dout("enter abort_on_full\n");
+
+	if (!osdmap_full && !have_pool_full(osdc))

net/ceph/osd_client.c static void complete_request(struct ceph_osd_request *req, int err) · line 1804

Quoted 10 lines
net/ceph/osd_client.c +10 -0
+		goto out;
+
+	for (n = rb_first(&osdc->osds); n; n = rb_next(n)) {
+		osd = rb_entry(n, struct ceph_osd, o_node);
+		mutex_lock(&osd->lock);
+		m = rb_first(&osd->o_requests);
+		while (m) {
+			req = rb_entry(m, struct ceph_osd_request, r_node);
+			m = rb_next(m);
+

For requests that have already got unsafe reply, we should ignore them or call req->r_unsafe_callback() to clean them up

Quoted 2 lines

Regards
Yan, Zheng

Ok, yeah. I had to stare at the r_unsafe_callback code a bit the other day to handle the ERROR_WRITE flag and what you say makes sense.

Honestly what we really need is a function like complete_request that hides all of these fiddly details about the request state. Maybe it would be simpler to just have complete_request handle the case where we've gotten an unsafe reply as well?

I'll see what I can come up with there.

Quoted 32 lines
Quoted 30 lines
net/ceph/osd_client.c +17 -0
+			if (req->r_abort_on_full &&
+			    (osdmap_full || pool_full(osdc, req->r_t.base_oloc.pool))) {
+				u32 cur_epoch = le32_to_cpu(req->r_replay_version.epoch);
+
+				dout("%s: abort tid=%llu flags 0x%x\n", __func__, req->r_tid, req->r_flags);
+				complete_request(req, -ENOSPC);
+				if (cur_epoch > latest_epoch)
+					latest_epoch = cur_epoch;
+			}
+		}
+		mutex_unlock(&osd->lock);
+	}
+out:
+	dout("return abort_on_full latest_epoch=%u\n", latest_epoch);
+}
+
 static void cancel_map_check(struct ceph_osd_request *req)
 {
 	struct ceph_osd_client *osdc = req->r_osdc;
@@ -3292,6 +3333,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg)
 
 	ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP,
 			  osdc->osdmap->epoch);
+	ceph_osdc_abort_on_full(osdc);
 	up_write(&osdc->lock);
 	wake_up_all(&osdc->client->auth_wq);
 	return;
Signature · 1 line
2.9.3
Signature · 1 line
Jeff Layton <[email protected]>
maintainer
#9 lore raw

Re: [PATCH v4 2/6] libceph: abort already submitted but abortable requests when map or pool goes full

Quoted 62 lines · Jeff Layton
Quoted 55 lines · Zheng
Quoted 41 lines · Jeff Layton

When a Ceph volume hits capacity, a flag is set in the OSD map to indicate that, and a new map is sprayed around the cluster. With cephfs we want it to shut down any abortable requests that are in progress with an -ENOSPC error as they'd just hang otherwise.

Add a new ceph_osdc_abort_on_full helper function to handle this. It will first check whether there is an out-of-space condition in the cluster. It will then walk the tree and abort any request that has r_abort_on_full set with an ENOSPC error. Call this new function directly whenever we get a new OSD map.

---
net/ceph/osd_client.c | 42 ++++++++++++++++++++++++++++++++++++++++++

1 file changed, 42 insertions(+)
net/ceph/osd_client.c +15 -0
index f68bb42da240..cdb0b58c4c99 100644
@@ -1777,6 +1777,47 @@ static void complete_request(struct ceph_osd_request *req, int err)
    ceph_osdc_put_request(req);
 }
 
+/*
+ * Drop all pending requests that are stalled waiting on a full condition to
+ * clear, and complete them with ENOSPC as the return code.
+ */
+static void ceph_osdc_abort_on_full(struct ceph_osd_client *osdc)
+{
+   struct ceph_osd_request *req;
+   struct ceph_osd *osd;
+   struct rb_node *m, *n;
+   u32 latest_epoch = 0;
+   bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL);
+
+   dout("enter abort_on_full\n");
+
+   if (!osdmap_full && !have_pool_full(osdc))

net/ceph/osd_client.c static void complete_request(struct ceph_osd_request *req, int err) · line 1804

Quoted 10 lines
net/ceph/osd_client.c +10 -0
+           goto out;
+
+   for (n = rb_first(&osdc->osds); n; n = rb_next(n)) {
+           osd = rb_entry(n, struct ceph_osd, o_node);
+           mutex_lock(&osd->lock);
+           m = rb_first(&osd->o_requests);
+           while (m) {
+                   req = rb_entry(m, struct ceph_osd_request, r_node);
+                   m = rb_next(m);
+

For requests that have already got unsafe reply, we should ignore them or call req->r_unsafe_callback() to clean them up

Quoted 2 lines

Regards
Yan, Zheng

Ok, yeah. I had to stare at the r_unsafe_callback code a bit the other day to handle the ERROR_WRITE flag and what you say makes sense.

Quoted 6 lines

Honestly what we really need is a function like complete_request that hides all of these fiddly details about the request state. Maybe it would be simpler to just have complete_request handle the case where we've gotten an unsafe reply as well?

I'll see what I can come up with there.

This is exactly what I meant in my reply to Artur yesterday. Note that ceph_osdc_cancel_request() is explicit about not completing the request. It shouldn't be hard to mend it though -- let me look into it.

Thanks,

Ilya

#10 lore raw

PATCH v4 3/6 libceph: add an epoch_barrier field to struct ceph_osd_client

In reply to Jeff Layton

Cephfs can get cap update requests that contain a new epoch barrier in them. When that happens we want to pause all OSD traffic until the right map epoch arrives.

Add an epoch_barrier field to ceph_osd_client that is protected by the osdc->lock rwsem. When the barrier is set, and the current OSD map epoch is below that, pause the request target when submitting the request or when revisiting it. Add a way for upper layers (cephfs) to update the epoch_barrier as well.

If we get a new map, compare the new epoch against the barrier before kicking requests and request another map if the map epoch is still lower than the one we want.

If we end up cancelling requests because of a new map showing a full OSD or pool condition, then set the barrier higher than the highest replay epoch of all the cancelled requests.

---
 include/linux/ceph/osd_client.h |  2 ++
 net/ceph/osd_client.c           | 51 +++++++++++++++++++++++++++++++++--------
 2 files changed, 43 insertions(+), 10 deletions(-)
include/linux/ceph/osd_client.h +2 -0
index 5da666cc5891..b4e5a1b45f24 100644
@@ -270,6 +270,7 @@ struct ceph_osd_client {
 	struct rb_root         osds;          /* osds */
 	struct list_head       osd_lru;       /* idle osds */
 	spinlock_t             osd_lru_lock;
+	u32		       epoch_barrier;
 	struct ceph_osd        homeless_osd;
 	atomic64_t             last_tid;      /* tid of last request */
 	u64                    last_linger_id;
@@ -308,6 +309,7 @@ extern void ceph_osdc_handle_reply(struct ceph_osd_client *osdc,
 				   struct ceph_msg *msg);
 extern void ceph_osdc_handle_map(struct ceph_osd_client *osdc,
 				 struct ceph_msg *msg);
+void ceph_osdc_update_epoch_barrier(struct ceph_osd_client *osdc, u32 eb);
 
 extern void osd_req_op_init(struct ceph_osd_request *osd_req,
 			    unsigned int which, u16 opcode, u32 flags);
net/ceph/osd_client.c +41 -10

105 lines not on this page · raw message

Signature · 1 line
2.9.3
#11 lore raw

PATCH v4 4/6 ceph: handle epoch barriers in cap messages

In reply to Jeff Layton

Have the client store and update the osdc epoch_barrier when a cap message comes in with one.

When sending cap messages, send the epoch barrier as well. This allows clients to inform servers that their released caps may not be used until a particular OSD map epoch.

---
 fs/ceph/caps.c       | 17 +++++++++++++----
 fs/ceph/mds_client.c | 20 ++++++++++++++++++++
 fs/ceph/mds_client.h |  7 +++++--
 3 files changed, 38 insertions(+), 6 deletions(-)
fs/ceph/caps.c +13 -4
index 3c2dfd72e5b2..d91d3f32a5b6 100644
@@ -1015,6 +1015,7 @@ static int send_cap_msg(struct cap_msg_args *arg)
 	void *p;
 	size_t extra_len;
 	struct timespec zerotime = {0};
+	struct ceph_osd_client *osdc = &arg->session->s_mdsc->fsc->client->osdc;
 
 	dout("send_cap_msg %s %llx %llx caps %s wanted %s dirty %s"
 	     " seq %u/%u tid %llu/%llu mseq %u follows %lld size %llu/%llu"
@@ -1077,7 +1078,9 @@ static int send_cap_msg(struct cap_msg_args *arg)
 	/* inline data size */
 	ceph_encode_32(&p, 0);
 	/* osd_epoch_barrier (version 5) */
-	ceph_encode_32(&p, 0);
+	down_read(&osdc->lock);
+	ceph_encode_32(&p, osdc->epoch_barrier);
+	up_read(&osdc->lock);
 	/* oldest_flush_tid (version 6) */
 	ceph_encode_64(&p, arg->oldest_flush_tid);
 
@@ -3635,13 +3638,19 @@ void ceph_handle_caps(struct ceph_mds_session *session,
 		p += inline_len;
 	}
 
+	if (le16_to_cpu(msg->hdr.version) >= 5) {
+		struct ceph_osd_client	*osdc = &mdsc->fsc->client->osdc;
+		u32			epoch_barrier;
+
+		ceph_decode_32_safe(&p, end, epoch_barrier, bad);
+		ceph_osdc_update_epoch_barrier(osdc, epoch_barrier);
+	}
+
 	if (le16_to_cpu(msg->hdr.version) >= 8) {
 		u64 flush_tid;
 		u32 caller_uid, caller_gid;
-		u32 osd_epoch_barrier;
 		u32 pool_ns_len;
-		/* version >= 5 */
-		ceph_decode_32_safe(&p, end, osd_epoch_barrier, bad);
+
 		/* version >= 6 */
 		ceph_decode_64_safe(&p, end, flush_tid, bad);
 		/* version >= 7 */
fs/ceph/mds_client.c +20 -0
index b47e97680a66..85ec84157d3d 100644
@@ -1550,9 +1550,15 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 	struct ceph_msg *msg = NULL;
 	struct ceph_mds_cap_release *head;
 	struct ceph_mds_cap_item *item;
+	struct ceph_osd_client *osdc = &mdsc->fsc->client->osdc;
 	struct ceph_cap *cap;
 	LIST_HEAD(tmp_list);
 	int num_cap_releases;
+	__le32	barrier, *cap_barrier;
+
+	down_read(&osdc->lock);
+	barrier = cpu_to_le32(osdc->epoch_barrier);
+	up_read(&osdc->lock);
 
 	spin_lock(&session->s_cap_lock);
 again:
@@ -1570,7 +1576,11 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 			head = msg->front.iov_base;
 			head->num = cpu_to_le32(0);
 			msg->front.iov_len = sizeof(*head);
+
+			msg->hdr.version = cpu_to_le16(2);
+			msg->hdr.compat_version = cpu_to_le16(1);
 		}
+
 		cap = list_first_entry(&tmp_list, struct ceph_cap,
 					session_caps);
 		list_del(&cap->session_caps);
@@ -1588,6 +1598,11 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 		ceph_put_cap(mdsc, cap);
 
 		if (le32_to_cpu(head->num) == CEPH_CAPS_PER_RELEASE) {
+			// Append cap_barrier field
+			cap_barrier = msg->front.iov_base + msg->front.iov_len;
+			*cap_barrier = barrier;
+			msg->front.iov_len += sizeof(*cap_barrier);
+
 			msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
 			dout("send_cap_releases mds%d %p\n", session->s_mds, msg);
 			ceph_con_send(&session->s_con, msg);
@@ -1603,6 +1618,11 @@ void ceph_send_cap_releases(struct ceph_mds_client *mdsc,
 	spin_unlock(&session->s_cap_lock);
 
 	if (msg) {
+		// Append cap_barrier field
+		cap_barrier = msg->front.iov_base + msg->front.iov_len;
+		*cap_barrier = barrier;
+		msg->front.iov_len += sizeof(*cap_barrier);
+
 		msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
 		dout("send_cap_releases mds%d %p\n", session->s_mds, msg);
 		ceph_con_send(&session->s_con, msg);
fs/ceph/mds_client.h +5 -2
index ac0475a2daa7..517684c7c5f0 100644
@@ -104,10 +104,13 @@ struct ceph_mds_reply_info_parsed {
 
 /*
  * cap releases are batched and sent to the MDS en masse.
+ *
+ * Account for per-message overhead of mds_cap_release header
+ * and __le32 for osd epoch barrier trailing field.
  */
-#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE -			\
+#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE - sizeof(u32) -		\
 				sizeof(struct ceph_mds_cap_release)) /	\
-			       sizeof(struct ceph_mds_cap_item))
+			        sizeof(struct ceph_mds_cap_item))
 
 
 /*
Signature · 1 line
2.9.3
#12 lore raw

PATCH v4 5/6 Revert "ceph: SetPageError() for writeback pages if writepages fails"

In reply to Jeff Layton

This reverts commit b109eec6f4332bd517e2f41e207037c4b9065094.

If I'm filling up a filesystem with this sort of command:

    $ dd if=/dev/urandom of=/mnt/cephfs/fillfile bs=2M oflag=sync

...then I'll eventually get back EIO on a write. Further calls will give us ENOSPC.

I'm not sure what prompted this change, but I don't think it's what we want to do. If writepages failed, we will have already set the mapping error appropriately, and that's what gets reported by fsync() or close().

__filemap_fdatawait_range however, does this:

	wait_on_page_writeback(page);
	if (TestClearPageError(page))
		ret = -EIO;

...and that -EIO ends up trumping the mapping's error if one exists.

When writepages fails, we only want to set the error in the mapping, and not flag the individual pages.

---
 fs/ceph/addr.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
fs/ceph/addr.c +1 -3
index 308787eeee2c..040d05c8f4a2 100644
@@ -668,6 +668,7 @@ static void writepages_finish(struct ceph_osd_request *req)
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
 	bool remove_page;
 
+
 	dout("writepages_finish %p rc %d\n", inode, rc);
 	if (rc < 0)
 		mapping_set_error(mapping, rc);
@@ -702,9 +703,6 @@ static void writepages_finish(struct ceph_osd_request *req)
 				clear_bdi_congested(&fsc->backing_dev_info,
 						    BLK_RW_ASYNC);
 
-			if (rc < 0)
-				SetPageError(page);
-
 			ceph_put_snap_context(page_snap_context(page));
 			page->private = 0;
 			ClearPagePrivate(page);
Signature · 1 line
2.9.3
#13 lore raw

Re: [PATCH v4 5/6] Revert "ceph: SetPageError() for writeback pages if writepages fails"

Quoted 19 lines · Jeff Layton

This reverts commit b109eec6f4332bd517e2f41e207037c4b9065094.

If I'm filling up a filesystem with this sort of command:

   $ dd if=/dev/urandom of=/mnt/cephfs/fillfile bs=2M oflag=sync

...then I'll eventually get back EIO on a write. Further calls will give us ENOSPC.

I'm not sure what prompted this change, but I don't think it's what we want to do. If writepages failed, we will have already set the mapping error appropriately, and that's what gets reported by fsync() or close().

__filemap_fdatawait_range however, does this:

Quoted 8 lines
	wait_on_page_writeback(page);
	if (TestClearPageError(page))
		ret = -EIO;

...and that -EIO ends up trumping the mapping's error if one exists.

When writepages fails, we only want to set the error in the mapping, and not flag the individual pages.

I think you are right. Maybe we should also remove the SetPageError in writepage_nounlock

Regards
Yan, Zheng

Quoted 31 lines

---
fs/ceph/addr.c | 4 +---

1 file changed, 1 insertion(+), 3 deletions(-)
fs/ceph/addr.c +1 -3
index 308787eeee2c..040d05c8f4a2 100644
@@ -668,6 +668,7 @@ static void writepages_finish(struct ceph_osd_request *req)
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
 	bool remove_page;
 
+
 	dout("writepages_finish %p rc %d\n", inode, rc);
 	if (rc < 0)
 		mapping_set_error(mapping, rc);
@@ -702,9 +703,6 @@ static void writepages_finish(struct ceph_osd_request *req)
 				clear_bdi_congested(&fsc->backing_dev_info,
 						    BLK_RW_ASYNC);
 
-			if (rc < 0)
-				SetPageError(page);
-
 			ceph_put_snap_context(page_snap_context(page));
 			page->private = 0;
 			ClearPagePrivate(page);
Signature · 1 line
2.9.3
#14 lore raw

Re: [PATCH v4 5/6] Revert "ceph: SetPageError() for writeback pages if writepages fails"

Quoted 22 lines · Zheng
Quoted 22 lines · Jeff Layton

This reverts commit b109eec6f4332bd517e2f41e207037c4b9065094.

If I'm filling up a filesystem with this sort of command:

   $ dd if=/dev/urandom of=/mnt/cephfs/fillfile bs=2M oflag=sync

...then I'll eventually get back EIO on a write. Further calls will give us ENOSPC.

I'm not sure what prompted this change, but I don't think it's what we want to do. If writepages failed, we will have already set the mapping error appropriately, and that's what gets reported by fsync() or close().

__filemap_fdatawait_range however, does this:

	wait_on_page_writeback(page);
	if (TestClearPageError(page))
		ret = -EIO;
Quoted 7 lines
Quoted 4 lines

...and that -EIO ends up trumping the mapping's error if one exists.

When writepages fails, we only want to set the error in the mapping, and not flag the individual pages.

I think you are right. Maybe we should also remove the SetPageError in writepage_nounlock

Yeah, good catch. The last patch in this series should probably also call ceph_set/clear_error_write in that function as well.

Quoted 33 lines
Quoted 31 lines

---
fs/ceph/addr.c | 4 +---

1 file changed, 1 insertion(+), 3 deletions(-)
fs/ceph/addr.c +1 -3
index 308787eeee2c..040d05c8f4a2 100644
@@ -668,6 +668,7 @@ static void writepages_finish(struct ceph_osd_request *req)
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
 	bool remove_page;
 
+
 	dout("writepages_finish %p rc %d\n", inode, rc);
 	if (rc < 0)
 		mapping_set_error(mapping, rc);
@@ -702,9 +703,6 @@ static void writepages_finish(struct ceph_osd_request *req)
 				clear_bdi_congested(&fsc->backing_dev_info,
 						    BLK_RW_ASYNC);
 
-			if (rc < 0)
-				SetPageError(page);
-
 			ceph_put_snap_context(page_snap_context(page));
 			page->private = 0;
 			ClearPagePrivate(page);
Signature · 1 line
2.9.3
Signature · 1 line
Jeff Layton <[email protected]>
#15 lore raw

PATCH v4 6/6 ceph: when seeing write errors on an inode, switch to sync writes

In reply to Jeff Layton

Currently, we don't have a real feedback mechanism in place for when we start seeing buffered writeback errors. If writeback is failing, there is nothing that prevents an application from continuing to dirty pages that aren't being cleaned.

In the event that we're seeing write errors of any sort occur on an inode, have the callback set a flag to force further writes to be synchronous. When the next write succeeds, clear the flag to allow buffered writeback to continue.

Since this is just a hint to the write submission mechanism, we only take the i_ceph_lock when a lockless check shows that the flag needs to be changed.

---
 fs/ceph/addr.c  |  6 +++++-
 fs/ceph/file.c  | 46 +++++++++++++++++++++++++++++++++-------------
 fs/ceph/super.h | 26 ++++++++++++++++++++++++++
 3 files changed, 64 insertions(+), 14 deletions(-)
fs/ceph/addr.c +5 -1
index 040d05c8f4a2..596f782fd35e 100644
@@ -670,8 +670,12 @@ static void writepages_finish(struct ceph_osd_request *req)
 
 
 	dout("writepages_finish %p rc %d\n", inode, rc);
-	if (rc < 0)
+	if (rc < 0) {
 		mapping_set_error(mapping, rc);
+		ceph_set_error_write(ci);
+	} else {
+		ceph_clear_error_write(ci);
+	}
 
 	/*
 	 * We lost the cache cap, need to truncate the page before
fs/ceph/file.c +33 -13

80 lines not on this page · raw message

fs/ceph/super.h +26 -0
index 950170136be9..3b987da37dcd 100644
@@ -474,6 +474,32 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,
 #define CEPH_I_CAP_DROPPED	(1 << 8)  /* caps were forcibly dropped */
 #define CEPH_I_KICK_FLUSH	(1 << 9)  /* kick flushing caps */
 #define CEPH_I_FLUSH_SNAPS	(1 << 10) /* need flush snapss */
+#define CEPH_I_ERROR_WRITE	(1 << 11) /* have seen write errors */
+
+/*
+ * We set the ERROR_WRITE bit when we start seeing write errors on an inode
+ * and then clear it when they start succeeding. Note that we do a lockless
+ * check first, and only take the lock if it looks like it needs to be changed.
+ * The write submission code just takes this as a hint, so we're not too
+ * worried if a few slip through in either direction.
+ */
+static inline void ceph_set_error_write(struct ceph_inode_info *ci)
+{
+	if (!(ci->i_ceph_flags & CEPH_I_ERROR_WRITE)) {
+		spin_lock(&ci->i_ceph_lock);
+		ci->i_ceph_flags |= CEPH_I_ERROR_WRITE;
+		spin_unlock(&ci->i_ceph_lock);
+	}
+}
+
+static inline void ceph_clear_error_write(struct ceph_inode_info *ci)
+{
+	if (ci->i_ceph_flags & CEPH_I_ERROR_WRITE) {
+		spin_lock(&ci->i_ceph_lock);
+		ci->i_ceph_flags &= ~CEPH_I_ERROR_WRITE;
+		spin_unlock(&ci->i_ceph_lock);
+	}
+}
 
 static inline void __ceph_dir_set_complete(struct ceph_inode_info *ci,
 					   long long release_count,
Signature · 1 line
2.9.3