Skip to contentSkip to conversation

ceph: using POOL FULL flag instead of OSDMAP FULL flag

PATCH In review Yanhu Cao 2 participants 6 messages Mar 3, 2020 – Mar 12, 2020

Subsystems (4)
lore
Share & tools
lore

Yanhu Cao

author
#0 lore raw

PATCH v2 1/1 ceph: using POOL FULL flag instead of OSDMAP FULL flag

CEPH_OSDMAP_FULL/NEARFULL has been deprecated since mimic, so it does not work well in new versions, added POOL flags to handle it.

---
 fs/ceph/file.c                  |  9 +++++++--
 include/linux/ceph/osd_client.h |  2 ++
 include/linux/ceph/osdmap.h     |  3 ++-
 net/ceph/osd_client.c           | 23 +++++++++++++----------
 4 files changed, 24 insertions(+), 13 deletions(-)
fs/ceph/file.c +7 -2
index 7e0190b1f821..84ec44f9d77a 100644
@@ -1482,7 +1482,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 
 	/* FIXME: not complete since it doesn't account for being at quota */
-	if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+	if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL) ||
+	    pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+						CEPH_POOL_FLAG_FULL)) {
 		err = -ENOSPC;
 		goto out;
 	}
@@ -1575,7 +1577,10 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 
 	if (written >= 0) {
-		if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+		if (ceph_osdmap_flag(&fsc->client->osdc,
+					CEPH_OSDMAP_NEARFULL) ||
+		    pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+					CEPH_POOL_FLAG_NEARFULL))
 			iocb->ki_flags |= IOCB_DSYNC;
 		written = generic_write_sync(iocb, written);
 	}
include/linux/ceph/osd_client.h +2 -0
index 5a62dbd3f4c2..be9007b93862 100644
@@ -375,6 +375,8 @@ static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
 	return osdc->osdmap->flags & flag;
 }
 
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag);
+
 extern int ceph_osdc_setup(void);
 extern void ceph_osdc_cleanup(void);
 
include/linux/ceph/osdmap.h +2 -1
index e081b56f1c1d..88faacc11f55 100644
@@ -36,7 +36,8 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 
 #define CEPH_POOL_FLAG_HASHPSPOOL	(1ULL << 0) /* hash pg seed and pool id
 						       together */
-#define CEPH_POOL_FLAG_FULL		(1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_FULL		(1ULL << 1)  /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL	(1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
 	struct rb_node node;
net/ceph/osd_client.c +13 -10
index b68b376d8c2f..9ad2b96c3e78 100644
@@ -1447,9 +1447,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
 		atomic_dec(&osd->o_osdc->num_homeless);
 }
 
-static bool __pool_full(struct ceph_pg_pool_info *pi)
+static bool __pool_flag(struct ceph_pg_pool_info *pi, int flag)
 {
-	return pi->flags & CEPH_POOL_FLAG_FULL;
+	return pi->flags & flag;
 }
 
 static bool have_pool_full(struct ceph_osd_client *osdc)
@@ -1460,14 +1460,14 @@ static bool have_pool_full(struct ceph_osd_client *osdc)
 		struct ceph_pg_pool_info *pi =
 		    rb_entry(n, struct ceph_pg_pool_info, node);
 
-		if (__pool_full(pi))
+		if (__pool_flag(pi, CEPH_POOL_FLAG_FULL))
 			return true;
 	}
 
 	return false;
 }
 
-static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag)
 {
 	struct ceph_pg_pool_info *pi;
 
@@ -1475,8 +1475,10 @@ static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
 	if (!pi)
 		return false;
 
-	return __pool_full(pi);
+	return __pool_flag(pi, flag);
 }
+EXPORT_SYMBOL(pool_flag);
+
 
 /*
  * Returns whether a request should be blocked from being sent
@@ -1489,7 +1491,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc,
 	bool pauserd = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD);
 	bool pausewr = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSEWR) ||
 		       ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-		       __pool_full(pi);
+		       __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 
 	WARN_ON(pi->id != t->target_oloc.pool);
 	return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) ||
@@ -2320,7 +2322,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
 		   !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY |
 				     CEPH_OSD_FLAG_FULL_FORCE)) &&
 		   (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-		    pool_full(osdc, req->r_t.base_oloc.pool))) {
+		   pool_flag(osdc, req->r_t.base_oloc.pool,
+			     CEPH_POOL_FLAG_FULL))) {
 		dout("req %p full/pool_full\n", req);
 		if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) {
 			err = -ENOSPC;
@@ -2539,7 +2542,7 @@ static int abort_on_full_fn(struct ceph_osd_request *req, void *arg)
 
 	if ((req->r_flags & CEPH_OSD_FLAG_WRITE) &&
 	    (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-	     pool_full(osdc, req->r_t.base_oloc.pool))) {
+	     pool_flag(osdc, req->r_t.base_oloc.pool, CEPH_POOL_FLAG_FULL))) {
 		if (!*victims) {
 			update_epoch_barrier(osdc, osdc->osdmap->epoch);
 			*victims = true;
@@ -3707,7 +3710,7 @@ static void set_pool_was_full(struct ceph_osd_client *osdc)
 		struct ceph_pg_pool_info *pi =
 		    rb_entry(n, struct ceph_pg_pool_info, node);
 
-		pi->was_full = __pool_full(pi);
+		pi->was_full = __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 	}
 }
 
@@ -3719,7 +3722,7 @@ static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id)
 	if (!pi)
 		return false;
 
-	return pi->was_full && !__pool_full(pi);
+	return pi->was_full && !__pool_flag(pi, CEPH_POOL_FLAG_FULL);
 }
 
 static enum calc_target_result
Signature · 1 line
2.21.1
maintainer
#1 lore raw
Quoted 149 lines · Yanhu Cao

CEPH_OSDMAP_FULL/NEARFULL has been deprecated since mimic, so it does not work well in new versions, added POOL flags to handle it.

---
 fs/ceph/file.c                  |  9 +++++++--
 include/linux/ceph/osd_client.h |  2 ++
 include/linux/ceph/osdmap.h     |  3 ++-
 net/ceph/osd_client.c           | 23 +++++++++++++----------
 4 files changed, 24 insertions(+), 13 deletions(-)
fs/ceph/file.c +7 -2
index 7e0190b1f821..84ec44f9d77a 100644
@@ -1482,7 +1482,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        /* FIXME: not complete since it doesn't account for being at quota */
-       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL) ||
+           pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                               CEPH_POOL_FLAG_FULL)) {
                err = -ENOSPC;
                goto out;
        }
@@ -1575,7 +1577,10 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        if (written >= 0) {
-               if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+               if (ceph_osdmap_flag(&fsc->client->osdc,
+                                       CEPH_OSDMAP_NEARFULL) ||
+                   pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                       CEPH_POOL_FLAG_NEARFULL))
                        iocb->ki_flags |= IOCB_DSYNC;
                written = generic_write_sync(iocb, written);
        }
include/linux/ceph/osd_client.h +2 -0
index 5a62dbd3f4c2..be9007b93862 100644
@@ -375,6 +375,8 @@ static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
        return osdc->osdmap->flags & flag;
 }
 
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag);
+
 extern int ceph_osdc_setup(void);
 extern void ceph_osdc_cleanup(void);
 
include/linux/ceph/osdmap.h +2 -1
index e081b56f1c1d..88faacc11f55 100644
@@ -36,7 +36,8 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 
 #define CEPH_POOL_FLAG_HASHPSPOOL      (1ULL << 0) /* hash pg seed and pool id
                                                       together */
-#define CEPH_POOL_FLAG_FULL            (1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_FULL            (1ULL << 1)  /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL        (1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
        struct rb_node node;
net/ceph/osd_client.c +12 -9
index b68b376d8c2f..9ad2b96c3e78 100644
@@ -1447,9 +1447,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
                atomic_dec(&osd->o_osdc->num_homeless);
 }
 
-static bool __pool_full(struct ceph_pg_pool_info *pi)
+static bool __pool_flag(struct ceph_pg_pool_info *pi, int flag)
 {
-       return pi->flags & CEPH_POOL_FLAG_FULL;
+       return pi->flags & flag;
 }
 
 static bool have_pool_full(struct ceph_osd_client *osdc)
@@ -1460,14 +1460,14 @@ static bool have_pool_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               if (__pool_full(pi))
+               if (__pool_flag(pi, CEPH_POOL_FLAG_FULL))
                        return true;
        }
 
        return false;
 }
 
-static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag)
 {
        struct ceph_pg_pool_info *pi;
 
@@ -1475,8 +1475,10 @@ static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return __pool_full(pi);
+       return __pool_flag(pi, flag);
 }
+EXPORT_SYMBOL(pool_flag);
+
 
 /*
  * Returns whether a request should be blocked from being sent
@@ -1489,7 +1491,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc,
        bool pauserd = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD);
        bool pausewr = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSEWR) ||
                       ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                      __pool_full(pi);
+                      __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 
        WARN_ON(pi->id != t->target_oloc.pool);
        return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) ||
@@ -2320,7 +2322,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
                   !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY |
                                     CEPH_OSD_FLAG_FULL_FORCE)) &&
                   (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                   pool_full(osdc, req->r_t.base_oloc.pool))) {
+                  pool_flag(osdc, req->r_t.base_oloc.pool,
+                            CEPH_POOL_FLAG_FULL))) {
                dout("req %p full/pool_full\n", req);
                if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) {
                        err = -ENOSPC;
@@ -2539,7 +2542,7 @@ static int abort_on_full_fn(struct ceph_osd_request *req, void *arg)
 
        if ((req->r_flags & CEPH_OSD_FLAG_WRITE) &&
            (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-            pool_full(osdc, req->r_t.base_oloc.pool))) {
+            pool_flag(osdc, req->r_t.base_oloc.pool, CEPH_POOL_FLAG_FULL))) {
                if (!*victims) {
                        update_epoch_barrier(osdc, osdc->osdmap->epoch);
                        *victims = true;
@@ -3707,7 +3710,7 @@ static void set_pool_was_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               pi->was_full = __pool_full(pi);
+               pi->was_full = __pool_flag(pi, CEPH_POOL_FLAG_FULL);
        }
 }
 

net/ceph/osd_client.c static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id) · line 3728

Quoted 8 lines
net/ceph/osd_client.c +1 -1
@@ -3719,7 +3722,7 @@ static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return pi->was_full && !__pool_full(pi);
+       return pi->was_full && !__pool_flag(pi, CEPH_POOL_FLAG_FULL);
 }
 
 static enum calc_target_result

Hi Yanhu,

Sorry for a late reply.

This adds some unnecessary churn and also exposes a helper that must be called under osdc->lock without making that obvious. How about the attached instead?

ceph_pg_pool_flags() takes osdmap instead of osdc, making it clear that the caller is resposibile for keeping the map stable.

Thanks,

Ilya

fs/ceph/file.c +11 -2
index ba46ba740628..c8d84e90a371 100644
@@ -1663,10 +1663,13 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	struct inode *inode = file_inode(file);
 	struct ceph_inode_info *ci = ceph_inode(inode);
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
+	struct ceph_osd_client *osdc = &fsc->client->osdc;
 	struct ceph_cap_flush *prealloc_cf;
 	ssize_t count, written = 0;
 	int err, want, got;
 	bool direct_lock = false;
+	u32 map_flags;
+	u64 pool_flags;
 	loff_t pos;
 	loff_t limit = max(i_size_read(inode), fsc->max_file_size);
 
@@ -1730,7 +1733,12 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 
 	/* FIXME: not complete since it doesn't account for being at quota */
-	if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+	down_read(&osdc->lock);
+	map_flags = osdc->osdmap->flags;
+	pool_flags = ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id);
+	up_read(&osdc->lock);
+	if ((map_flags & CEPH_OSDMAP_FULL) ||
+	    (pool_flags & CEPH_POOL_FLAG_FULL)) {
 		err = -ENOSPC;
 		goto out;
 	}
@@ -1823,7 +1831,8 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 
 	if (written >= 0) {
-		if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+		if ((map_flags & CEPH_OSDMAP_NEARFULL) ||
+		    (pool_flags & CEPH_POOL_FLAG_NEARFULL))
 			iocb->ki_flags |= IOCB_DSYNC;
 		written = generic_write_sync(iocb, written);
 	}
include/linux/ceph/osdmap.h +2 -0
index e081b56f1c1d..651e1b967608 100644
@@ -37,6 +37,7 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 #define CEPH_POOL_FLAG_HASHPSPOOL	(1ULL << 0) /* hash pg seed and pool id
 						       together */
 #define CEPH_POOL_FLAG_FULL		(1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL		(1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
 	struct rb_node node;
@@ -304,5 +305,6 @@ extern struct ceph_pg_pool_info *ceph_pg_pool_by_id(struct ceph_osdmap *map,
 
 extern const char *ceph_pg_pool_name_by_id(struct ceph_osdmap *map, u64 id);
 extern int ceph_pg_poolid_by_name(struct ceph_osdmap *map, const char *name);
+u64 ceph_pg_pool_flags(struct ceph_osdmap *map, u64 id);
 
 #endif
net/ceph/osdmap.c +9 -0
index 4e0de14f80bb..2a6e63a8edbe 100644
@@ -710,6 +710,15 @@ int ceph_pg_poolid_by_name(struct ceph_osdmap *map, const char *name)
 }
 EXPORT_SYMBOL(ceph_pg_poolid_by_name);
 
+u64 ceph_pg_pool_flags(struct ceph_osdmap *map, u64 id)
+{
+	struct ceph_pg_pool_info *pi;
+
+	pi = __lookup_pg_pool(&map->pg_pools, id);
+	return pi ? pi->flags : 0;
+}
+EXPORT_SYMBOL(ceph_pg_pool_flags);
+
 static void __remove_pg_pool(struct rb_root *root, struct ceph_pg_pool_info *pi)
 {
 	rb_erase(&pi->node, root);
author
#2 lore raw
Quoted 168 lines · Ilya Dryomov
Quoted 149 lines · Yanhu Cao

CEPH_OSDMAP_FULL/NEARFULL has been deprecated since mimic, so it does not work well in new versions, added POOL flags to handle it.

---
 fs/ceph/file.c                  |  9 +++++++--
 include/linux/ceph/osd_client.h |  2 ++
 include/linux/ceph/osdmap.h     |  3 ++-
 net/ceph/osd_client.c           | 23 +++++++++++++----------
 4 files changed, 24 insertions(+), 13 deletions(-)
fs/ceph/file.c +7 -2
index 7e0190b1f821..84ec44f9d77a 100644
@@ -1482,7 +1482,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        /* FIXME: not complete since it doesn't account for being at quota */
-       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL) ||
+           pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                               CEPH_POOL_FLAG_FULL)) {
                err = -ENOSPC;
                goto out;
        }
@@ -1575,7 +1577,10 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        if (written >= 0) {
-               if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+               if (ceph_osdmap_flag(&fsc->client->osdc,
+                                       CEPH_OSDMAP_NEARFULL) ||
+                   pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                       CEPH_POOL_FLAG_NEARFULL))
                        iocb->ki_flags |= IOCB_DSYNC;
                written = generic_write_sync(iocb, written);
        }
include/linux/ceph/osd_client.h +2 -0
index 5a62dbd3f4c2..be9007b93862 100644
@@ -375,6 +375,8 @@ static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
        return osdc->osdmap->flags & flag;
 }
 
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag);
+
 extern int ceph_osdc_setup(void);
 extern void ceph_osdc_cleanup(void);
 
include/linux/ceph/osdmap.h +2 -1
index e081b56f1c1d..88faacc11f55 100644
@@ -36,7 +36,8 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 
 #define CEPH_POOL_FLAG_HASHPSPOOL      (1ULL << 0) /* hash pg seed and pool id
                                                       together */
-#define CEPH_POOL_FLAG_FULL            (1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_FULL            (1ULL << 1)  /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL        (1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
        struct rb_node node;
net/ceph/osd_client.c +12 -9
index b68b376d8c2f..9ad2b96c3e78 100644
@@ -1447,9 +1447,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
                atomic_dec(&osd->o_osdc->num_homeless);
 }
 
-static bool __pool_full(struct ceph_pg_pool_info *pi)
+static bool __pool_flag(struct ceph_pg_pool_info *pi, int flag)
 {
-       return pi->flags & CEPH_POOL_FLAG_FULL;
+       return pi->flags & flag;
 }
 
 static bool have_pool_full(struct ceph_osd_client *osdc)
@@ -1460,14 +1460,14 @@ static bool have_pool_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               if (__pool_full(pi))
+               if (__pool_flag(pi, CEPH_POOL_FLAG_FULL))
                        return true;
        }
 
        return false;
 }
 
-static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag)
 {
        struct ceph_pg_pool_info *pi;
 
@@ -1475,8 +1475,10 @@ static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return __pool_full(pi);
+       return __pool_flag(pi, flag);
 }
+EXPORT_SYMBOL(pool_flag);
+
 
 /*
  * Returns whether a request should be blocked from being sent
@@ -1489,7 +1491,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc,
        bool pauserd = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD);
        bool pausewr = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSEWR) ||
                       ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                      __pool_full(pi);
+                      __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 
        WARN_ON(pi->id != t->target_oloc.pool);
        return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) ||
@@ -2320,7 +2322,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
                   !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY |
                                     CEPH_OSD_FLAG_FULL_FORCE)) &&
                   (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                   pool_full(osdc, req->r_t.base_oloc.pool))) {
+                  pool_flag(osdc, req->r_t.base_oloc.pool,
+                            CEPH_POOL_FLAG_FULL))) {
                dout("req %p full/pool_full\n", req);
                if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) {
                        err = -ENOSPC;
@@ -2539,7 +2542,7 @@ static int abort_on_full_fn(struct ceph_osd_request *req, void *arg)
 
        if ((req->r_flags & CEPH_OSD_FLAG_WRITE) &&
            (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-            pool_full(osdc, req->r_t.base_oloc.pool))) {
+            pool_flag(osdc, req->r_t.base_oloc.pool, CEPH_POOL_FLAG_FULL))) {
                if (!*victims) {
                        update_epoch_barrier(osdc, osdc->osdmap->epoch);
                        *victims = true;
@@ -3707,7 +3710,7 @@ static void set_pool_was_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               pi->was_full = __pool_full(pi);
+               pi->was_full = __pool_flag(pi, CEPH_POOL_FLAG_FULL);
        }
 }
 

net/ceph/osd_client.c static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id) · line 3728

Quoted 8 lines
net/ceph/osd_client.c +1 -1
@@ -3719,7 +3722,7 @@ static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return pi->was_full && !__pool_full(pi);
+       return pi->was_full && !__pool_flag(pi, CEPH_POOL_FLAG_FULL);
 }
 
 static enum calc_target_result

Hi Yanhu,

Sorry for a late reply.

This adds some unnecessary churn and also exposes a helper that must be called under osdc->lock without making that obvious. How about the attached instead?

Quoted 6 lines

ceph_pg_pool_flags() takes osdmap instead of osdc, making it clear that the caller is resposibile for keeping the map stable.

Thanks,

Ilya

net/ceph/osdmap.c
--------------------------
bool ceph_pg_pool_flags(struct ceph_osdmap *map, s64 pool_id, int flag)

{
        struct ceph_pg_pool_info *pi;

        /* CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL deprecated since mimic */
        if (flag & (CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_NEARFULL))
                if (map->flags & (CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL))
                        return true;

        pi = ceph_pg_pool_by_id(map, pool_id);
        if (!pi)
                return false;

        return pi->flags & flag;
}

fs/ceph/file.c
-----------------
ceph_write_iter() {
...

        down_read(&osdc->lock);
        if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                               CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_FULL_QUOTA)) {
                err = -ENOSPC;
                up_read(&osdc->lock);
                goto out;
        }
        up_read(&osdc->lock);
...
         down_read(&osdc->lock);
          if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                                        CEPH_POOL_FLAG_NEARFULL))
              iocb->ki_flags |= IOCB_DSYNC;
          up_read(&osdc->lock);

...
}

how about this?

Thanks.

maintainer
#3 lore raw
Quoted 213 lines · Yanhu Cao
Quoted 168 lines · Ilya Dryomov
Quoted 149 lines · Yanhu Cao

CEPH_OSDMAP_FULL/NEARFULL has been deprecated since mimic, so it does not work well in new versions, added POOL flags to handle it.

---
 fs/ceph/file.c                  |  9 +++++++--
 include/linux/ceph/osd_client.h |  2 ++
 include/linux/ceph/osdmap.h     |  3 ++-
 net/ceph/osd_client.c           | 23 +++++++++++++----------
 4 files changed, 24 insertions(+), 13 deletions(-)
fs/ceph/file.c +7 -2
index 7e0190b1f821..84ec44f9d77a 100644
@@ -1482,7 +1482,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        /* FIXME: not complete since it doesn't account for being at quota */
-       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL) ||
+           pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                               CEPH_POOL_FLAG_FULL)) {
                err = -ENOSPC;
                goto out;
        }
@@ -1575,7 +1577,10 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        if (written >= 0) {
-               if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+               if (ceph_osdmap_flag(&fsc->client->osdc,
+                                       CEPH_OSDMAP_NEARFULL) ||
+                   pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                       CEPH_POOL_FLAG_NEARFULL))
                        iocb->ki_flags |= IOCB_DSYNC;
                written = generic_write_sync(iocb, written);
        }
include/linux/ceph/osd_client.h +2 -0
index 5a62dbd3f4c2..be9007b93862 100644
@@ -375,6 +375,8 @@ static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
        return osdc->osdmap->flags & flag;
 }
 
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag);
+
 extern int ceph_osdc_setup(void);
 extern void ceph_osdc_cleanup(void);
 
include/linux/ceph/osdmap.h +2 -1
index e081b56f1c1d..88faacc11f55 100644
@@ -36,7 +36,8 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 
 #define CEPH_POOL_FLAG_HASHPSPOOL      (1ULL << 0) /* hash pg seed and pool id
                                                       together */
-#define CEPH_POOL_FLAG_FULL            (1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_FULL            (1ULL << 1)  /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL        (1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
        struct rb_node node;
net/ceph/osd_client.c +12 -9
index b68b376d8c2f..9ad2b96c3e78 100644
@@ -1447,9 +1447,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
                atomic_dec(&osd->o_osdc->num_homeless);
 }
 
-static bool __pool_full(struct ceph_pg_pool_info *pi)
+static bool __pool_flag(struct ceph_pg_pool_info *pi, int flag)
 {
-       return pi->flags & CEPH_POOL_FLAG_FULL;
+       return pi->flags & flag;
 }
 
 static bool have_pool_full(struct ceph_osd_client *osdc)
@@ -1460,14 +1460,14 @@ static bool have_pool_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               if (__pool_full(pi))
+               if (__pool_flag(pi, CEPH_POOL_FLAG_FULL))
                        return true;
        }
 
        return false;
 }
 
-static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag)
 {
        struct ceph_pg_pool_info *pi;
 
@@ -1475,8 +1475,10 @@ static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return __pool_full(pi);
+       return __pool_flag(pi, flag);
 }
+EXPORT_SYMBOL(pool_flag);
+
 
 /*
  * Returns whether a request should be blocked from being sent
@@ -1489,7 +1491,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc,
        bool pauserd = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD);
        bool pausewr = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSEWR) ||
                       ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                      __pool_full(pi);
+                      __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 
        WARN_ON(pi->id != t->target_oloc.pool);
        return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) ||
@@ -2320,7 +2322,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
                   !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY |
                                     CEPH_OSD_FLAG_FULL_FORCE)) &&
                   (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                   pool_full(osdc, req->r_t.base_oloc.pool))) {
+                  pool_flag(osdc, req->r_t.base_oloc.pool,
+                            CEPH_POOL_FLAG_FULL))) {
                dout("req %p full/pool_full\n", req);
                if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) {
                        err = -ENOSPC;
@@ -2539,7 +2542,7 @@ static int abort_on_full_fn(struct ceph_osd_request *req, void *arg)
 
        if ((req->r_flags & CEPH_OSD_FLAG_WRITE) &&
            (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-            pool_full(osdc, req->r_t.base_oloc.pool))) {
+            pool_flag(osdc, req->r_t.base_oloc.pool, CEPH_POOL_FLAG_FULL))) {
                if (!*victims) {
                        update_epoch_barrier(osdc, osdc->osdmap->epoch);
                        *victims = true;
@@ -3707,7 +3710,7 @@ static void set_pool_was_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               pi->was_full = __pool_full(pi);
+               pi->was_full = __pool_flag(pi, CEPH_POOL_FLAG_FULL);
        }
 }
 

net/ceph/osd_client.c static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id) · line 3728

Quoted 8 lines
net/ceph/osd_client.c +1 -1
@@ -3719,7 +3722,7 @@ static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return pi->was_full && !__pool_full(pi);
+       return pi->was_full && !__pool_flag(pi, CEPH_POOL_FLAG_FULL);
 }
 
 static enum calc_target_result

Hi Yanhu,

Sorry for a late reply.

This adds some unnecessary churn and also exposes a helper that must be called under osdc->lock without making that obvious. How about the attached instead?

Quoted 6 lines

ceph_pg_pool_flags() takes osdmap instead of osdc, making it clear that the caller is resposibile for keeping the map stable.

Thanks,

Ilya

net/ceph/osdmap.c
--------------------------
bool ceph_pg_pool_flags(struct ceph_osdmap *map, s64 pool_id, int flag)

{
        struct ceph_pg_pool_info *pi;

        /* CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL deprecated since mimic */
        if (flag & (CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_NEARFULL))
                if (map->flags & (CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL))
                        return true;

        pi = ceph_pg_pool_by_id(map, pool_id);
        if (!pi)
                return false;

        return pi->flags & flag;
}

fs/ceph/file.c
-----------------
ceph_write_iter() {
...

        down_read(&osdc->lock);
        if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                               CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_FULL_QUOTA)) {
                err = -ENOSPC;
                up_read(&osdc->lock);
                goto out;
        }
        up_read(&osdc->lock);
...
         down_read(&osdc->lock);
          if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                                        CEPH_POOL_FLAG_NEARFULL))
              iocb->ki_flags |= IOCB_DSYNC;
          up_read(&osdc->lock);
Quoted 4 lines

...
}

how about this?

Well, this takes osdc->lock and looks up ceph_pg_pool_info twice. Given that these checks are inherently racy, I think doing it once at the top makes more sense.

Also, I don't think this does what you intended it to do. Your ceph_pg_pool_flags(..., CEPH_POOL_FLAG_FULL) returns true even if the map only has CEPH_OSDMAP_NEARFULL, triggering early ENOSPC.

Checking CEPH_POOL_FLAG_FULL_QUOTA is not necessary, because it is set together with CEPH_POOL_FLAG_FULL:

src/osd/osd_types.h:

1199 FLAG_FULL_QUOTA = 1<<10, // pool is currently running out of
quota, will set FLAG_FULL too

Thanks,

Ilya

author
#4 lore raw
Quoted 218 lines · Ilya Dryomov
Quoted 216 lines · Yanhu Cao
Quoted 168 lines · Ilya Dryomov
Quoted 149 lines · Yanhu Cao

CEPH_OSDMAP_FULL/NEARFULL has been deprecated since mimic, so it does not work well in new versions, added POOL flags to handle it.

---
 fs/ceph/file.c                  |  9 +++++++--
 include/linux/ceph/osd_client.h |  2 ++
 include/linux/ceph/osdmap.h     |  3 ++-
 net/ceph/osd_client.c           | 23 +++++++++++++----------
 4 files changed, 24 insertions(+), 13 deletions(-)
fs/ceph/file.c +7 -2
index 7e0190b1f821..84ec44f9d77a 100644
@@ -1482,7 +1482,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        /* FIXME: not complete since it doesn't account for being at quota */
-       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL) ||
+           pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                               CEPH_POOL_FLAG_FULL)) {
                err = -ENOSPC;
                goto out;
        }
@@ -1575,7 +1577,10 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        if (written >= 0) {
-               if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+               if (ceph_osdmap_flag(&fsc->client->osdc,
+                                       CEPH_OSDMAP_NEARFULL) ||
+                   pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                       CEPH_POOL_FLAG_NEARFULL))
                        iocb->ki_flags |= IOCB_DSYNC;
                written = generic_write_sync(iocb, written);
        }
include/linux/ceph/osd_client.h +2 -0
index 5a62dbd3f4c2..be9007b93862 100644
@@ -375,6 +375,8 @@ static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
        return osdc->osdmap->flags & flag;
 }
 
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag);
+
 extern int ceph_osdc_setup(void);
 extern void ceph_osdc_cleanup(void);
 
include/linux/ceph/osdmap.h +2 -1
index e081b56f1c1d..88faacc11f55 100644
@@ -36,7 +36,8 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 
 #define CEPH_POOL_FLAG_HASHPSPOOL      (1ULL << 0) /* hash pg seed and pool id
                                                       together */
-#define CEPH_POOL_FLAG_FULL            (1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_FULL            (1ULL << 1)  /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL        (1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
        struct rb_node node;
net/ceph/osd_client.c +12 -9
index b68b376d8c2f..9ad2b96c3e78 100644
@@ -1447,9 +1447,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
                atomic_dec(&osd->o_osdc->num_homeless);
 }
 
-static bool __pool_full(struct ceph_pg_pool_info *pi)
+static bool __pool_flag(struct ceph_pg_pool_info *pi, int flag)
 {
-       return pi->flags & CEPH_POOL_FLAG_FULL;
+       return pi->flags & flag;
 }
 
 static bool have_pool_full(struct ceph_osd_client *osdc)
@@ -1460,14 +1460,14 @@ static bool have_pool_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               if (__pool_full(pi))
+               if (__pool_flag(pi, CEPH_POOL_FLAG_FULL))
                        return true;
        }
 
        return false;
 }
 
-static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag)
 {
        struct ceph_pg_pool_info *pi;
 
@@ -1475,8 +1475,10 @@ static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return __pool_full(pi);
+       return __pool_flag(pi, flag);
 }
+EXPORT_SYMBOL(pool_flag);
+
 
 /*
  * Returns whether a request should be blocked from being sent
@@ -1489,7 +1491,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc,
        bool pauserd = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD);
        bool pausewr = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSEWR) ||
                       ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                      __pool_full(pi);
+                      __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 
        WARN_ON(pi->id != t->target_oloc.pool);
        return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) ||
@@ -2320,7 +2322,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
                   !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY |
                                     CEPH_OSD_FLAG_FULL_FORCE)) &&
                   (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                   pool_full(osdc, req->r_t.base_oloc.pool))) {
+                  pool_flag(osdc, req->r_t.base_oloc.pool,
+                            CEPH_POOL_FLAG_FULL))) {
                dout("req %p full/pool_full\n", req);
                if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) {
                        err = -ENOSPC;
@@ -2539,7 +2542,7 @@ static int abort_on_full_fn(struct ceph_osd_request *req, void *arg)
 
        if ((req->r_flags & CEPH_OSD_FLAG_WRITE) &&
            (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-            pool_full(osdc, req->r_t.base_oloc.pool))) {
+            pool_flag(osdc, req->r_t.base_oloc.pool, CEPH_POOL_FLAG_FULL))) {
                if (!*victims) {
                        update_epoch_barrier(osdc, osdc->osdmap->epoch);
                        *victims = true;
@@ -3707,7 +3710,7 @@ static void set_pool_was_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               pi->was_full = __pool_full(pi);
+               pi->was_full = __pool_flag(pi, CEPH_POOL_FLAG_FULL);
        }
 }
 

net/ceph/osd_client.c static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id) · line 3728

Quoted 8 lines
net/ceph/osd_client.c +1 -1
@@ -3719,7 +3722,7 @@ static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return pi->was_full && !__pool_full(pi);
+       return pi->was_full && !__pool_flag(pi, CEPH_POOL_FLAG_FULL);
 }
 
 static enum calc_target_result

Hi Yanhu,

Sorry for a late reply.

This adds some unnecessary churn and also exposes a helper that must be called under osdc->lock without making that obvious. How about the attached instead?

Quoted 6 lines

ceph_pg_pool_flags() takes osdmap instead of osdc, making it clear that the caller is resposibile for keeping the map stable.

Thanks,

Ilya

net/ceph/osdmap.c
--------------------------
bool ceph_pg_pool_flags(struct ceph_osdmap *map, s64 pool_id, int flag)

{
        struct ceph_pg_pool_info *pi;

        /* CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL deprecated since mimic */
        if (flag & (CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_NEARFULL))
                if (map->flags & (CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL))
                        return true;

        pi = ceph_pg_pool_by_id(map, pool_id);
        if (!pi)
                return false;

        return pi->flags & flag;
}

fs/ceph/file.c
-----------------
ceph_write_iter() {
...

        down_read(&osdc->lock);
        if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                               CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_FULL_QUOTA)) {
                err = -ENOSPC;
                up_read(&osdc->lock);
                goto out;
        }
        up_read(&osdc->lock);
...
         down_read(&osdc->lock);
          if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                                        CEPH_POOL_FLAG_NEARFULL))
              iocb->ki_flags |= IOCB_DSYNC;
          up_read(&osdc->lock);

...
}

Quoted 5 lines
Quoted 1 line

how about this?

Well, this takes osdc->lock and looks up ceph_pg_pool_info twice. Given that these checks are inherently racy, I think doing it once at the top makes more sense.

be modified as follow.

ceph_write_iter() {
...

        down_read(&osdc->lock);
        pi = ceph_pg_pool_by_id(osdc->osdmap, ci->i_layout.pool_id);
        if (!pi) {
                err = -ENOENT;
                up_read(&osdc->lock);
                goto out;
        }
        up_read(&osdc->lock);

...
}

Quoted 5 lines

Also, I don't think this does what you intended it to do. Your ceph_pg_pool_flags(..., CEPH_POOL_FLAG_FULL) returns true even if the map only has CEPH_OSDMAP_NEARFULL, triggering early ENOSPC.

Ah... my mistake. According to OSDMAP/POOL flag to do respectively.

        if (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
            ceph_pg_pool_flags(pi, CEPH_POOL_FLAG_FULL)) {
                err = -ENOSPC;
                goto out;
        }

include/linux/ceph/osdmap.h
--------------------------------------
static inline bool ceph_pg_pool_flags(struct ceph_pg_pool_info *pi, int flag)

{
        return pi->flags & flag;
}

Thanks.

maintainer
#5 lore raw
Quoted 238 lines · Yanhu Cao
Quoted 218 lines · Ilya Dryomov
Quoted 216 lines · Yanhu Cao
Quoted 168 lines · Ilya Dryomov
Quoted 149 lines · Yanhu Cao

CEPH_OSDMAP_FULL/NEARFULL has been deprecated since mimic, so it does not work well in new versions, added POOL flags to handle it.

---
 fs/ceph/file.c                  |  9 +++++++--
 include/linux/ceph/osd_client.h |  2 ++
 include/linux/ceph/osdmap.h     |  3 ++-
 net/ceph/osd_client.c           | 23 +++++++++++++----------
 4 files changed, 24 insertions(+), 13 deletions(-)
fs/ceph/file.c +7 -2
index 7e0190b1f821..84ec44f9d77a 100644
@@ -1482,7 +1482,9 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        /* FIXME: not complete since it doesn't account for being at quota */
-       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL)) {
+       if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_FULL) ||
+           pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                               CEPH_POOL_FLAG_FULL)) {
                err = -ENOSPC;
                goto out;
        }
@@ -1575,7 +1577,10 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
        }
 
        if (written >= 0) {
-               if (ceph_osdmap_flag(&fsc->client->osdc, CEPH_OSDMAP_NEARFULL))
+               if (ceph_osdmap_flag(&fsc->client->osdc,
+                                       CEPH_OSDMAP_NEARFULL) ||
+                   pool_flag(&fsc->client->osdc, ci->i_layout.pool_id,
+                                       CEPH_POOL_FLAG_NEARFULL))
                        iocb->ki_flags |= IOCB_DSYNC;
                written = generic_write_sync(iocb, written);
        }
include/linux/ceph/osd_client.h +2 -0
index 5a62dbd3f4c2..be9007b93862 100644
@@ -375,6 +375,8 @@ static inline bool ceph_osdmap_flag(struct ceph_osd_client *osdc, int flag)
        return osdc->osdmap->flags & flag;
 }
 
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag);
+
 extern int ceph_osdc_setup(void);
 extern void ceph_osdc_cleanup(void);
 
include/linux/ceph/osdmap.h +2 -1
index e081b56f1c1d..88faacc11f55 100644
@@ -36,7 +36,8 @@ int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs);
 
 #define CEPH_POOL_FLAG_HASHPSPOOL      (1ULL << 0) /* hash pg seed and pool id
                                                       together */
-#define CEPH_POOL_FLAG_FULL            (1ULL << 1) /* pool is full */
+#define CEPH_POOL_FLAG_FULL            (1ULL << 1)  /* pool is full */
+#define CEPH_POOL_FLAG_NEARFULL        (1ULL << 11) /* pool is nearfull */
 
 struct ceph_pg_pool_info {
        struct rb_node node;
net/ceph/osd_client.c +12 -9
index b68b376d8c2f..9ad2b96c3e78 100644
@@ -1447,9 +1447,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
                atomic_dec(&osd->o_osdc->num_homeless);
 }
 
-static bool __pool_full(struct ceph_pg_pool_info *pi)
+static bool __pool_flag(struct ceph_pg_pool_info *pi, int flag)
 {
-       return pi->flags & CEPH_POOL_FLAG_FULL;
+       return pi->flags & flag;
 }
 
 static bool have_pool_full(struct ceph_osd_client *osdc)
@@ -1460,14 +1460,14 @@ static bool have_pool_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               if (__pool_full(pi))
+               if (__pool_flag(pi, CEPH_POOL_FLAG_FULL))
                        return true;
        }
 
        return false;
 }
 
-static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
+bool pool_flag(struct ceph_osd_client *osdc, s64 pool_id, int flag)
 {
        struct ceph_pg_pool_info *pi;
 
@@ -1475,8 +1475,10 @@ static bool pool_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return __pool_full(pi);
+       return __pool_flag(pi, flag);
 }
+EXPORT_SYMBOL(pool_flag);
+
 
 /*
  * Returns whether a request should be blocked from being sent
@@ -1489,7 +1491,7 @@ static bool target_should_be_paused(struct ceph_osd_client *osdc,
        bool pauserd = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSERD);
        bool pausewr = ceph_osdmap_flag(osdc, CEPH_OSDMAP_PAUSEWR) ||
                       ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                      __pool_full(pi);
+                      __pool_flag(pi, CEPH_POOL_FLAG_FULL);
 
        WARN_ON(pi->id != t->target_oloc.pool);
        return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) ||
@@ -2320,7 +2322,8 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
                   !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY |
                                     CEPH_OSD_FLAG_FULL_FORCE)) &&
                   (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-                   pool_full(osdc, req->r_t.base_oloc.pool))) {
+                  pool_flag(osdc, req->r_t.base_oloc.pool,
+                            CEPH_POOL_FLAG_FULL))) {
                dout("req %p full/pool_full\n", req);
                if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) {
                        err = -ENOSPC;
@@ -2539,7 +2542,7 @@ static int abort_on_full_fn(struct ceph_osd_request *req, void *arg)
 
        if ((req->r_flags & CEPH_OSD_FLAG_WRITE) &&
            (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
-            pool_full(osdc, req->r_t.base_oloc.pool))) {
+            pool_flag(osdc, req->r_t.base_oloc.pool, CEPH_POOL_FLAG_FULL))) {
                if (!*victims) {
                        update_epoch_barrier(osdc, osdc->osdmap->epoch);
                        *victims = true;
@@ -3707,7 +3710,7 @@ static void set_pool_was_full(struct ceph_osd_client *osdc)
                struct ceph_pg_pool_info *pi =
                    rb_entry(n, struct ceph_pg_pool_info, node);
 
-               pi->was_full = __pool_full(pi);
+               pi->was_full = __pool_flag(pi, CEPH_POOL_FLAG_FULL);
        }
 }
 

net/ceph/osd_client.c static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id) · line 3728

Quoted 8 lines
net/ceph/osd_client.c +1 -1
@@ -3719,7 +3722,7 @@ static bool pool_cleared_full(struct ceph_osd_client *osdc, s64 pool_id)
        if (!pi)
                return false;
 
-       return pi->was_full && !__pool_full(pi);
+       return pi->was_full && !__pool_flag(pi, CEPH_POOL_FLAG_FULL);
 }
 
 static enum calc_target_result

Hi Yanhu,

Sorry for a late reply.

This adds some unnecessary churn and also exposes a helper that must be called under osdc->lock without making that obvious. How about the attached instead?

Quoted 6 lines

ceph_pg_pool_flags() takes osdmap instead of osdc, making it clear that the caller is resposibile for keeping the map stable.

Thanks,

Ilya

net/ceph/osdmap.c
--------------------------
bool ceph_pg_pool_flags(struct ceph_osdmap *map, s64 pool_id, int flag)

{
        struct ceph_pg_pool_info *pi;

        /* CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL deprecated since mimic */
        if (flag & (CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_NEARFULL))
                if (map->flags & (CEPH_OSDMAP_FULL|CEPH_OSDMAP_NEARFULL))
                        return true;

        pi = ceph_pg_pool_by_id(map, pool_id);
        if (!pi)
                return false;

        return pi->flags & flag;
}

fs/ceph/file.c
-----------------
ceph_write_iter() {
...

        down_read(&osdc->lock);
        if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                               CEPH_POOL_FLAG_FULL|CEPH_POOL_FLAG_FULL_QUOTA)) {
                err = -ENOSPC;
                up_read(&osdc->lock);
                goto out;
        }
        up_read(&osdc->lock);
...
         down_read(&osdc->lock);
          if (ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id,
                                        CEPH_POOL_FLAG_NEARFULL))
              iocb->ki_flags |= IOCB_DSYNC;
          up_read(&osdc->lock);

...
}

Quoted 5 lines
Quoted 1 line

how about this?

Well, this takes osdc->lock and looks up ceph_pg_pool_info twice. Given that these checks are inherently racy, I think doing it once at the top makes more sense.

be modified as follow.

ceph_write_iter() {
...

        down_read(&osdc->lock);
        pi = ceph_pg_pool_by_id(osdc->osdmap, ci->i_layout.pool_id);
        if (!pi) {
                err = -ENOENT;
                up_read(&osdc->lock);
                goto out;
        }
        up_read(&osdc->lock);
Quoted 2 lines

...
}

No, this won't work because as soon as you release osdc->lock, pi may get invalidated.

Quoted 11 lines
Quoted 6 lines

Also, I don't think this does what you intended it to do. Your ceph_pg_pool_flags(..., CEPH_POOL_FLAG_FULL) returns true even if the map only has CEPH_OSDMAP_NEARFULL, triggering early ENOSPC.

Ah... my mistake. According to OSDMAP/POOL flag to do respectively.

        if (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) ||
            ceph_pg_pool_flags(pi, CEPH_POOL_FLAG_FULL)) {

... and you may reference invalid memory here.

Quoted 10 lines
                err = -ENOSPC;
                goto out;
        }

include/linux/ceph/osdmap.h
--------------------------------------
static inline bool ceph_pg_pool_flags(struct ceph_pg_pool_info *pi, int flag)

{
        return pi->flags & flag;
}

I'll go ahead with my version.

Thanks,

Ilya