ceph: don't request caps for idle open files
Subsystems (2)
v5 cover ceph: don't request caps for idle open files
This series make cephfs client not request caps for open files that idle for a long time. For the case that one active client and multiple standby clients open the same file, this increase the possibility that mds issues exclusive caps to the active client.
Yan, Zheng (7):
ceph: always renew caps if mds_wanted is insufficient
ceph: consider inode's last read/write when calculating wanted caps
ceph: remove delay check logic from ceph_check_caps()
ceph: simplify calling of ceph_get_fmode()
ceph: update i_requested_max_size only when sending cap msg to auth mds
ceph: check all mds' caps after page writeback
ceph: calculate dir's wanted caps according to recent dirops
fs/ceph/caps.c | 360 ++++++++++++++++------------------- fs/ceph/dir.c | 21 +- fs/ceph/file.c | 45 ++--- fs/ceph/inode.c | 21 +- fs/ceph/ioctl.c | 2 + fs/ceph/mds_client.c | 16 +- fs/ceph/super.h | 37 ++-- include/linux/ceph/ceph_fs.h | 1 + 8 files changed, 243 insertions(+), 260 deletions(-)
changes since v2
- make __ceph_caps_file_wanted() more readable
- add patch 5 and 6, which fix hung write during testing patch 1~4
changes since v3
- don't queue delayed cap check for snap inode
- initialize ci->{last_rd,last_wr} to jiffies - 3600 * HZ
- make __ceph_caps_file_wanted() check inode type
changes since v4
- add patch 7, improve how to calculate dir's wanted caps
Signature · 1 line
2.21.1
PATCH v5 1/7 ceph: always renew caps if mds_wanted is insufficient
Original code only renews caps for inodes with CEPH_I_CAP_DROPPED flag, which indicates that mds has closed the session and caps were dropped. Remove this flag in preparation for not requesting caps for idle open files.
--- fs/ceph/caps.c | 36 +++++++++++++++--------------------- fs/ceph/mds_client.c | 5 ----- fs/ceph/super.h | 13 ++++++------- 3 files changed, 21 insertions(+), 33 deletions(-)
fs/ceph/caps.c +15 -21
@@ -2686,6 +2686,7 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, } }else {int session_readonly = false; +int mds_wanted;if (ci->i_auth_cap && (need & (CEPH_CAP_FILE_WR |CEPH_CAP_FILE_EXCL ))) {struct ceph_mds_session *s = ci->i_auth_cap->session;
@@ -2694,32 +2695,27 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,spin_unlock (&s->s_cap_lock); }if (session_readonly) { -dout ("get_cap_refs %p needed %s but mds%d readonly\n" , +dout ("get_cap_refs %p need %s but mds%d readonly\n" , inode,ceph_cap_string (need), ci->i_auth_cap->mds); ret = -EROFS ; goto out_unlock; } -if (ci->i_ceph_flags &CEPH_I_CAP_DROPPED ) { -int mds_wanted; -if (READ_ONCE (mdsc->fsc->mount_state) == -CEPH_MOUNT_SHUTDOWN ) { -dout ("get_cap_refs %p forced umount\n" , inode); - ret = -EIO ; - goto out_unlock; - } - mds_wanted =__ceph_caps_mds_wanted (ci, false); -if (need & ~(mds_wanted & need)) { -dout ("get_cap_refs %p caps were dropped" -" (session killed?)\n" , inode); - ret = -ESTALE ; - goto out_unlock; - } -if (!(file_wanted & ~mds_wanted)) - ci->i_ceph_flags &= ~CEPH_I_CAP_DROPPED ; +if (READ_ONCE (mdsc->fsc->mount_state) ==CEPH_MOUNT_SHUTDOWN ) { +dout ("get_cap_refs %p forced umount\n" , inode); + ret = -EIO ; + goto out_unlock; + } + mds_wanted =__ceph_caps_mds_wanted (ci, false); +if (need & ~mds_wanted) { +dout ("get_cap_refs %p need %s > mds_wanted %s\n" , + inode,ceph_cap_string (need), +ceph_cap_string (mds_wanted)); + ret = -ESTALE ; + goto out_unlock; } -dout ("get_cap_refs %p have %s needed %s\n" , inode, +dout ("get_cap_refs %p have %s need %s\n" , inode,ceph_cap_string (have),ceph_cap_string (need)); } out_unlock:
@@ -3678,8 +3674,6 @@ static void handle_cap_export(struct inode *inode, struct ceph_mds_caps *ex, goto out_unlock;if (target <0 ) { -if (cap->mds_wanted | cap->issued) - ci->i_ceph_flags |=CEPH_I_CAP_DROPPED ;__ceph_remove_cap (cap, false); goto out_unlock; }
fs/ceph/mds_client.c +0 -5
@@ -1514,8 +1514,6 @@ static int remove_session_caps_cb(struct inode *inode, struct ceph_cap *cap,dout ("removing cap %p, ci is %p, inode is %p\n" , cap, ci, &ci->vfs_inode);spin_lock (&ci->i_ceph_lock); -if (cap->mds_wanted | cap->issued) - ci->i_ceph_flags |=CEPH_I_CAP_DROPPED ;__ceph_remove_cap (cap, false);if (!ci->i_auth_cap) {struct ceph_cap_flush *cf;
@@ -1681,9 +1679,6 @@ static int wake_up_session_cb(struct inode *inode, struct ceph_cap *cap,/* mds did not re-issue stale cap */ spin_lock (&ci->i_ceph_lock); cap->issued = cap->implemented =CEPH_CAP_PIN ; -/* make sure mds knows what we want */ -if (__ceph_caps_file_wanted (ci) & ~cap->mds_wanted) - ci->i_ceph_flags |=CEPH_I_CAP_DROPPED ;spin_unlock (&ci->i_ceph_lock); } }else if (ev ==FORCE_RO ) {
fs/ceph/super.h +6 -7
@@ -522,13 +522,12 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,#define CEPH_I_POOL_RD (1 << 4)/* can read from pool */ #define CEPH_I_POOL_WR (1 << 5)/* can write to pool */ #define CEPH_I_SEC_INITED (1 << 6)/* security initialized */ -#define CEPH_I_CAP_DROPPED (1 << 7)/* caps were forcibly dropped */ -#define CEPH_I_KICK_FLUSH (1 << 8)/* kick flushing caps */ -#define CEPH_I_FLUSH_SNAPS (1 << 9)/* need flush snapss */ -#define CEPH_I_ERROR_WRITE (1 << 10)/* have seen write errors */ -#define CEPH_I_ERROR_FILELOCK (1 << 11)/* have seen file lock errors */ -#define CEPH_I_ODIRECT (1 << 12)/* inode in direct I/O mode */ -#define CEPH_ASYNC_CREATE_BIT (13)/* async create in flight for this */ +#define CEPH_I_KICK_FLUSH (1 << 7)/* kick flushing caps */ +#define CEPH_I_FLUSH_SNAPS (1 << 8)/* need flush snapss */ +#define CEPH_I_ERROR_WRITE (1 << 9)/* have seen write errors */ +#define CEPH_I_ERROR_FILELOCK (1 << 10)/* have seen file lock errors */ +#define CEPH_I_ODIRECT (1 << 11)/* inode in direct I/O mode */ +#define CEPH_ASYNC_CREATE_BIT (12)/* async create in flight for this */ #define CEPH_I_ASYNC_CREATE (1 << CEPH_ASYNC_CREATE_BIT)/*
Signature · 1 line
2.21.1
PATCH v5 2/7 ceph: consider inode's last read/write when calculating wanted caps
In reply to Yan, ZhengAdd i_last_rd and i_last_wr to ceph_inode_info. These fields are used to track the last time the client acquired read/write caps for the inode.
If there is no read/write on an inode for 'caps_wanted_delay_max' seconds, __ceph_caps_file_wanted() does not request caps for read/write even there are open files.
--- fs/ceph/caps.c | 163 ++++++++++++++++++++++++----------- fs/ceph/file.c | 21 +++-- fs/ceph/inode.c | 10 ++- fs/ceph/ioctl.c | 2 + fs/ceph/super.h | 13 ++- include/linux/ceph/ceph_fs.h | 1 + 6 files changed, 147 insertions(+), 63 deletions(-)
fs/ceph/caps.c +112 -51
@@ -978,19 +978,51 @@ int __ceph_caps_used(struct ceph_inode_info *ci)return used; } +#define FMODE_WAIT_BIAS 1000 +/* * wanted, by virtue of open file modes */ int __ceph_caps_file_wanted (struct ceph_inode_info *ci) { -int i, bits =0 ; -for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) { -if (ci->i_nr_by_mode[i]) - bits |=1 << i; +const int PIN_SHIFT =ffs (CEPH_FILE_MODE_PIN ); +const int RD_SHIFT =ffs (CEPH_FILE_MODE_RD ); +const int WR_SHIFT =ffs (CEPH_FILE_MODE_WR ); +const int LAZY_SHIFT =ffs (CEPH_FILE_MODE_LAZY ); +struct ceph_mount_options *opt = +ceph_inode_to_client (&ci->vfs_inode)->mount_options; +unsigned long used_cutoff = jiffies - opt->caps_wanted_delay_max *HZ ; +unsigned long idle_cutoff = jiffies - opt->caps_wanted_delay_min *HZ ; +int bits =0 ; + +if (S_ISDIR (ci->vfs_inode.i_mode)) { +if (ci->i_nr_by_mode[PIN_SHIFT ] >0 ) + bits |=1 <<PIN_SHIFT ; + }else { +if (ci->i_nr_by_mode[RD_SHIFT ] >0 ) { +if (ci->i_nr_by_mode[RD_SHIFT ] >=FMODE_WAIT_BIAS || +time_after (ci->i_last_rd, used_cutoff)) + bits |=1 <<RD_SHIFT ; + }else if (time_after (ci->i_last_rd, idle_cutoff)) { + bits |=1 <<RD_SHIFT ; + } + +if (ci->i_nr_by_mode[WR_SHIFT ] >0 ) { +if (ci->i_nr_by_mode[WR_SHIFT ] >=FMODE_WAIT_BIAS || +time_after (ci->i_last_wr, used_cutoff)) + bits |=1 <<WR_SHIFT ; + }else if (time_after (ci->i_last_wr, idle_cutoff)) { + bits |=1 <<WR_SHIFT ; + } + +/* check lazyio only when read/write is wanted */ +if ((bits & (CEPH_FILE_MODE_RDWR <<1 )) && + ci->i_nr_by_mode[LAZY_SHIFT ] >0 ) + bits |=1 <<LAZY_SHIFT ; + } -if (bits ==0 ) -return 0 ; -return ceph_caps_for_mode (bits >>1 ); + +return bits ?ceph_caps_for_mode (bits >>1 ) :0 ; }/*
@@ -1032,14 +1064,6 @@ int __ceph_caps_mds_wanted(struct ceph_inode_info *ci, bool check)return mds_wanted; } -/* -* called under i_ceph_lock -*/ -static int __ceph_is_single_caps (struct ceph_inode_info *ci) -{ -return rb_first (&ci->i_caps) ==rb_last (&ci->i_caps); -} -int ceph_is_any_caps (struct inode *inode) {struct ceph_inode_info *ci =ceph_inode (inode);
@@ -1877,10 +1901,6 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,if (ci->i_ceph_flags &CEPH_I_FLUSH ) flags |=CHECK_CAPS_FLUSH ; -if (!(flags &CHECK_CAPS_AUTHONLY ) || - (ci->i_auth_cap &&__ceph_is_single_caps (ci))) -__cap_delay_cancel (mdsc, ci); - goto retry_locked; retry:spin_lock (&ci->i_ceph_lock);
@@ -2105,9 +2125,17 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags, goto retry;/* retake i_ceph_lock and restart our cap scan. */ } -/* Reschedule delayed caps release if we delayed anything */ -if (delayed) -__cap_delay_requeue (mdsc, ci, false); +if (list_empty (&ci->i_cap_delay_list)) { +if (delayed) { +/* Reschedule delayed caps release if we delayed anything */ +__cap_delay_requeue (mdsc, ci, false); + }else if (__ceph_is_any_real_caps (ci) && + (file_wanted & ~CEPH_CAP_PIN ) && + !(used & (CEPH_CAP_FILE_RD |CEPH_CAP_ANY_FILE_WR ))) { +/* periodically re-calculate caps wanted by open files */ +__cap_delay_requeue (mdsc, ci, true); + } + }spin_unlock (&ci->i_ceph_lock);
@@ -2573,8 +2601,9 @@ void ceph_take_cap_refs(struct ceph_inode_info *ci, int got,* FIXME: how does a 0 return differ from -EAGAIN? */ enum { -NON_BLOCKING =1 , -CHECK_FILELOCK =2 , +/* first 8 bits are reserved for CEPH_FILE_MODE_FOO */ +NON_BLOCKING = (1 <<8 ), +CHECK_FILELOCK = (1 <<9 ), }; static int try_get_cap_refs(struct inode *inode, int need, int want,
@@ -2584,7 +2613,6 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc;int ret =0 ;int have, implemented; -int file_wanted;bool snap_rwsem_locked = false; dout("get_cap_refs %p need %s want %s\n", inode,
@@ -2600,15 +2628,6 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, goto out_unlock; } -/* make sure file is actually open */ - file_wanted =__ceph_caps_file_wanted (ci); -if ((file_wanted & need) != need) { -dout ("try_get_cap_refs need %s file_wanted %s, EBADF\n" , -ceph_cap_string (need),ceph_cap_string (file_wanted)); - ret = -EBADF ; - goto out_unlock; - } -/* finish pending truncate */ while (ci->i_truncate_pending) {spin_unlock (&ci->i_ceph_lock);
@@ -2719,6 +2738,9 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, ceph_cap_string(have), ceph_cap_string(need)); } out_unlock: + +__ceph_touch_fmode (ci, mdsc, flags); +spin_unlock (&ci->i_ceph_lock);if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem);
@@ -2756,10 +2778,20 @@ static void check_max_size(struct inode *inode, loff_t endoff)ceph_check_caps (ci,CHECK_CAPS_AUTHONLY ,NULL ); } +static inline int get_used_fmode (int caps) +{ +int fmode =0 ; +if (caps &CEPH_CAP_FILE_RD ) + fmode |=CEPH_FILE_MODE_RD ; +if (caps &CEPH_CAP_FILE_WR ) + fmode |=CEPH_FILE_MODE_WR ; +return fmode; +} +int ceph_try_get_caps (struct inode *inode,int need,int want,bool nonblock,int *got) { -int ret; +int ret, flags;BUG_ON (need & ~CEPH_CAP_FILE_RD ); BUG_ON(want & ~(CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO |
@@ -2771,8 +2803,11 @@ int ceph_try_get_caps(struct inode *inode, int need, int want,return ret; } - ret =try_get_cap_refs (inode, need, want,0 , - (nonblock ?NON_BLOCKING :0 ), got); + flags =get_used_fmode (need | want); +if (nonblock) + flags |=NON_BLOCKING ; + + ret =try_get_cap_refs (inode, need, want,0 , flags, got);return ret == -EAGAIN ?0 : ret; }
@@ -2798,11 +2833,15 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen))return -EBADF ; + flags =get_used_fmode (need | want); +while (true) {if (endoff >0 )check_max_size (inode, endoff); - flags =atomic_read (&fi->num_locks) ?CHECK_FILELOCK :0 ; + flags &=CEPH_FILE_MODE_MASK ; +if (atomic_read (&fi->num_locks)) + flags |=CHECK_FILELOCK ; _got =0 ; ret =try_get_cap_refs (inode, need, want, endoff, flags, &_got);
@@ -2822,6 +2861,8 @@ int ceph_get_caps(struct file *filp, int need, int want,list_add (&cw.list, &mdsc->cap_wait_list);spin_unlock (&mdsc->caps_list_lock); +/* make sure used fmode not timeout */ +ceph_get_fmode (ci, flags,FMODE_WAIT_BIAS );add_wait_queue (&ci->i_cap_wq, &wait); flags |=NON_BLOCKING ;
@@ -2835,6 +2876,7 @@ int ceph_get_caps(struct file *filp, int need, int want, }remove_wait_queue (&ci->i_cap_wq, &wait); +ceph_put_fmode (ci, flags,FMODE_WAIT_BIAS );spin_lock (&mdsc->caps_list_lock);list_del (&cw.list);
@@ -2854,7 +2896,7 @@ int ceph_get_caps(struct file *filp, int need, int want,if (ret <0 ) {if (ret == -ESTALE ) {/* session was killed, try renew caps */ - ret =ceph_renew_caps (inode); + ret =ceph_renew_caps (inode, flags);if (ret ==0 )continue ; }
@@ -4153,6 +4195,33 @@ void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc)dout ("flush_dirty_caps done\n" ); } +void __ceph_touch_fmode (struct ceph_inode_info *ci, +struct ceph_mds_client *mdsc,int fmode) +{ +unsigned long now = jiffies; +if (fmode &CEPH_FILE_MODE_RD ) + ci->i_last_rd = now; +if (fmode &CEPH_FILE_MODE_WR ) + ci->i_last_wr = now; +/* queue periodic check */ +if (fmode && +__ceph_is_any_real_caps (ci) && +list_empty (&ci->i_cap_delay_list)) +__cap_delay_requeue (mdsc, ci, true); +} + +void ceph_get_fmode (struct ceph_inode_info *ci,int fmode,int count) +{ +int i; +int bits = (fmode <<1 ) |1 ; +spin_lock (&ci->i_ceph_lock); +for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) { +if (bits & (1 << i)) + ci->i_nr_by_mode[i] += count; + } +spin_unlock (&ci->i_ceph_lock); +} +void __ceph_get_fmode (struct ceph_inode_info *ci,int fmode) {int i;
@@ -4168,26 +4237,18 @@ void __ceph_get_fmode(struct ceph_inode_info *ci, int fmode)* we may need to release capabilities to the MDS (or schedule * their delayed release). */ -void ceph_put_fmode (struct ceph_inode_info *ci,int fmode) +void ceph_put_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i, last =0 ; +int i;int bits = (fmode <<1 ) |1 ;spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) { -BUG_ON (ci->i_nr_by_mode[i] ==0 ); -if (--ci->i_nr_by_mode[i] ==0 ) - last++; +BUG_ON (ci->i_nr_by_mode[i] < count); + ci->i_nr_by_mode[i] -= count; } } -dout ("put_fmode %p fmode %d {%d,%d,%d,%d}\n" , - &ci->vfs_inode, fmode, - ci->i_nr_by_mode[0 ], ci->i_nr_by_mode[1 ], - ci->i_nr_by_mode[2 ], ci->i_nr_by_mode[3 ]);spin_unlock (&ci->i_ceph_lock); - -if (last && ci->i_vino.snap ==CEPH_NOSNAP ) -ceph_check_caps (ci,0 ,NULL ); }/*
fs/ceph/file.c +12 -9
@@ -213,7 +213,7 @@ static int ceph_init_file_info(struct inode *inode, struct file *file,struct ceph_dir_file_info *dfi =kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL );if (!dfi) { -ceph_put_fmode (ci, fmode);/* clean up */ +ceph_put_fmode (ci, fmode,1 );/* clean up */ return -ENOMEM ; }
@@ -224,7 +224,7 @@ static int ceph_init_file_info(struct inode *inode, struct file *file, }else { fi =kmem_cache_zalloc (ceph_file_cachep,GFP_KERNEL );if (!fi) { -ceph_put_fmode (ci, fmode);/* clean up */ +ceph_put_fmode (ci, fmode,1 );/* clean up */ return -ENOMEM ; }
@@ -263,7 +263,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode)case S_IFLNK :dout ("init_file %p %p 0%o (symlink)\n" , inode, file, inode->i_mode); -ceph_put_fmode (ceph_inode (inode), fmode);/* clean up */ +ceph_put_fmode (ceph_inode (inode), fmode,1 );/* clean up */ break ;default :
@@ -273,7 +273,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode)* we need to drop the open ref now, since we don't * have .release set to ceph_release. */ -ceph_put_fmode (ceph_inode (inode), fmode);/* clean up */ +ceph_put_fmode (ceph_inode (inode), fmode,1 );/* clean up */ BUG_ON (inode->i_fop->release == ceph_release);/* call the proper open fop */
@@ -285,14 +285,15 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode)/* * try renew caps after session gets killed. */ -int ceph_renew_caps (struct inode *inode) +int ceph_renew_caps (struct inode *inode,int fmode) { -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc;struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_mds_request *req;int err, flags, wanted;spin_lock (&ci->i_ceph_lock); +__ceph_touch_fmode (ci, mdsc, fmode); wanted =__ceph_caps_file_wanted (ci);if (__ceph_is_any_real_caps (ci) && (!(wanted &CEPH_CAP_ANY_WR ) || ci->i_auth_cap)) {
@@ -405,6 +406,7 @@ int ceph_open(struct inode *inode, struct file *file) }else if (ceph_snap (inode) !=CEPH_NOSNAP && (ci->i_snap_caps & wanted) == wanted) {__ceph_get_fmode (ci, fmode); +__ceph_touch_fmode (ci, mdsc, fmode);spin_unlock (&ci->i_ceph_lock);return ceph_init_file (inode, file, fmode); }
@@ -781,7 +783,7 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, } out_req:if (!req->r_err && req->r_target_inode) -ceph_put_fmode (ceph_inode (req->r_target_inode), req->r_fmode); +ceph_put_fmode (ceph_inode (req->r_target_inode), req->r_fmode,1 );ceph_mdsc_put_request (req); out_ctx:ceph_release_acl_sec_ctx (&as_ctx);
@@ -798,7 +800,7 @@ int ceph_release(struct inode *inode, struct file *file)dout ("release inode %p dir file %p\n" , inode, file);WARN_ON (!list_empty (&dfi->file_info.rw_contexts)); -ceph_put_fmode (ci, dfi->file_info.fmode); +ceph_put_fmode (ci, dfi->file_info.fmode,1 );if (dfi->last_readdir)ceph_mdsc_put_request (dfi->last_readdir);
@@ -810,7 +812,8 @@ int ceph_release(struct inode *inode, struct file *file)dout ("release inode %p regular file %p\n" , inode, file);WARN_ON (!list_empty (&fi->rw_contexts)); -ceph_put_fmode (ci, fi->fmode); +ceph_put_fmode (ci, fi->fmode,1 ); +kmem_cache_free (ceph_file_cachep, fi); }
fs/ceph/inode.c +9 -1
@@ -479,6 +479,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_head_snapc =NULL ; ci->i_snap_caps =0 ; + ci->i_last_rd = ci->i_last_wr = jiffies -3600 *HZ ;for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) ci->i_nr_by_mode[i] =0 ;
@@ -639,7 +640,7 @@ int ceph_fill_file_size(struct inode *inode, int issued,if ((issued & (CEPH_CAP_FILE_CACHE |CEPH_CAP_FILE_BUFFER )) ||mapping_mapped (inode->i_mapping) || -__ceph_caps_file_wanted (ci)) { +__ceph_is_file_opened (ci)) { ci->i_truncate_pending++; queue_trunc =1 ; }
@@ -1013,6 +1014,13 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, fill_inline = true; } +if (cap_fmode >=0 ) { +if (!info_caps) +pr_warn ("mds issued no caps on %llx.%llx\n" , +ceph_vinop (inode)); +__ceph_touch_fmode (ci, mdsc, cap_fmode); + } +spin_unlock (&ci->i_ceph_lock);if (fill_inline)
fs/ceph/ioctl.c +2 -0
@@ -243,11 +243,13 @@ static long ceph_ioctl_lazyio(struct file *file)struct ceph_file_info *fi = file->private_data;struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode); +struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc;if ((fi->fmode &CEPH_FILE_MODE_LAZY ) ==0 ) {spin_lock (&ci->i_ceph_lock); fi->fmode |=CEPH_FILE_MODE_LAZY ; ci->i_nr_by_mode[ffs (CEPH_FILE_MODE_LAZY )]++; +__ceph_touch_fmode (ci, mdsc, fi->fmode);spin_unlock (&ci->i_ceph_lock);dout ("ioctl_layzio: file %p marked lazy\n" , file);
fs/ceph/super.h +11 -2
@@ -366,6 +366,8 @@ struct ceph_inode_info {dirty|flushing caps */ unsigned i_snap_caps;/* cap bits for snapped files */ +unsigned long i_last_rd; +unsigned long i_last_wr;int i_nr_by_mode[CEPH_FILE_MODE_BITS ];/* open file counts */ struct mutex i_truncate_mutex;
@@ -680,6 +682,10 @@ extern int __ceph_caps_revoking_other(struct ceph_inode_info *ci,extern int ceph_caps_revoking (struct ceph_inode_info *ci,int mask);extern int __ceph_caps_used (struct ceph_inode_info *ci); +static inline bool __ceph_is_file_opened (struct ceph_inode_info *ci) +{ +return ci->i_nr_by_mode[0 ]; +}extern int __ceph_caps_file_wanted (struct ceph_inode_info *ci);extern int __ceph_caps_wanted (struct ceph_inode_info *ci);
@@ -1093,7 +1099,10 @@ extern int ceph_try_get_caps(struct inode *inode,/* for counting open files by mode */ extern void __ceph_get_fmode (struct ceph_inode_info *ci,int mode); -extern void ceph_put_fmode (struct ceph_inode_info *ci,int mode); +extern void ceph_get_fmode (struct ceph_inode_info *ci,int mode,int count); +extern void ceph_put_fmode (struct ceph_inode_info *ci,int mode,int count); +extern void __ceph_touch_fmode (struct ceph_inode_info *ci, +struct ceph_mds_client *mdsc,int fmode);/* addr.c */ extern const struct address_space_operations ceph_aops;
@@ -1105,7 +1114,7 @@ extern void ceph_pool_perm_destroy(struct ceph_mds_client* mdsc);/* file.c */ extern const struct file_operations ceph_file_fops; -extern int ceph_renew_caps (struct inode *inode); +extern int ceph_renew_caps (struct inode *inode,int fmode);extern int ceph_open (struct inode *inode,struct file *file);extern int ceph_atomic_open (struct inode *dir,struct dentry *dentry,struct file *file,unsigned flags,umode_t mode);
include/linux/ceph/ceph_fs.h +1 -0
@@ -568,6 +568,7 @@ struct ceph_filelock {#define CEPH_FILE_MODE_RDWR 3/* RD | WR */ #define CEPH_FILE_MODE_LAZY 4/* lazy io */ #define CEPH_FILE_MODE_BITS 4 +#define CEPH_FILE_MODE_MASK ((1 << CEPH_FILE_MODE_BITS) - 1)int ceph_flags_to_mode (int flags);
Signature · 1 line
2.21.1
PATCH v5 3/7 ceph: remove delay check logic from ceph_check_caps()
In reply to Yan, Zheng__ceph_caps_file_wanted() already checks 'caps_wanted_delay_min' and 'caps_wanted_delay_max'. There is no need to duplicte the logic in ceph_check_caps() and __send_cap()
--- fs/ceph/caps.c | 148 ++++++++++++------------------------------------ fs/ceph/file.c | 13 ++--- fs/ceph/inode.c | 1 - fs/ceph/super.h | 8 +-- 4 files changed, 44 insertions(+), 126 deletions(-)
fs/ceph/caps.c +37 -111
@@ -490,13 +490,10 @@ static void __cap_set_timeouts(struct ceph_mds_client *mdsc, struct ceph_inode_info *ci) {struct ceph_mount_options *opt = mdsc->fsc->mount_options; - - ci->i_hold_caps_min =round_jiffies (jiffies + - opt->caps_wanted_delay_min *HZ ); ci->i_hold_caps_max =round_jiffies (jiffies + opt->caps_wanted_delay_max *HZ ); -dout ("__cap_set_timeouts %p min %lu max %lu\n" , &ci->vfs_inode, - ci->i_hold_caps_min - jiffies, ci->i_hold_caps_max - jiffies); +dout ("__cap_set_timeouts %p %lu\n" , &ci->vfs_inode, + ci->i_hold_caps_max - jiffies); }/*
@@ -508,8 +505,7 @@ static void __cap_set_timeouts(struct ceph_mds_client *mdsc,* -> we take mdsc->cap_delay_lock */ static void __cap_delay_requeue (struct ceph_mds_client *mdsc, -struct ceph_inode_info *ci, -bool set_timeout) +struct ceph_inode_info *ci) {dout ("__cap_delay_requeue %p flags 0x%lx at %lu\n" , &ci->vfs_inode, ci->i_ceph_flags, ci->i_hold_caps_max);
@@ -520,8 +516,7 @@ static void __cap_delay_requeue(struct ceph_mds_client *mdsc, goto no_change;list_del_init (&ci->i_cap_delay_list); } -if (set_timeout) -__cap_set_timeouts (mdsc, ci); +__cap_set_timeouts (mdsc, ci);list_add_tail (&ci->i_cap_delay_list, &mdsc->cap_delay_list); no_change:spin_unlock (&mdsc->cap_delay_lock);
@@ -733,7 +728,7 @@ void ceph_add_cap(struct inode *inode,dout (" issued %s, mds wanted %s, actual %s, queueing\n" ,ceph_cap_string (issued),ceph_cap_string (wanted),ceph_cap_string (actual_wanted)); -__cap_delay_requeue (mdsc, ci, true); +__cap_delay_requeue (mdsc, ci); }if (flags &CEPH_CAP_FLAG_AUTH ) {
@@ -1326,7 +1321,6 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap,struct cap_msg_args arg;int held, revoking;int wake =0 ; -int delayed =0 ;int ret;/* Don't send anything if it's still being created. Return delayed */
@@ -1346,28 +1340,7 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, ceph_cap_string(revoking));BUG_ON ((retain &CEPH_CAP_PIN ) ==0 ); - arg.session = cap->session; - -/* don't release wanted unless we've waited a bit. */ -if ((ci->i_ceph_flags &CEPH_I_NODELAY ) ==0 && -time_before (jiffies, ci->i_hold_caps_min)) { -dout (" delaying issued %s -> %s, wanted %s -> %s on send\n" , -ceph_cap_string (cap->issued), -ceph_cap_string (cap->issued & retain), -ceph_cap_string (cap->mds_wanted), -ceph_cap_string (want)); - want |= cap->mds_wanted; - retain |= cap->issued; - delayed =1 ; - } - ci->i_ceph_flags &= ~(CEPH_I_NODELAY |CEPH_I_FLUSH ); -if (want & ~cap->mds_wanted) { -/* user space may open/close single file frequently. -* This avoids droping mds_wanted immediately after -* requesting new mds_wanted. -*/ -__cap_set_timeouts (mdsc, ci); - } + ci->i_ceph_flags &= ~CEPH_I_FLUSH ; cap->issued &= retain;/* drop bits we don't want */ if (cap->implemented & ~cap->issued) {
@@ -1382,6 +1355,7 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, cap->implemented &= cap->issued | used; cap->mds_wanted = want; + arg.session = cap->session; arg.ino =ceph_vino (inode).ino; arg.cid = cap->cap_id; arg.follows = flushing ? ci->i_head_snapc->seq :0 ;
@@ -1442,14 +1416,19 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, ret =send_cap_msg (&arg);if (ret <0 ) { -dout ("error sending cap msg, must requeue %p\n" , inode); - delayed =1 ; +pr_err ("error sending cap msg, ino (%llx.%llx) " +"flushing %s tid %llu, requeue\n" , +ceph_vinop (inode),ceph_cap_string (flushing), + flush_tid); +spin_lock (&ci->i_ceph_lock); +__cap_delay_requeue (mdsc, ci); +spin_unlock (&ci->i_ceph_lock); }if (wake)wake_up_all (&ci->i_cap_wq); -return delayed; +return ret; } static inline int __send_flush_snap(struct inode *inode,
@@ -1715,7 +1694,7 @@ int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask,if (((was | ci->i_flushing_caps) &CEPH_CAP_FILE_BUFFER ) && (mask &CEPH_CAP_FILE_BUFFER )) dirty |=I_DIRTY_DATASYNC ; -__cap_delay_requeue (mdsc, ci, true); +__cap_delay_requeue (mdsc, ci);return dirty; }
@@ -1867,8 +1846,6 @@ bool __ceph_should_report_size(struct ceph_inode_info *ci)* versus held caps. Release, flush, ack revoked caps to mds as * appropriate. * -* CHECK_CAPS_NODELAY - caller is delayed work and we should not delay -* cap release further. * CHECK_CAPS_AUTHONLY - we should only check the auth cap * CHECK_CAPS_FLUSH - we should flush any dirty caps immediately, without * further delay.
@@ -1887,17 +1864,10 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,int mds =-1 ;/* keep track of how far we've gone through i_caps list to avoid an infinite loop on retry */ struct rb_node *p; -int delayed =0 , sent =0 ; -bool no_delay = flags &CHECK_CAPS_NODELAY ;bool queue_invalidate = false;bool tried_invalidate = false; -/* if we are unmounting, flush any unused caps immediately. */ -if (mdsc->stopping) - no_delay = true; -spin_lock (&ci->i_ceph_lock); -if (ci->i_ceph_flags &CEPH_I_FLUSH ) flags |=CHECK_CAPS_FLUSH ;
@@ -1946,14 +1916,13 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags, }dout ("check_caps %p file_want %s used %s dirty %s flushing %s" -" issued %s revoking %s retain %s %s%s%s\n" , inode, +" issued %s revoking %s retain %s %s%s\n" , inode,ceph_cap_string (file_wanted),ceph_cap_string (used),ceph_cap_string (ci->i_dirty_caps),ceph_cap_string (ci->i_flushing_caps),ceph_cap_string (issued),ceph_cap_string (revoking),ceph_cap_string (retain), (flags &CHECK_CAPS_AUTHONLY ) ?" AUTHONLY" :"" , - (flags &CHECK_CAPS_NODELAY ) ?" NODELAY" :"" , (flags &CHECK_CAPS_FLUSH ) ?" FLUSH" :"" );/*
@@ -1961,7 +1930,7 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,* have cached pages, but don't want them, then try to invalidate. * If we fail, it's because pages are locked.... try again later. */ - if ((!no_delay || mdsc->stopping) && + if ((!(flags & CHECK_CAPS_NOINVAL) || mdsc->stopping) && S_ISREG(inode->i_mode) && !(ci->i_wb_ref || ci->i_wrbuffer_ref) && /* no dirty pages... */ inode->i_data.nrpages && /* have cached pages */
@@ -2041,21 +2010,6 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,if ((cap->issued & ~retain) ==0 )continue ;/* nope, all good */ -if (no_delay) - goto ack; - -/* delay? */ -if ((ci->i_ceph_flags &CEPH_I_NODELAY ) ==0 && -time_before (jiffies, ci->i_hold_caps_max)) { -dout (" delaying issued %s -> %s, wanted %s -> %s\n" , -ceph_cap_string (cap->issued), -ceph_cap_string (cap->issued & retain), -ceph_cap_string (cap->mds_wanted), -ceph_cap_string (want)); - delayed++; -continue ; - } - ack:if (session && session != cap->session) {dout ("oops, wrong session %p mutex\n" , session);
@@ -2116,25 +2070,19 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags, } mds = cap->mds;/* remember mds, so we don't repeat */ - sent++;/* __send_cap drops i_ceph_lock */ - delayed +=__send_cap (mdsc, cap,CEPH_CAP_OP_UPDATE ,0 , - cap_used, want, retain, flushing, - flush_tid, oldest_flush_tid); +__send_cap (mdsc, cap,CEPH_CAP_OP_UPDATE ,0 , cap_used, want, + retain, flushing, flush_tid, oldest_flush_tid); goto retry;/* retake i_ceph_lock and restart our cap scan. */ } -if (list_empty (&ci->i_cap_delay_list)) { -if (delayed) { -/* Reschedule delayed caps release if we delayed anything */ -__cap_delay_requeue (mdsc, ci, false); - }else if (__ceph_is_any_real_caps (ci) && - (file_wanted & ~CEPH_CAP_PIN ) && - !(used & (CEPH_CAP_FILE_RD |CEPH_CAP_ANY_FILE_WR ))) { -/* periodically re-calculate caps wanted by open files */ -__cap_delay_requeue (mdsc, ci, true); - } +/* periodically re-calculate caps wanted by open files */ +if (__ceph_is_any_real_caps (ci) && +list_empty (&ci->i_cap_delay_list) && + (file_wanted & ~CEPH_CAP_PIN ) && + !(used & (CEPH_CAP_FILE_RD |CEPH_CAP_ANY_FILE_WR ))) { +__cap_delay_requeue (mdsc, ci); }spin_unlock (&ci->i_ceph_lock);
@@ -2164,7 +2112,6 @@ static int try_flush_caps(struct inode *inode, u64 *ptid) retry_locked:if (ci->i_dirty_caps && ci->i_auth_cap) {struct ceph_cap *cap = ci->i_auth_cap; -int delayed;if (session != cap->session) {spin_unlock (&ci->i_ceph_lock);
@@ -2193,18 +2140,10 @@ static int try_flush_caps(struct inode *inode, u64 *ptid) &oldest_flush_tid);/* __send_cap drops i_ceph_lock */ - delayed =__send_cap (mdsc, cap,CEPH_CAP_OP_FLUSH , -CEPH_CLIENT_CAPS_SYNC , -__ceph_caps_used (ci), -__ceph_caps_wanted (ci), - (cap->issued | cap->implemented), - flushing, flush_tid, oldest_flush_tid); - -if (delayed) { -spin_lock (&ci->i_ceph_lock); -__cap_delay_requeue (mdsc, ci, true); -spin_unlock (&ci->i_ceph_lock); - } +__send_cap (mdsc, cap,CEPH_CAP_OP_FLUSH ,CEPH_CLIENT_CAPS_SYNC , +__ceph_caps_used (ci),__ceph_caps_wanted (ci), + (cap->issued | cap->implemented), + flushing, flush_tid, oldest_flush_tid); }else {if (!list_empty (&ci->i_cap_flush_list)) { struct ceph_cap_flush *cf =
@@ -2408,22 +2347,13 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc,if (cf->caps) {dout ("kick_flushing_caps %p cap %p tid %llu %s\n" , inode, cap, cf->tid,ceph_cap_string (cf->caps)); - ci->i_ceph_flags |=CEPH_I_NODELAY ; - - ret =__send_cap (mdsc, cap,CEPH_CAP_OP_FLUSH , +__send_cap (mdsc, cap,CEPH_CAP_OP_FLUSH , (cf->tid < last_snap_flush ?CEPH_CLIENT_CAPS_PENDING_CAPSNAP :0 ),__ceph_caps_used (ci),__ceph_caps_wanted (ci), (cap->issued | cap->implemented), cf->caps, cf->tid, oldest_flush_tid); -if (ret) { -pr_err ("kick_flushing_caps: error sending " -"cap flush, ino (%llx.%llx) " -"tid %llu flushing %s\n" , -ceph_vinop (inode), cf->tid, -ceph_cap_string (cf->caps)); - } }else {struct ceph_cap_snap *capsnap = container_of(cf, struct ceph_cap_snap,
@@ -3045,7 +2975,7 @@ void ceph_put_cap_refs(struct ceph_inode_info *ci, int had)dout ("put_cap_refs %p had %s%s%s\n" , inode,ceph_cap_string (had), last ?" last" :"" , put ?" put" :"" ); -if (last && !flushsnaps) +if (last)ceph_check_caps (ci,0 ,NULL );else if (flushsnaps)ceph_flush_snaps (ci,NULL );
@@ -3464,10 +3394,10 @@ static void handle_cap_grant(struct inode *inode,wake_up_all (&ci->i_cap_wq);if (check_caps ==1 ) -ceph_check_caps (ci,CHECK_CAPS_NODELAY |CHECK_CAPS_AUTHONLY , +ceph_check_caps (ci,CHECK_CAPS_AUTHONLY |CHECK_CAPS_NOINVAL , session);else if (check_caps ==2 ) -ceph_check_caps (ci,CHECK_CAPS_NODELAY , session); +ceph_check_caps (ci,CHECK_CAPS_NOINVAL , session);else mutex_unlock (&session->s_mutex); }
@@ -4142,7 +4072,6 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc) {struct inode *inode;struct ceph_inode_info *ci; -int flags =CHECK_CAPS_NODELAY ;dout ("check_delayed_caps\n" );while (1 ) {
@@ -4162,7 +4091,7 @@ void ceph_check_delayed_caps(struct ceph_mds_client *mdsc)if (inode) {dout ("check_delayed_caps on %p\n" , inode); -ceph_check_caps (ci, flags,NULL ); +ceph_check_caps (ci,0 ,NULL );/* avoid calling iput_final() in tick thread */ ceph_async_iput (inode); }
@@ -4187,7 +4116,7 @@ void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc)ihold (inode);dout ("flush_dirty_caps %p\n" , inode);spin_unlock (&mdsc->cap_dirty_lock); -ceph_check_caps (ci,CHECK_CAPS_NODELAY |CHECK_CAPS_FLUSH ,NULL ); +ceph_check_caps (ci,CHECK_CAPS_FLUSH ,NULL );iput (inode);spin_lock (&mdsc->cap_dirty_lock); }
@@ -4207,7 +4136,7 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci,if (fmode &&__ceph_is_any_real_caps (ci) &&list_empty (&ci->i_cap_delay_list)) -__cap_delay_requeue (mdsc, ci, true); +__cap_delay_requeue (mdsc, ci); }void ceph_get_fmode (struct ceph_inode_info *ci,int fmode,int count)
@@ -4266,7 +4195,6 @@ int ceph_drop_caps_for_unlink(struct inode *inode)if (inode->i_nlink ==1 ) { drop |= ~(__ceph_caps_wanted (ci) |CEPH_CAP_PIN ); - ci->i_ceph_flags |=CEPH_I_NODELAY ;if (__ceph_caps_dirty (ci)) {struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc;
@@ -4322,8 +4250,6 @@ int ceph_encode_inode_release(void **p, struct inode *inode,if (force || (cap->issued & drop)) {if (cap->issued & drop) {int wanted =__ceph_caps_wanted (ci); -if ((ci->i_ceph_flags &CEPH_I_NODELAY ) ==0 ) - wanted |= cap->mds_wanted; dout("encode_inode_release %p cap %p " "%s -> %s, wanted %s -> %s\n", inode, cap, ceph_cap_string(cap->issued),
fs/ceph/file.c +4 -9
@@ -1819,7 +1819,7 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)if (dirty)__mark_inode_dirty (inode, dirty);if (ceph_quota_is_max_bytes_approaching (inode, iocb->ki_pos)) -ceph_check_caps (ci,CHECK_CAPS_NODELAY ,NULL ); +ceph_check_caps (ci,0 ,NULL ); } dout("aio_write %p %llx.%llx %llu~%u dropping cap refs on %s\n",
@@ -2419,15 +2419,10 @@ static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,inode_inc_iversion_raw (dst_inode);if (dst_off > size) { -int caps_flags =0 ; -/* Let the MDS know about dst file size change */ -if (ceph_quota_is_max_bytes_approaching (dst_inode, dst_off)) - caps_flags |=CHECK_CAPS_NODELAY ; -if (ceph_inode_set_size (dst_inode, dst_off)) - caps_flags |=CHECK_CAPS_AUTHONLY ; -if (caps_flags) -ceph_check_caps (dst_ci, caps_flags,NULL ); +if (ceph_inode_set_size (dst_inode, dst_off) || +ceph_quota_is_max_bytes_approaching (dst_inode, dst_off)) +ceph_check_caps (dst_ci,CHECK_CAPS_AUTHONLY ,NULL ); }/* Mark Fw dirty */ spin_lock (&dst_ci->i_ceph_lock);
fs/ceph/inode.c +0 -1
@@ -472,7 +472,6 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_prealloc_cap_flush =NULL ;INIT_LIST_HEAD (&ci->i_cap_flush_list);init_waitqueue_head (&ci->i_cap_wq); - ci->i_hold_caps_min =0 ; ci->i_hold_caps_max =0 ;INIT_LIST_HEAD (&ci->i_cap_delay_list);INIT_LIST_HEAD (&ci->i_cap_snaps);
fs/ceph/super.h +3 -5
@@ -173,9 +173,9 @@ struct ceph_cap {struct list_head caps_item; }; -#define CHECK_CAPS_NODELAY 1/* do not delay any further */ -#define CHECK_CAPS_AUTHONLY 2/* only check auth cap */ -#define CHECK_CAPS_FLUSH 4/* flush any dirty caps */ +#define CHECK_CAPS_AUTHONLY 1/* only check auth cap */ +#define CHECK_CAPS_FLUSH 2/* flush any dirty caps */ +#define CHECK_CAPS_NOINVAL 4/* don't invalidate pagecache */ struct ceph_cap_flush {u64 tid;
@@ -357,7 +357,6 @@ struct ceph_inode_info {struct ceph_cap_flush *i_prealloc_cap_flush;struct list_head i_cap_flush_list;wait_queue_head_t i_cap_wq;/* threads waiting on a capability */ -unsigned long i_hold_caps_min;/* jiffies */ unsigned long i_hold_caps_max;/* jiffies */ struct list_head i_cap_delay_list;/* for delayed cap release to mds */ struct ceph_cap_reservation i_cap_migration_resv;
@@ -518,7 +517,6 @@ static inline struct inode *ceph_find_inode(struct super_block *sb,* Ceph inode. */ #define CEPH_I_DIR_ORDERED (1 << 0)/* dentries in dir are ordered */ -#define CEPH_I_NODELAY (1 << 1)/* do not delay cap release */ #define CEPH_I_FLUSH (1 << 2)/* do not delay flush of dirty metadata */ #define CEPH_I_POOL_PERM (1 << 3)/* pool rd/wr bits are valid */ #define CEPH_I_POOL_RD (1 << 4)/* can read from pool */
Signature · 1 line
2.21.1
PATCH v5 4/7 ceph: simplify calling of ceph_get_fmode()
In reply to Yan, ZhengOriginally, calling ceph_get_fmode() for open files is by thread that handles request reply. There is a small window between updating caps and and waking the request initiator. We need to prevent ceph_check_caps() from releasing wanted caps in the window.
Previous patchies made fill_inode() call __ceph_touch_fmode() for open file requests. This prevented ceph_check_caps() from releasing wanted caps for 'caps_wanted_delay_min' seconds, enough for request initiator to get woken up and call ceph_get_fmode().
This allows us to now call ceph_get_fmode() in ceph_open() instead.
--- fs/ceph/caps.c | 26 +++----------------------- fs/ceph/file.c | 21 +++++---------------- fs/ceph/inode.c | 8 +------- fs/ceph/super.h | 3 +-- 4 files changed, 10 insertions(+), 48 deletions(-)
fs/ceph/caps.c +3 -23
@@ -608,7 +608,7 @@ static void __check_cap_issue(struct ceph_inode_info *ci, struct ceph_cap *cap,*/ void ceph_add_cap (struct inode *inode,struct ceph_mds_session *session,u64 cap_id, -int fmode,unsigned issued,unsigned wanted, +unsigned issued,unsigned wanted,unsigned seq,unsigned mseq,u64 realmino,int flags,struct ceph_cap **new_cap) {
@@ -624,13 +624,6 @@ void ceph_add_cap(struct inode *inode,dout ("add_cap %p mds%d cap %llx %s seq %d\n" , inode, session->s_mds, cap_id,ceph_cap_string (issued), seq); -/* -* If we are opening the file, include file mode wanted bits -* in wanted. -*/ -if (fmode >=0 ) - wanted |=ceph_caps_for_mode (fmode); -spin_lock (&session->s_gen_ttl_lock); gen = session->s_cap_gen;spin_unlock (&session->s_gen_ttl_lock);
@@ -755,9 +748,6 @@ void ceph_add_cap(struct inode *inode, cap->issue_seq = seq; cap->mseq = mseq; cap->cap_gen = gen; - -if (fmode >=0 ) -__ceph_get_fmode (ci, fmode); }/*
@@ -3693,7 +3683,7 @@ static void handle_cap_export(struct inode *inode, struct ceph_mds_caps *ex,/* add placeholder for the export tagert */ int flag = (cap == ci->i_auth_cap) ?CEPH_CAP_FLAG_AUTH :0 ; tcap = new_cap; -ceph_add_cap (inode, tsession, t_cap_id,-1 , issued,0 , +ceph_add_cap (inode, tsession, t_cap_id, issued,0 , t_seq -1 , t_mseq, (u64)-1 , flag, &new_cap); if (!list_empty(&ci->i_cap_flush_list) &&
@@ -3798,7 +3788,7 @@ static void handle_cap_import(struct ceph_mds_client *mdsc,__ceph_caps_issued (ci, &issued); issued |=__ceph_caps_dirty (ci); -ceph_add_cap (inode, session, cap_id,-1 , caps, wanted, seq, mseq, +ceph_add_cap (inode, session, cap_id, caps, wanted, seq, mseq, realmino,CEPH_CAP_FLAG_AUTH , &new_cap); ocap = peer >=0 ?__get_cap_for_mds (ci, peer) :NULL ;
@@ -4151,16 +4141,6 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)spin_unlock (&ci->i_ceph_lock); } -void __ceph_get_fmode (struct ceph_inode_info *ci,int fmode) -{ -int i; -int bits = (fmode <<1 ) |1 ; -for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) { -if (bits & (1 << i)) - ci->i_nr_by_mode[i]++; - } -} -/* * Drop open file reference. If we were the last open file, * we may need to release capabilities to the MDS (or schedule
fs/ceph/file.c +5 -16
@@ -212,10 +212,8 @@ static int ceph_init_file_info(struct inode *inode, struct file *file,if (isdir) {struct ceph_dir_file_info *dfi =kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL ); -if (!dfi) { -ceph_put_fmode (ci, fmode,1 );/* clean up */ +if (!dfi)return -ENOMEM ; - } file->private_data = dfi; fi = &dfi->file_info;
@@ -223,15 +221,15 @@ static int ceph_init_file_info(struct inode *inode, struct file *file, dfi->readdir_cache_idx =-1 ; }else { fi =kmem_cache_zalloc (ceph_file_cachep,GFP_KERNEL ); -if (!fi) { -ceph_put_fmode (ci, fmode,1 );/* clean up */ +if (!fi)return -ENOMEM ; - } file->private_data = fi; } +ceph_get_fmode (ci, fmode,1 ); fi->fmode = fmode; +spin_lock_init (&fi->rw_contexts_lock);INIT_LIST_HEAD (&fi->rw_contexts); fi->meta_err =errseq_sample (&ci->i_meta_err);
@@ -263,7 +261,6 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode)case S_IFLNK :dout ("init_file %p %p 0%o (symlink)\n" , inode, file, inode->i_mode); -ceph_put_fmode (ceph_inode (inode), fmode,1 );/* clean up */ break ;default :
@@ -273,7 +270,6 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode)* we need to drop the open ref now, since we don't * have .release set to ceph_release. */ -ceph_put_fmode (ceph_inode (inode), fmode,1 );/* clean up */ BUG_ON (inode->i_fop->release == ceph_release);/* call the proper open fop */
@@ -327,7 +323,6 @@ int ceph_renew_caps(struct inode *inode, int fmode) req->r_inode = inode;ihold (inode); req->r_num_caps =1 ; - req->r_fmode =-1 ; err =ceph_mdsc_do_request (mdsc,NULL , req);ceph_mdsc_put_request (req);
@@ -373,9 +368,6 @@ int ceph_open(struct inode *inode, struct file *file)/* trivially open snapdir */ if (ceph_snap (inode) ==CEPH_SNAPDIR ) { -spin_lock (&ci->i_ceph_lock); -__ceph_get_fmode (ci, fmode); -spin_unlock (&ci->i_ceph_lock);return ceph_init_file (inode, file, fmode); }
@@ -393,7 +385,7 @@ int ceph_open(struct inode *inode, struct file *file)dout ("open %p fmode %d want %s issued %s using existing\n" , inode, fmode,ceph_cap_string (wanted),ceph_cap_string (issued)); -__ceph_get_fmode (ci, fmode); +__ceph_touch_fmode (ci, mdsc, fmode);spin_unlock (&ci->i_ceph_lock);/* adjust wanted? */
@@ -405,7 +397,6 @@ int ceph_open(struct inode *inode, struct file *file)return ceph_init_file (inode, file, fmode); }else if (ceph_snap (inode) !=CEPH_NOSNAP && (ci->i_snap_caps & wanted) == wanted) { -__ceph_get_fmode (ci, fmode);__ceph_touch_fmode (ci, mdsc, fmode);spin_unlock (&ci->i_ceph_lock);return ceph_init_file (inode, file, fmode);
@@ -782,8 +773,6 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry, err =finish_open (file, dentry, ceph_open); } out_req: -if (!req->r_err && req->r_target_inode) -ceph_put_fmode (ceph_inode (req->r_target_inode), req->r_fmode,1 );ceph_mdsc_put_request (req); out_ctx:ceph_release_acl_sec_ctx (&as_ctx);
fs/ceph/inode.c +1 -7
@@ -970,7 +970,7 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page,if (ceph_snap (inode) ==CEPH_NOSNAP ) { ceph_add_cap(inode, session, le64_to_cpu(info->cap.cap_id), - cap_fmode, info_caps, + info_caps, le32_to_cpu(info->cap.wanted), le32_to_cpu(info->cap.seq), le32_to_cpu(info->cap.mseq),
@@ -995,13 +995,7 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page,dout (" %p got snap_caps %s\n" , inode,ceph_cap_string (info_caps)); ci->i_snap_caps |= info_caps; -if (cap_fmode >=0 ) -__ceph_get_fmode (ci, cap_fmode); } - }else if (cap_fmode >=0 ) { -pr_warn ("mds issued no caps on %llx.%llx\n" , -ceph_vinop (inode)); -__ceph_get_fmode (ci, cap_fmode); } if (iinfo->inline_version > 0 &&
fs/ceph/super.h +1 -2
@@ -1050,7 +1050,7 @@ extern struct ceph_cap *ceph_get_cap(struct ceph_mds_client *mdsc, struct ceph_cap_reservation *ctx);extern void ceph_add_cap (struct inode *inode,struct ceph_mds_session *session,u64 cap_id, -int fmode,unsigned issued,unsigned wanted, +unsigned issued,unsigned wanted,unsigned cap,unsigned seq,u64 realmino,int flags,struct ceph_cap **new_cap);extern void __ceph_remove_cap (struct ceph_cap *cap,bool queue_release);
@@ -1096,7 +1096,6 @@ extern int ceph_try_get_caps(struct inode *inode, int need, int want, bool nonblock, int *got);/* for counting open files by mode */ -extern void __ceph_get_fmode (struct ceph_inode_info *ci,int mode);extern void ceph_get_fmode (struct ceph_inode_info *ci,int mode,int count);extern void ceph_put_fmode (struct ceph_inode_info *ci,int mode,int count); extern void __ceph_touch_fmode(struct ceph_inode_info *ci,
Signature · 1 line
2.21.1
PATCH v5 5/7 ceph: update i_requested_max_size only when sending cap msg to auth mds
In reply to Yan, ZhengNon-auth mds can't do anything to 'update max' cap message.
--- fs/ceph/caps.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-)
fs/ceph/caps.c +2 -1
@@ -1355,7 +1355,8 @@ static int __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, arg.size = inode->i_size; ci->i_reported_size = arg.size; arg.max_size = ci->i_wanted_max_size; - ci->i_requested_max_size = arg.max_size; +if (cap == ci->i_auth_cap) + ci->i_requested_max_size = arg.max_size;if (flushing &CEPH_CAP_XATTR_EXCL ) { old_blob =__ceph_build_xattrs_blob (ci);
Signature · 1 line
2.21.1
PATCH v5 6/7 ceph: check all mds' caps after page writeback
In reply to Yan, ZhengIf an inode has caps from multiple mds's, the following can happen:
- non-auth mds revokes Fsc. Fcb is used, so page writeback is queued.
- when writeback finishes, ceph_check_caps() is called with auth only flag. ceph_check_caps() invalidates pagecache, but skips checking any non-auth caps.
--- fs/ceph/caps.c | 2 +- fs/ceph/inode.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
fs/ceph/caps.c +1 -1
@@ -3048,7 +3048,7 @@ void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr,spin_unlock (&ci->i_ceph_lock);if (last) { -ceph_check_caps (ci,CHECK_CAPS_AUTHONLY ,NULL ); +ceph_check_caps (ci,0 ,NULL ); }else if (flush_snaps) {ceph_flush_snaps (ci,NULL ); }
fs/ceph/inode.c +1 -1
@@ -1984,7 +1984,7 @@ void __ceph_do_pending_vmtruncate(struct inode *inode)mutex_unlock (&ci->i_truncate_mutex);if (wrbuffer_refs ==0 ) -ceph_check_caps (ci,CHECK_CAPS_AUTHONLY ,NULL ); +ceph_check_caps (ci,0 ,NULL );wake_up_all (&ci->i_cap_wq); }
Signature · 1 line
2.21.1
PATCH v5 7/7 ceph: calculate dir's wanted caps according to recent dirops
In reply to Yan, ZhengCall __ceph_touch_fmode() for dir operations. __ceph_caps_file_wanted() calculate dir's wanted caps according to last dir read/modification. If there is recent dir read, dir inode wants CEPH_CAP_ANY_SHARED caps, if there is recent dir modification, also wants CEPH_CAP_FILE_EXCL.
Readdir is a special, dir inode wants CEPH_CAP_FILE_EXCL after readdir. Because if dir indoe has CEPH_CAP_FILE_EXCL, later dir modifications do not need to release CEPH_CAP_FILE_SHARED, invalidate all dentry leases issued by readdir.
--- fs/ceph/caps.c | 30 ++++++++++++++++++++++-------- fs/ceph/dir.c | 21 +++++++++++++++------ fs/ceph/mds_client.c | 11 +++++++++-- 3 files changed, 46 insertions(+), 16 deletions(-)
fs/ceph/caps.c +22 -8
@@ -978,12 +978,29 @@ int __ceph_caps_file_wanted(struct ceph_inode_info *ci)ceph_inode_to_client (&ci->vfs_inode)->mount_options;unsigned long used_cutoff = jiffies - opt->caps_wanted_delay_max *HZ ;unsigned long idle_cutoff = jiffies - opt->caps_wanted_delay_min *HZ ; -int bits =0 ;if (S_ISDIR (ci->vfs_inode.i_mode)) { -if (ci->i_nr_by_mode[PIN_SHIFT ] >0 ) - bits |=1 <<PIN_SHIFT ; +int want =0 ; + +/* use used_cutoff here, to keep dir's wanted caps longer */ +if (ci->i_nr_by_mode[RD_SHIFT ] >0 || +time_after (ci->i_last_rd, used_cutoff)) + want |=CEPH_CAP_ANY_SHARED ; + +if (ci->i_nr_by_mode[WR_SHIFT ] >0 || +time_after (ci->i_last_wr, used_cutoff)) { + want |=CEPH_CAP_ANY_SHARED |CEPH_CAP_FILE_EXCL ; +if (opt->flags &CEPH_MOUNT_OPT_ASYNC_DIROPS ) + want |=CEPH_CAP_ANY_DIR_OPS ; + } + +if (want || ci->i_nr_by_mode[PIN_SHIFT ] >0 ) + want |=CEPH_CAP_PIN ; + +return want; }else { +int bits =0 ; +if (ci->i_nr_by_mode[RD_SHIFT ] >0 ) {if (ci->i_nr_by_mode[RD_SHIFT ] >=FMODE_WAIT_BIAS ||time_after (ci->i_last_rd, used_cutoff))
@@ -1005,9 +1022,8 @@ int __ceph_caps_file_wanted(struct ceph_inode_info *ci) ci->i_nr_by_mode[LAZY_SHIFT] > 0) bits |=1 <<LAZY_SHIFT ; +return bits ?ceph_caps_for_mode (bits >>1 ) :0 ; } - -return bits ?ceph_caps_for_mode (bits >>1 ) :0 ; }/*
@@ -1888,9 +1904,7 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags,if (IS_RDONLY (inode)) { want =CEPH_CAP_ANY_SHARED ; }else { - want =CEPH_CAP_ANY_SHARED | -CEPH_CAP_FILE_EXCL | -CEPH_CAP_ANY_DIR_OPS ; + want |=CEPH_CAP_ANY_SHARED |CEPH_CAP_FILE_EXCL ; } retain |= want; }else {
fs/ceph/dir.c +15 -6
@@ -335,8 +335,11 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) ctx->pos =2 ; } -/* can we use the dcache? */ spin_lock (&ci->i_ceph_lock); +/* request Fx cap. if have Fx, we don't need to release Fs cap +* for later create/unlink. */ +__ceph_touch_fmode (ci, mdsc,CEPH_FILE_MODE_WR ); +/* can we use the dcache? */ if (ceph_test_mount_opt(fsc, DCACHE) && !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR &&
@@ -760,6 +763,7 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, ceph_test_mount_opt(fsc, DCACHE) && __ceph_dir_is_complete(ci) && (__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) { +__ceph_touch_fmode (ci, mdsc,CEPH_FILE_MODE_RD );spin_unlock (&ci->i_ceph_lock);dout (" dir %p complete, -ENOENT\n" , dir);d_add (dentry,NULL );
@@ -1621,7 +1625,8 @@ static int __dir_lease_try_check(const struct dentry *dentry)/* * Check if directory-wide content lease/cap is valid. */ -static int dir_lease_is_valid (struct inode *dir,struct dentry *dentry) +static int dir_lease_is_valid (struct inode *dir,struct dentry *dentry, +struct ceph_mds_client *mdsc) {struct ceph_inode_info *ci =ceph_inode (dir);int valid;
@@ -1629,7 +1634,10 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry)spin_lock (&ci->i_ceph_lock); valid =__ceph_caps_issued_mask (ci,CEPH_CAP_FILE_SHARED ,1 ); - shared_gen =atomic_read (&ci->i_shared_gen); +if (valid) { +__ceph_touch_fmode (ci, mdsc,CEPH_FILE_MODE_RD ); + shared_gen =atomic_read (&ci->i_shared_gen); + }spin_unlock (&ci->i_ceph_lock);if (valid) {struct ceph_dentry_info *di;
@@ -1655,6 +1663,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)int valid =0 ;struct dentry *parent;struct inode *dir, *inode; +struct ceph_mds_client *mdsc;if (flags &LOOKUP_RCU ) { parent =READ_ONCE (dentry->d_parent);
@@ -1671,6 +1680,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)dout ("d_revalidate %p '%pd' inode %p offset 0x%llx\n" , dentry, dentry, inode,ceph_dentry (dentry)->offset); + mdsc =ceph_sb_to_client (dir->i_sb)->mdsc; +/* always trust cached snapped dentries, snapdir dentry */ if (ceph_snap (dir) !=CEPH_NOSNAP ) { dout("d_revalidate %p '%pd' inode %p is SNAPPED\n", dentry,
@@ -1682,7 +1693,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) valid =dentry_lease_is_valid (dentry, flags);if (valid == -ECHILD )return valid; -if (valid ||dir_lease_is_valid (dir, dentry)) { +if (valid ||dir_lease_is_valid (dir, dentry, mdsc)) {if (inode) valid =ceph_is_any_caps (inode);else
@@ -1691,8 +1702,6 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) }if (!valid) { -struct ceph_mds_client *mdsc = -ceph_sb_to_client (dir->i_sb)->mdsc;struct ceph_mds_request *req;int op, err;u32 mask;
fs/ceph/mds_client.c +9 -2
@@ -2486,7 +2486,8 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,if (req->r_inode_drop) releases +=ceph_encode_inode_release (&p, req->r_inode ? req->r_inode :d_inode (req->r_dentry), - mds, req->r_inode_drop, req->r_inode_unless,0 ); + mds, req->r_inode_drop, req->r_inode_unless, + req->r_op ==CEPH_MDS_OP_READDIR );if (req->r_dentry_drop) releases += ceph_encode_dentry_release(&p, req->r_dentry, req->r_parent, mds, req->r_dentry_drop,
@@ -2833,7 +2834,13 @@ int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir,if (req->r_inode)ceph_get_cap_refs (ceph_inode (req->r_inode),CEPH_CAP_PIN );if (req->r_parent) { -ceph_get_cap_refs (ceph_inode (req->r_parent),CEPH_CAP_PIN ); +struct ceph_inode_info *ci =ceph_inode (req->r_parent); +int fmode = (req->r_op &CEPH_MDS_OP_WRITE ) ? +CEPH_FILE_MODE_WR :CEPH_FILE_MODE_RD ; +spin_lock (&ci->i_ceph_lock); +ceph_take_cap_refs (ci,CEPH_CAP_PIN , false); +__ceph_touch_fmode (ci, mdsc, fmode); +spin_unlock (&ci->i_ceph_lock);ihold (req->r_parent); }if (req->r_old_dentry_dir)
Signature · 1 line
2.21.1
Quoted 28 lines · Zheng
This series make cephfs client not request caps for open files that idle for a long time. For the case that one active client and multiple standby clients open the same file, this increase the possibility that mds issues exclusive caps to the active client.
Yan, Zheng (7):
ceph: always renew caps ifmds_wantedis insufficient
ceph: consider inode's last read/write when calculating wanted caps
ceph: remove delay check logic fromceph_check_caps()
ceph: simplify calling ofceph_get_fmode()
ceph: updatei_requested_max_sizeonly when sending cap msg to auth mds
ceph: check all mds' caps after page writeback
ceph: calculate dir's wanted caps according to recent diropsfs/ceph/caps.c | 360 ++++++++++++++++------------------- fs/ceph/dir.c | 21 +- fs/ceph/file.c | 45 ++--- fs/ceph/inode.c | 21 +- fs/ceph/ioctl.c | 2 + fs/ceph/mds_client.c | 16 +- fs/ceph/super.h | 37 ++-- include/linux/ceph/ceph_fs.h | 1 + 8 files changed, 243 insertions(+), 260 deletions(-)changes since v2
- make__ceph_caps_file_wanted()more readable
- add patch 5 and 6, which fix hung write during testing patch 1~4
Quoted 8 lines
changes since v3
- don't queue delayed cap check for snap inode
- initialize ci->{last_rd,last_wr} to jiffies - 3600 * HZ
- make__ceph_caps_file_wanted()check inode typechanges since v4
- add patch 7, improve how to calculate dir's wanted caps
Thanks Zheng. This one seems to work just fine. Merged into ceph- client/testing branch with the following changes:
- squashed patch 7 into patch 2
- cleaned up a few changelog grammatical and spelling errors
- fix a small bit of whitespace damage
Thanks!
Signature · 1 line
Jeff Layton <[email protected]>