ceph: metrics for opened files, pinned caps and opened inodes
v3 cover ceph: metrics for opened files, pinned caps and opened inodes
From: Xiubo Li <[email protected]>
Changed in V3:
- Resend for V2 just forgot one patch, which is adding some helpers support to simplify the code.
Changed in V2:
- Add number of inodes that have opened files.
- Remove the dir metrics and fold into files.
Xiubo Li (2):
ceph: add helpers for parsing inode/ci/sb to mdsc
ceph: metrics for opened files, pinned caps and opened inodes
fs/ceph/caps.c | 30 ++++++++++++++++++++++++++---- fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/dir.c | 22 ++++++++-------------- fs/ceph/file.c | 17 ++++++++--------- fs/ceph/inode.c | 17 +++++++++++------ fs/ceph/ioctl.c | 6 +++--- fs/ceph/locks.c | 2 +- fs/ceph/metric.c | 14 ++++++++++++++ fs/ceph/metric.h | 7 +++++++ fs/ceph/quota.c | 8 ++++---- fs/ceph/snap.c | 2 +- fs/ceph/super.h | 25 +++++++++++++++++++++++++ fs/ceph/xattr.c | 2 +- 13 files changed, 120 insertions(+), 43 deletions(-)
Signature · 1 line
2.18.4
PATCH v3 1/2 ceph: add helpers for parsing inode/ci/sb to mdsc
From: Xiubo Li <[email protected]>
This will help simplify the code.
--- fs/ceph/caps.c | 3 +-- fs/ceph/dir.c | 22 ++++++++-------------- fs/ceph/file.c | 12 +++++------- fs/ceph/inode.c | 10 ++++------ fs/ceph/ioctl.c | 6 +++--- fs/ceph/locks.c | 2 +- fs/ceph/quota.c | 8 ++++---- fs/ceph/snap.c | 2 +- fs/ceph/super.h | 24 ++++++++++++++++++++++++ fs/ceph/xattr.c | 2 +- 10 files changed, 52 insertions(+), 39 deletions(-)
fs/ceph/caps.c +1 -2
@@ -1906,8 +1906,7 @@ bool __ceph_should_report_size(struct ceph_inode_info *ci)void ceph_check_caps (struct ceph_inode_info *ci,int flags,struct ceph_mds_session *session) { -struct ceph_fs_client *fsc =ceph_inode_to_client (&ci->vfs_inode); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);struct inode *inode = &ci->vfs_inode;struct ceph_cap *cap;u64 flush_tid, oldest_flush_tid;
fs/ceph/dir.c +8 -14
@@ -38,8 +38,7 @@ static int __dir_lease_try_check(const struct dentry *dentry);static int ceph_d_init (struct dentry *dentry) {struct ceph_dentry_info *di; -struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (dentry->d_sb); di =kmem_cache_zalloc (ceph_dentry_cachep,GFP_KERNEL );if (!di)
@@ -311,7 +310,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);int i;int err;unsigned frag =-1 ;
@@ -743,7 +742,7 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) {struct ceph_fs_client *fsc =ceph_sb_to_client (dir->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (dir->i_sb);struct ceph_mds_request *req;int op;int mask;
@@ -832,8 +831,7 @@ int ceph_handle_notrace_create(struct inode *dir, struct dentry *dentry)static int ceph_mknod (struct inode *dir,struct dentry *dentry,umode_t mode,dev_t rdev) { -struct ceph_fs_client *fsc =ceph_sb_to_client (dir->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (dir->i_sb);struct ceph_mds_request *req;struct ceph_acl_sec_ctx as_ctx = {};int err;
@@ -894,8 +892,7 @@ static int ceph_create(struct inode *dir, struct dentry *dentry, umode_t mode,static int ceph_symlink (struct inode *dir,struct dentry *dentry,const char *dest) { -struct ceph_fs_client *fsc =ceph_sb_to_client (dir->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (dir->i_sb);struct ceph_mds_request *req;struct ceph_acl_sec_ctx as_ctx = {};int err;
@@ -947,8 +944,7 @@ static int ceph_symlink(struct inode *dir, struct dentry *dentry,static int ceph_mkdir (struct inode *dir,struct dentry *dentry,umode_t mode) { -struct ceph_fs_client *fsc =ceph_sb_to_client (dir->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (dir->i_sb);struct ceph_mds_request *req;struct ceph_acl_sec_ctx as_ctx = {};int err = -EROFS ;
@@ -1015,8 +1011,7 @@ static int ceph_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)static int ceph_link (struct dentry *old_dentry,struct inode *dir,struct dentry *dentry) { -struct ceph_fs_client *fsc =ceph_sb_to_client (dir->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (dir->i_sb);struct ceph_mds_request *req;int err;
@@ -1197,8 +1192,7 @@ static int ceph_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { - struct ceph_fs_client *fsc = ceph_sb_to_client(old_dir->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(old_dir->i_sb);struct ceph_mds_request *req;int op =CEPH_MDS_OP_RENAME ;int err;
fs/ceph/file.c +5 -7
@@ -182,8 +182,7 @@ static void put_bvecs(struct bio_vec *bvecs, int num_bvecs, bool should_dirty)static struct ceph_mds_request *prepare_open_request (struct super_block *sb,int flags,int create_mode) { -struct ceph_fs_client *fsc =ceph_sb_to_client (sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (sb);struct ceph_mds_request *req;int want_auth =USE_ANY_MDS ;int op = (flags &O_CREAT ) ?CEPH_MDS_OP_CREATE :CEPH_MDS_OP_OPEN ;
@@ -283,7 +282,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode)*/ int ceph_renew_caps (struct inode *inode,int fmode) { -struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_mds_request *req;int err, flags, wanted;
@@ -340,8 +339,7 @@ int ceph_renew_caps(struct inode *inode, int fmode)int ceph_open (struct inode *inode,struct file *file) {struct ceph_inode_info *ci =ceph_inode (inode); -struct ceph_fs_client *fsc =ceph_sb_to_client (inode->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_mds_request *req;struct ceph_file_info *fi = file->private_data;int err;
@@ -1050,8 +1048,8 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); -struct ceph_fs_client *fsc =ceph_inode_to_client (inode); -struct ceph_client_metric *metric = &fsc->mdsc->metric; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode); +struct ceph_client_metric *metric = &mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs);
fs/ceph/inode.c +4 -6
@@ -559,8 +559,7 @@ void ceph_evict_inode(struct inode *inode)* caps in i_snap_caps. */ if (ci->i_snap_realm) { -struct ceph_mds_client *mdsc = -ceph_inode_to_client (inode)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);if (ceph_snap (inode) ==CEPH_NOSNAP ) {struct ceph_snap_realm *realm = ci->i_snap_realm; dout(" dropping residual ref to snap realm %p\n",
@@ -740,7 +739,7 @@ int ceph_fill_inode(struct inode *inode, struct page *locked_page, struct ceph_mds_session *session, int cap_fmode, struct ceph_cap_reservation *caps_reservation) { - struct ceph_mds_client *mdsc = ceph_inode_to_client(inode)->mdsc; + struct ceph_mds_client *mdsc = ceph_inode_to_mdsc(inode);struct ceph_mds_reply_inode *info = iinfo->in;struct ceph_inode_info *ci =ceph_inode (inode);int issued, new_issued, info_caps;
@@ -2027,7 +2026,7 @@ int __ceph_setattr(struct inode *inode, struct iattr *attr)struct ceph_inode_info *ci =ceph_inode (inode);unsigned int ia_valid = attr->ia_valid;struct ceph_mds_request *req; -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_cap_flush *prealloc_cf;int issued;int release =0 , dirtied =0 ;
@@ -2275,8 +2274,7 @@ int ceph_setattr(struct dentry *dentry, struct iattr *attr)int __ceph_do_getattr (struct inode *inode,struct page *locked_page,int mask,bool force) { -struct ceph_fs_client *fsc =ceph_sb_to_client (inode->i_sb); -struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_mds_request *req;int mode;int err;
fs/ceph/ioctl.c +3 -3
@@ -64,7 +64,7 @@ static long __validate_layout(struct ceph_mds_client *mdsc,static long ceph_ioctl_set_layout (struct file *file,void __user *arg) {struct inode *inode =file_inode (file); -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_mds_request *req;struct ceph_ioctl_layout l;struct ceph_inode_info *ci =ceph_inode (file_inode (file));
@@ -139,7 +139,7 @@ static long ceph_ioctl_set_layout_policy (struct file *file, void __user *arg)struct ceph_mds_request *req;struct ceph_ioctl_layout l;int err; -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);/* copy and validate */ if (copy_from_user (&l, arg,sizeof (l)))
@@ -243,7 +243,7 @@ static long ceph_ioctl_lazyio(struct file *file)struct ceph_file_info *fi = file->private_data;struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode); -struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);if ((fi->fmode &CEPH_FILE_MODE_LAZY ) ==0 ) {spin_lock (&ci->i_ceph_lock);
fs/ceph/locks.c +1 -1
@@ -63,7 +63,7 @@ static const struct file_lock_operations ceph_fl_lock_ops = {static int ceph_lock_message (u8 lock_type,u16 operation,struct inode *inode,int cmd,u8 wait,struct file_lock *fl) { -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_mds_request *req;int err;u64 length =0 ;
fs/ceph/quota.c +4 -4
@@ -12,7 +12,7 @@void ceph_adjust_quota_realms_count (struct inode *inode,bool inc) { -struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);if (inc)atomic64_inc (&mdsc->quotarealms_count);else
@@ -21,7 +21,7 @@ void ceph_adjust_quota_realms_count(struct inode *inode, bool inc)static inline bool ceph_has_realms_with_quotas (struct inode *inode) { -struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct super_block *sb = mdsc->fsc->sb;if (atomic64_read (&mdsc->quotarealms_count) >0 )
@@ -266,7 +266,7 @@ static struct ceph_snap_realm *get_quota_realm(struct ceph_mds_client *mdsc,static bool ceph_quota_is_same_realm (struct inode *old,struct inode *new) { -struct ceph_mds_client *mdsc =ceph_inode_to_client (old)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (old);struct ceph_snap_realm *old_realm, *new_realm;bool is_same;
@@ -313,7 +313,7 @@ enum quota_check_op {static bool check_quota_exceeded (struct inode *inode,enum quota_check_op op,loff_t delta) { -struct ceph_mds_client *mdsc =ceph_inode_to_client (inode)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_inode_info *ci;struct ceph_snap_realm *realm, *next;struct inode *in;
fs/ceph/snap.c +1 -1
@@ -602,7 +602,7 @@ int __ceph_finish_cap_snap(struct ceph_inode_info *ci, struct ceph_cap_snap *capsnap) {struct inode *inode = &ci->vfs_inode; -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);BUG_ON (capsnap->writing); capsnap->size = inode->i_size;
fs/ceph/super.h +24 -0
@@ -451,6 +451,30 @@ ceph_sb_to_client(const struct super_block *sb)return (struct ceph_fs_client *)sb->s_fs_info; } +static inline struct ceph_fs_client * +ceph_ci_to_client (const struct ceph_inode_info *ci) +{ +return (struct ceph_fs_client *)ci->vfs_inode.i_sb->s_fs_info; +} + +static inline struct ceph_mds_client * +ceph_inode_to_mdsc (const struct inode *inode) +{ +return (struct ceph_mds_client *)ceph_inode_to_client (inode)->mdsc; +} + +static inline struct ceph_mds_client * +ceph_sb_to_mdsc (const struct super_block *sb) +{ +return (struct ceph_mds_client *)ceph_sb_to_client (sb)->mdsc; +} + +static inline struct ceph_mds_client * +ceph_ci_to_mdsc (const struct ceph_inode_info *ci) +{ +return (struct ceph_mds_client *)ceph_ci_to_client (ci)->mdsc; +} +static inline struct ceph_vino ceph_vino (const struct inode *inode) {
fs/ceph/xattr.c +1 -1
@@ -1012,7 +1012,7 @@ int __ceph_setxattr(struct inode *inode, const char *name, {struct ceph_vxattr *vxattr;struct ceph_inode_info *ci =ceph_inode (inode); -struct ceph_mds_client *mdsc =ceph_sb_to_client (inode->i_sb)->mdsc; +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_cap_flush *prealloc_cf =NULL ;struct ceph_buffer *old_blob =NULL ;int issued;
Signature · 1 line
2.18.4
PATCH v3 2/2 ceph: metrics for opened files, pinned caps and opened inodes
In reply to [email protected]From: Xiubo Li <[email protected]>
In client for each inode, it may have many opened files and may have been pinned in more than one MDS servers. And some inodes are idle, which have no any opened files.
This patch will show these metrics in the debugfs, likes:
item total
-----------------------------------------
opened files / total inodes 14 / 5 pinned i_caps / total inodes 7 / 5 opened inodes / total inodes 3 / 5
Will send these metrics to ceph, which will be used by the fs top, later.
URL: https://tracker.ceph.com/issues/47005
--- fs/ceph/caps.c | 27 +++++++++++++++++++++++++-- fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/file.c | 5 +++-- fs/ceph/inode.c | 7 +++++++ fs/ceph/metric.c | 14 ++++++++++++++ fs/ceph/metric.h | 7 +++++++ fs/ceph/super.h | 1 + 7 files changed, 68 insertions(+), 4 deletions(-)
fs/ceph/caps.c +25 -2
@@ -4283,13 +4283,23 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci,void ceph_get_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);int bits = (fmode <<1 ) |1 ; +int i; + +if (count ==1 ) +atomic64_inc (&mdsc->metric.opened_files); +spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) ci->i_nr_by_mode[i] += count; } + +if (!ci->is_opened && fmode) { + ci->is_opened = true; +percpu_counter_inc (&mdsc->metric.opened_inodes); + }spin_unlock (&ci->i_ceph_lock); }
@@ -4300,15 +4310,28 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)*/ void ceph_put_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);int bits = (fmode <<1 ) |1 ; +bool empty = true; +int i; + +if (count ==1 ) +atomic64_dec (&mdsc->metric.opened_files); +spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) {BUG_ON (ci->i_nr_by_mode[i] < count); ci->i_nr_by_mode[i] -= count; +if (ci->i_nr_by_mode[i] && i)/* Skip the pin ref */ + empty = false; } } + +if (ci->is_opened && empty && fmode) { + ci->is_opened = false; +percpu_counter_dec (&mdsc->metric.opened_inodes); + }spin_unlock (&ci->i_ceph_lock); }
fs/ceph/debugfs.c +11 -0
@@ -148,6 +148,17 @@ static int metric_show(struct seq_file *s, void *p)int nr_caps =0 ;s64 total, sum, avg, min, max, sq; + sum =percpu_counter_sum (&m->total_inodes); +seq_printf (s,"item total\n" ); +seq_printf (s,"------------------------------------------\n" ); +seq_printf (s,"%-35s%lld / %lld\n" ,"opened files / total inodes" , +atomic64_read (&m->opened_files), sum); +seq_printf (s,"%-35s%lld / %lld\n" ,"pinned i_caps / total inodes" , +atomic64_read (&m->total_caps), sum); +seq_printf (s,"%-35s%lld / %lld\n" ,"opened inodes / total inodes" , +percpu_counter_sum (&m->opened_inodes), sum); + +seq_printf (s,"\n" );seq_printf (s,"item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n" );seq_printf (s,"-----------------------------------------------------------------------------------\n" );
fs/ceph/file.c +3 -2
@@ -211,8 +211,9 @@ static int ceph_init_file_info(struct inode *inode, struct file *file,BUG_ON (inode->i_fop->release != ceph_release);if (isdir) { -struct ceph_dir_file_info *dfi = -kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL ); +struct ceph_dir_file_info *dfi; + + dfi =kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL );if (!dfi)return -ENOMEM ;
fs/ceph/inode.c +7 -0
@@ -426,6 +426,7 @@ static int ceph_fill_fragtree(struct inode *inode,*/ struct inode *ceph_alloc_inode (struct super_block *sb) { +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (sb);struct ceph_inode_info *ci;int i;
@@ -485,6 +486,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_last_rd = ci->i_last_wr = jiffies -3600 *HZ ;for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) ci->i_nr_by_mode[i] =0 ; + ci->is_opened = false;mutex_init (&ci->i_truncate_mutex); ci->i_truncate_seq =0 ;
@@ -525,6 +527,8 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_meta_err =0 ; +percpu_counter_inc (&mdsc->metric.total_inodes); +return &ci->vfs_inode; }
@@ -539,6 +543,7 @@ void ceph_free_inode(struct inode *inode)void ceph_evict_inode (struct inode *inode) {struct ceph_inode_info *ci =ceph_inode (inode); +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_inode_frag *frag;struct rb_node *n;
@@ -592,6 +597,8 @@ void ceph_evict_inode(struct inode *inode)ceph_put_string (rcu_dereference_raw (ci->i_layout.pool_ns));ceph_put_string (rcu_dereference_raw (ci->i_cached_layout.pool_ns)); + +percpu_counter_dec (&mdsc->metric.total_inodes); }static inline blkcnt_t calc_inode_blocks (u64 size)
fs/ceph/metric.c +14 -0
@@ -192,11 +192,23 @@ int ceph_metric_init(struct ceph_client_metric *m) m->total_metadatas =0 ; m->metadata_latency_sum =0 ; +atomic64_set (&m->opened_files,0 ); + ret =percpu_counter_init (&m->opened_inodes,0 ,GFP_KERNEL ); +if (ret) + goto err_opened_inodes; + ret =percpu_counter_init (&m->opened_inodes,0 ,GFP_KERNEL ); +if (ret) + goto err_total_inodes; + m->session =NULL ;INIT_DELAYED_WORK (&m->delayed_work, metric_delayed_work);return 0 ; +err_total_inodes: +percpu_counter_destroy (&m->opened_inodes); +err_opened_inodes: +percpu_counter_destroy (&m->i_caps_mis); err_i_caps_mis:percpu_counter_destroy (&m->i_caps_hit); err_i_caps_hit:
@@ -212,6 +224,8 @@ void ceph_metric_destroy(struct ceph_client_metric *m)if (!m)return ; +percpu_counter_destroy (&m->total_inodes); +percpu_counter_destroy (&m->opened_inodes);percpu_counter_destroy (&m->i_caps_mis);percpu_counter_destroy (&m->i_caps_hit);percpu_counter_destroy (&m->d_lease_mis);
fs/ceph/metric.h +7 -0
@@ -115,6 +115,13 @@ struct ceph_client_metric {ktime_t metadata_latency_min;ktime_t metadata_latency_max; +/* The total number of directories and files that are opened */ +atomic64_t opened_files; + +/* The total number of inodes that have opened files or directories */ +struct percpu_counter opened_inodes; +struct percpu_counter total_inodes; +struct ceph_mds_session *session;struct delayed_work delayed_work;/* delayed work */ };
fs/ceph/super.h +1 -0
@@ -387,6 +387,7 @@ struct ceph_inode_info {unsigned long i_last_rd;unsigned long i_last_wr;int i_nr_by_mode[CEPH_FILE_MODE_BITS ];/* open file counts */ +bool is_opened;/* has opened files or directors */ struct mutex i_truncate_mutex;u32 i_truncate_seq;/* last truncate to smaller size */
Signature · 1 line
2.18.4
Quoted 164 lines · On Mon, 2020-08-24 at 20:54 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
In client for each inode, it may have many opened files and may have been pinned in more than one MDS servers. And some inodes are idle, which have no any opened files.
This patch will show these metrics in the debugfs, likes:
item total
-----------------------------------------opened files / total inodes 14 / 5 pinned i_caps / total inodes 7 / 5 opened inodes / total inodes 3 / 5Will send these metrics to ceph, which will be used by the
fs top, later.URL: https://tracker.ceph.com/issues/47005
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/caps.c | 27 +++++++++++++++++++++++++-- fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/file.c | 5 +++-- fs/ceph/inode.c | 7 +++++++ fs/ceph/metric.c | 14 ++++++++++++++ fs/ceph/metric.h | 7 +++++++ fs/ceph/super.h | 1 + 7 files changed, 68 insertions(+), 4 deletions(-)fs/ceph/caps.c +25
-2@@ -4283,13 +4283,23 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci,void ceph_get_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);int bits = (fmode <<1 ) |1 ; +int i; + +if (count ==1 ) +atomic64_inc (&mdsc->metric.opened_files); +spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) ci->i_nr_by_mode[i] += count; } + +if (!ci->is_opened && fmode) { + ci->is_opened = true; +percpu_counter_inc (&mdsc->metric.opened_inodes); + }spin_unlock (&ci->i_ceph_lock); }@@ -4300,15 +4310,28 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)*/ void ceph_put_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);int bits = (fmode <<1 ) |1 ; +bool empty = true; +int i; + +if (count ==1 ) +atomic64_dec (&mdsc->metric.opened_files); +spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) {BUG_ON (ci->i_nr_by_mode[i] < count); ci->i_nr_by_mode[i] -= count; +if (ci->i_nr_by_mode[i] && i)/* Skip the pin ref */ + empty = false; } } + +if (ci->is_opened && empty && fmode) { + ci->is_opened = false; +percpu_counter_dec (&mdsc->metric.opened_inodes); + }spin_unlock (&ci->i_ceph_lock); }fs/ceph/debugfs.c +11
-0@@ -148,6 +148,17 @@ static int metric_show(struct seq_file *s, void *p)int nr_caps =0 ;s64 total, sum, avg, min, max, sq; + sum =percpu_counter_sum (&m->total_inodes); +seq_printf (s,"item total\n" ); +seq_printf (s,"------------------------------------------\n" ); +seq_printf (s,"%-35s%lld / %lld\n" ,"opened files / total inodes" , +atomic64_read (&m->opened_files), sum); +seq_printf (s,"%-35s%lld / %lld\n" ,"pinned i_caps / total inodes" , +atomic64_read (&m->total_caps), sum); +seq_printf (s,"%-35s%lld / %lld\n" ,"opened inodes / total inodes" , +percpu_counter_sum (&m->opened_inodes), sum); + +seq_printf (s,"\n" );seq_printf (s,"item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n" );seq_printf (s,"-----------------------------------------------------------------------------------\n" );fs/ceph/file.c +3
-2@@ -211,8 +211,9 @@ static int ceph_init_file_info(struct inode *inode, struct file *file,BUG_ON (inode->i_fop->release != ceph_release);if (isdir) { -struct ceph_dir_file_info *dfi = -kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL ); +struct ceph_dir_file_info *dfi; + + dfi =kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL );if (!dfi)return -ENOMEM ;fs/ceph/inode.c +5
-0@@ -426,6 +426,7 @@ static int ceph_fill_fragtree(struct inode *inode,*/ struct inode *ceph_alloc_inode (struct super_block *sb) { +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (sb);struct ceph_inode_info *ci;int i;@@ -485,6 +486,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_last_rd = ci->i_last_wr = jiffies -3600 *HZ ;for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) ci->i_nr_by_mode[i] =0 ; + ci->is_opened = false;mutex_init (&ci->i_truncate_mutex); ci->i_truncate_seq =0 ;@@ -525,6 +527,8 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_meta_err =0 ; +percpu_counter_inc (&mdsc->metric.total_inodes); +return &ci->vfs_inode; }@@ -539,6 +543,7 @@ void ceph_free_inode(struct inode *inode)void ceph_evict_inode (struct inode *inode) {struct ceph_inode_info *ci =ceph_inode (inode); +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_inode_frag *frag;struct rb_node *n;
fs/ceph/inode.c void ceph_evict_inode(struct inode *inode) · line 601
Quoted 5 lines
fs/ceph/inode.c +2
-0@@ -592,6 +597,8 @@ void ceph_evict_inode(struct inode *inode)ceph_put_string (rcu_dereference_raw (ci->i_layout.pool_ns));ceph_put_string (rcu_dereference_raw (ci->i_cached_layout.pool_ns)); + +percpu_counter_dec (&mdsc->metric.total_inodes);
So you're counting every allocated inode but decrementing the count on eviction. What about inodes that never end up being hashed? They'll increment the count, but it'll never be decremented. This should probably be done in ceph_free_inode.
Quoted 9 lines
fs/ceph/inode.c +0
-0}static inline blkcnt_t calc_inode_blocks (u64 size)fs/ceph/metric.c +0
-0@@ -192,11 +192,23 @@ int ceph_metric_init(struct ceph_client_metric *m) m->total_metadatas =0 ;
fs/ceph/metric.c int ceph_metric_init(struct ceph_client_metric *m) · line 202
Quoted 10 lines
fs/ceph/metric.c +8
-0m->metadata_latency_sum =0 ; +atomic64_set (&m->opened_files,0 ); + ret =percpu_counter_init (&m->opened_inodes,0 ,GFP_KERNEL ); +if (ret) + goto err_opened_inodes; + ret =percpu_counter_init (&m->opened_inodes,0 ,GFP_KERNEL ); +if (ret) + goto err_total_inodes; +
Shouldn't the above be initializing total_inodes? In fact, when I tried to give this a test drive, the kernel crashed here with list corruption...
Quoted 51 lines
fs/ceph/metric.c +6
-0m->session =NULL ;INIT_DELAYED_WORK (&m->delayed_work, metric_delayed_work);return 0 ; +err_total_inodes: +percpu_counter_destroy (&m->opened_inodes); +err_opened_inodes: +percpu_counter_destroy (&m->i_caps_mis); err_i_caps_mis:percpu_counter_destroy (&m->i_caps_hit); err_i_caps_hit:@@ -212,6 +224,8 @@ void ceph_metric_destroy(struct ceph_client_metric *m)if (!m)return ; +percpu_counter_destroy (&m->total_inodes); +percpu_counter_destroy (&m->opened_inodes);percpu_counter_destroy (&m->i_caps_mis);percpu_counter_destroy (&m->i_caps_hit);percpu_counter_destroy (&m->d_lease_mis);fs/ceph/metric.h +7
-0@@ -115,6 +115,13 @@ struct ceph_client_metric {ktime_t metadata_latency_min;ktime_t metadata_latency_max; +/* The total number of directories and files that are opened */ +atomic64_t opened_files; + +/* The total number of inodes that have opened files or directories */ +struct percpu_counter opened_inodes; +struct percpu_counter total_inodes; +struct ceph_mds_session *session;struct delayed_work delayed_work;/* delayed work */ };fs/ceph/super.h +1
-0@@ -387,6 +387,7 @@ struct ceph_inode_info {unsigned long i_last_rd;unsigned long i_last_wr;int i_nr_by_mode[CEPH_FILE_MODE_BITS ];/* open file counts */ +bool is_opened;/* has opened files or directors */ struct mutex i_truncate_mutex;u32 i_truncate_seq;/* last truncate to smaller size */
Signature · 1 line
Jeff Layton <[email protected]>
Quoted 166 lines · Jeff Layton
Quoted 165 lines · On Mon, 2020-08-24 at 20:54 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
In client for each inode, it may have many opened files and may have been pinned in more than one MDS servers. And some inodes are idle, which have no any opened files.
This patch will show these metrics in the debugfs, likes:
item total
-----------------------------------------opened files / total inodes 14 / 5 pinned i_caps / total inodes 7 / 5 opened inodes / total inodes 3 / 5Will send these metrics to ceph, which will be used by the
fs top, later.URL: https://tracker.ceph.com/issues/47005
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/caps.c | 27 +++++++++++++++++++++++++-- fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/file.c | 5 +++-- fs/ceph/inode.c | 7 +++++++ fs/ceph/metric.c | 14 ++++++++++++++ fs/ceph/metric.h | 7 +++++++ fs/ceph/super.h | 1 + 7 files changed, 68 insertions(+), 4 deletions(-)fs/ceph/caps.c +25
-2@@ -4283,13 +4283,23 @@ void __ceph_touch_fmode(struct ceph_inode_info *ci,void ceph_get_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);int bits = (fmode <<1 ) |1 ; +int i; + +if (count ==1 ) +atomic64_inc (&mdsc->metric.opened_files); +spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) ci->i_nr_by_mode[i] += count; } + +if (!ci->is_opened && fmode) { + ci->is_opened = true; +percpu_counter_inc (&mdsc->metric.opened_inodes); + }spin_unlock (&ci->i_ceph_lock); }@@ -4300,15 +4310,28 @@ void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count)*/ void ceph_put_fmode (struct ceph_inode_info *ci,int fmode,int count) { -int i; +struct ceph_mds_client *mdsc =ceph_ci_to_mdsc (ci);int bits = (fmode <<1 ) |1 ; +bool empty = true; +int i; + +if (count ==1 ) +atomic64_dec (&mdsc->metric.opened_files); +spin_lock (&ci->i_ceph_lock);for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) {if (bits & (1 << i)) {BUG_ON (ci->i_nr_by_mode[i] < count); ci->i_nr_by_mode[i] -= count; +if (ci->i_nr_by_mode[i] && i)/* Skip the pin ref */ + empty = false; } } + +if (ci->is_opened && empty && fmode) { + ci->is_opened = false; +percpu_counter_dec (&mdsc->metric.opened_inodes); + }spin_unlock (&ci->i_ceph_lock); }fs/ceph/debugfs.c +11
-0@@ -148,6 +148,17 @@ static int metric_show(struct seq_file *s, void *p)int nr_caps =0 ;s64 total, sum, avg, min, max, sq; + sum =percpu_counter_sum (&m->total_inodes); +seq_printf (s,"item total\n" ); +seq_printf (s,"------------------------------------------\n" ); +seq_printf (s,"%-35s%lld / %lld\n" ,"opened files / total inodes" , +atomic64_read (&m->opened_files), sum); +seq_printf (s,"%-35s%lld / %lld\n" ,"pinned i_caps / total inodes" , +atomic64_read (&m->total_caps), sum); +seq_printf (s,"%-35s%lld / %lld\n" ,"opened inodes / total inodes" , +percpu_counter_sum (&m->opened_inodes), sum); + +seq_printf (s,"\n" );seq_printf (s,"item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n" );seq_printf (s,"-----------------------------------------------------------------------------------\n" );fs/ceph/file.c +3
-2@@ -211,8 +211,9 @@ static int ceph_init_file_info(struct inode *inode, struct file *file,BUG_ON (inode->i_fop->release != ceph_release);if (isdir) { -struct ceph_dir_file_info *dfi = -kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL ); +struct ceph_dir_file_info *dfi; + + dfi =kmem_cache_zalloc (ceph_dir_file_cachep,GFP_KERNEL );if (!dfi)return -ENOMEM ;fs/ceph/inode.c +5
-0@@ -426,6 +426,7 @@ static int ceph_fill_fragtree(struct inode *inode,*/ struct inode *ceph_alloc_inode (struct super_block *sb) { +struct ceph_mds_client *mdsc =ceph_sb_to_mdsc (sb);struct ceph_inode_info *ci;int i;@@ -485,6 +486,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_last_rd = ci->i_last_wr = jiffies -3600 *HZ ;for (i =0 ; i <CEPH_FILE_MODE_BITS ; i++) ci->i_nr_by_mode[i] =0 ; + ci->is_opened = false;mutex_init (&ci->i_truncate_mutex); ci->i_truncate_seq =0 ;@@ -525,6 +527,8 @@ struct inode *ceph_alloc_inode(struct super_block *sb) ci->i_meta_err =0 ; +percpu_counter_inc (&mdsc->metric.total_inodes); +return &ci->vfs_inode; }@@ -539,6 +543,7 @@ void ceph_free_inode(struct inode *inode)void ceph_evict_inode (struct inode *inode) {struct ceph_inode_info *ci =ceph_inode (inode); +struct ceph_mds_client *mdsc =ceph_inode_to_mdsc (inode);struct ceph_inode_frag *frag;struct rb_node *n;@@ -592,6 +597,8 @@ void ceph_evict_inode(struct inode *inode)
Quoted 9 lines
fs/ceph/inode.c void ceph_evict_inode(struct inode *inode) · line 601
Quoted 4 lines
fs/ceph/inode.c +2
-0ceph_put_string (rcu_dereference_raw (ci->i_layout.pool_ns));ceph_put_string (rcu_dereference_raw (ci->i_cached_layout.pool_ns)); + +percpu_counter_dec (&mdsc->metric.total_inodes);So you're counting every allocated inode but decrementing the count on eviction. What about inodes that never end up being hashed? They'll increment the count, but it'll never be decremented. This should probably be done in
ceph_free_inode.
Okay, I will fix it.
Quoted 11 lines
Quoted 11 lines
fs/ceph/inode.c +0
-0}static inline blkcnt_t calc_inode_blocks (u64 size)fs/ceph/metric.c +0
-0@@ -192,11 +192,23 @@ int ceph_metric_init(struct ceph_client_metric *m) m->total_metadatas =0 ; m->metadata_latency_sum =0 ;
Quoted 11 lines
fs/ceph/metric.c int ceph_metric_init(struct ceph_client_metric *m) · line 202
Quoted 8 lines
fs/ceph/metric.c +8
-0+atomic64_set (&m->opened_files,0 ); + ret =percpu_counter_init (&m->opened_inodes,0 ,GFP_KERNEL ); +if (ret) + goto err_opened_inodes; + ret =percpu_counter_init (&m->opened_inodes,0 ,GFP_KERNEL ); +if (ret) + goto err_total_inodes; +Shouldn't the above be initializing
total_inodes? In fact, when I tried to give this a test drive, the kernel crashed here with list corruption...
Yeah, my fault, I forgot to change it.
Thanks
BRs
Quoted 51 lines
Quoted 51 lines
fs/ceph/metric.c +6
-0m->session =NULL ;INIT_DELAYED_WORK (&m->delayed_work, metric_delayed_work);return 0 ; +err_total_inodes: +percpu_counter_destroy (&m->opened_inodes); +err_opened_inodes: +percpu_counter_destroy (&m->i_caps_mis); err_i_caps_mis:percpu_counter_destroy (&m->i_caps_hit); err_i_caps_hit:@@ -212,6 +224,8 @@ void ceph_metric_destroy(struct ceph_client_metric *m)if (!m)return ; +percpu_counter_destroy (&m->total_inodes); +percpu_counter_destroy (&m->opened_inodes);percpu_counter_destroy (&m->i_caps_mis);percpu_counter_destroy (&m->i_caps_hit);percpu_counter_destroy (&m->d_lease_mis);fs/ceph/metric.h +7
-0@@ -115,6 +115,13 @@ struct ceph_client_metric {ktime_t metadata_latency_min;ktime_t metadata_latency_max; +/* The total number of directories and files that are opened */ +atomic64_t opened_files; + +/* The total number of inodes that have opened files or directories */ +struct percpu_counter opened_inodes; +struct percpu_counter total_inodes; +struct ceph_mds_session *session;struct delayed_work delayed_work;/* delayed work */ };fs/ceph/super.h +1
-0@@ -387,6 +387,7 @@ struct ceph_inode_info {unsigned long i_last_rd;unsigned long i_last_wr;int i_nr_by_mode[CEPH_FILE_MODE_BITS ];/* open file counts */ +bool is_opened;/* has opened files or directors */ struct mutex i_truncate_mutex;u32 i_truncate_seq;/* last truncate to smaller size */