ceph: add perf metrics support
Subsystems (2)
v9 cover ceph: add perf metrics support
From: Xiubo Li <[email protected]>
Changed in V9:
- add an r_ended field to the mds request struct and use that to calculate the metric
- fix some commit comments
We can get the metrics from the debugfs:
$ cat /sys/kernel/debug/ceph/0c93a60d-5645-4c46-8568-4c8f63db4c7f.client4267/metrics item total sum_lat(us) avg_lat(us) ----------------------------------------------------- read 13 417000 32076 write 42 131205000 3123928 metadata 104 493000 4740
item total miss hit
-------------------------------------------------
d_lease 204 0 918 caps 204 213 368218
Xiubo Li (5):
ceph: add global dentry lease metric support
ceph: add caps perf metric for each session
ceph: add global read latency metric support
ceph: add global write latency metric support
ceph: add global metadata perf metric support
fs/ceph/acl.c | 2 +- fs/ceph/addr.c | 18 +++++++ fs/ceph/caps.c | 19 ++++++++ fs/ceph/debugfs.c | 71 ++++++++++++++++++++++++++-- fs/ceph/dir.c | 17 ++++++- fs/ceph/file.c | 26 ++++++++++ fs/ceph/inode.c | 4 +- fs/ceph/mds_client.c | 102 +++++++++++++++++++++++++++++++++++++++- fs/ceph/mds_client.h | 7 ++- fs/ceph/metric.h | 69 +++++++++++++++++++++++++++ fs/ceph/super.h | 9 ++-- fs/ceph/xattr.c | 4 +- include/linux/ceph/osd_client.h | 1 + net/ceph/osd_client.c | 2 + 14 files changed, 334 insertions(+), 17 deletions(-) create mode 100644 fs/ceph/metric.h
Signature · 1 line
1.8.3.1
PATCH v9 1/5 ceph: add global dentry lease metric support
From: Xiubo Li <[email protected]>
For the dentry lease we will only count the hit/miss info triggered from the vfs calls, for the cases like request reply handling and perodically ceph_trim_dentries() we will ignore them.
Currently only the debugfs is support:
The output will be:
item total miss hit
-------------------------------------------------d_lease 11 7 141
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 32 ++++++++++++++++++++++++++++---- fs/ceph/dir.c | 12 ++++++++++++ fs/ceph/mds_client.c | 37 +++++++++++++++++++++++++++++++++++-- fs/ceph/mds_client.h | 4 ++++ fs/ceph/metric.h | 11 +++++++++++ fs/ceph/super.h | 1 + 6 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 fs/ceph/metric.h
fs/ceph/debugfs.c +28 -4
@@ -124,6 +124,22 @@ static int mdsc_show(struct seq_file *s, void *p)return 0 ; } +static int metric_show (struct seq_file *s,void *p) +{ +struct ceph_fs_client *fsc = s->private; +struct ceph_mds_client *mdsc = fsc->mdsc; + +seq_printf (s,"item total miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +seq_printf (s,"%-14s%-16lld%-16lld%lld\n" ,"d_lease" , +atomic64_read (&mdsc->metric.total_dentries), +percpu_counter_sum (&mdsc->metric.d_lease_mis), +percpu_counter_sum (&mdsc->metric.d_lease_hit)); + +return 0 ; +} +static int caps_show_cb (struct inode *inode,struct ceph_cap *cap,void *p) {struct seq_file *s = p;
@@ -222,6 +238,7 @@ static int mds_sessions_show(struct seq_file *s, void *ptr)DEFINE_SHOW_ATTRIBUTE (mdsc);DEFINE_SHOW_ATTRIBUTE (caps);DEFINE_SHOW_ATTRIBUTE (mds_sessions); +DEFINE_SHOW_ATTRIBUTE (metric);/*
@@ -255,6 +272,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_mdsmap);debugfs_remove (fsc->debugfs_mds_sessions);debugfs_remove (fsc->debugfs_caps); +debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_mdsc); }
@@ -295,11 +313,17 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &mdsc_fops); + fsc->debugfs_metric =debugfs_create_file ("metrics" , +0400 , + fsc->client->debugfs_dir, + fsc, + &metric_fops); + fsc->debugfs_caps =debugfs_create_file ("caps" , -0400 , - fsc->client->debugfs_dir, - fsc, - &caps_fops); +0400 , + fsc->client->debugfs_dir, + fsc, + &caps_fops); }
fs/ceph/dir.c +12 -0
@@ -38,6 +38,8 @@static int ceph_d_init (struct dentry *dentry) {struct ceph_dentry_info *di; +struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb); +struct ceph_mds_client *mdsc = fsc->mdsc; di =kmem_cache_zalloc (ceph_dentry_cachep,GFP_KERNEL );if (!di)
@@ -48,6 +50,9 @@ static int ceph_d_init(struct dentry *dentry) di->time = jiffies; dentry->d_fsdata = di;INIT_LIST_HEAD (&di->lease_list); + +atomic64_inc (&mdsc->metric.total_dentries); +return 0 ; }
@@ -1709,6 +1714,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)if (flags &LOOKUP_RCU )return -ECHILD ; +percpu_counter_inc (&mdsc->metric.d_lease_mis); + op =ceph_snap (dir) ==CEPH_SNAPDIR ?CEPH_MDS_OP_LOOKUPSNAP :CEPH_MDS_OP_LOOKUP ; req =ceph_mdsc_create_request (mdsc, op,USE_ANY_MDS );
@@ -1740,6 +1747,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)dout ("d_revalidate %p lookup result=%d\n" , dentry, err); } + }else { +percpu_counter_inc (&mdsc->metric.d_lease_hit); }dout ("d_revalidate %p %s\n" , dentry, valid ?"valid" :"invalid" );
@@ -1782,9 +1791,12 @@ static int ceph_d_delete(const struct dentry *dentry)static void ceph_d_release (struct dentry *dentry) {struct ceph_dentry_info *di =ceph_dentry (dentry); +struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb);dout ("d_release %p\n" , dentry); +atomic64_dec (&fsc->mdsc->metric.total_dentries); +spin_lock (&dentry->d_lock);__dentry_lease_unlist (di); dentry->d_fsdata =NULL ;
fs/ceph/mds_client.c +35 -2
@@ -4321,10 +4321,31 @@ static void delayed_work(struct work_struct *work)schedule_delayed (mdsc); } +static int ceph_mdsc_metric_init (struct ceph_client_metric *metric) +{ +int ret; + +if (!metric) +return -EINVAL ; + +atomic64_set (&metric->total_dentries,0 ); + ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL ); +if (ret) +return ret; + ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL ); +if (ret) { +percpu_counter_destroy (&metric->d_lease_hit); +return ret; + } + +return 0 ; +} +int ceph_mdsc_init (struct ceph_fs_client *fsc) {struct ceph_mds_client *mdsc; +int err; mdsc =kzalloc (sizeof (struct ceph_mds_client ),GFP_NOFS );if (!mdsc)
@@ -4333,8 +4354,8 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)mutex_init (&mdsc->mutex); mdsc->mdsmap =kzalloc (sizeof (*mdsc->mdsmap),GFP_NOFS );if (!mdsc->mdsmap) { -kfree (mdsc); -return -ENOMEM ; + err = -ENOMEM ; + goto err_mdsc; } fsc->mdsc = mdsc;
@@ -4373,6 +4394,9 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)init_waitqueue_head (&mdsc->cap_flushing_wq);INIT_WORK (&mdsc->cap_reclaim_work, ceph_cap_reclaim_work);atomic_set (&mdsc->cap_reclaim_pending,0 ); + err =ceph_mdsc_metric_init (&mdsc->metric); +if (err) + goto err_mdsmap;spin_lock_init (&mdsc->dentry_list_lock);INIT_LIST_HEAD (&mdsc->dentry_leases);
@@ -4391,6 +4415,12 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)strscpy (mdsc->nodename,utsname ()->nodename,sizeof (mdsc->nodename));return 0 ; + +err_mdsmap: +kfree (mdsc->mdsmap); +err_mdsc: +kfree (mdsc); +return err; }/*
@@ -4648,6 +4678,9 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.d_lease_mis); +percpu_counter_destroy (&mdsc->metric.d_lease_hit); + fsc->mdsc =NULL ;kfree (mdsc);dout ("mdsc_destroy %p done\n" , mdsc);
fs/ceph/mds_client.h +4 -0
@@ -16,6 +16,8 @@#include <linux/ceph/mdsmap.h> #include <linux/ceph/auth.h> +#include "metric.h" +/* The first 8 bits are reserved for old ceph releases */ enum ceph_feature_type {CEPHFS_FEATURE_MIMIC =8 ,
@@ -454,6 +456,8 @@ struct ceph_mds_client {struct list_head dentry_leases;/* fifo list */ struct list_head dentry_dir_leases;/* lru list */ +struct ceph_client_metric metric; +spinlock_t snapid_map_lock;struct rb_root snapid_map_tree;struct list_head snapid_map_lru;
fs/ceph/metric.h new file +11 -0
@@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_MDS_METRIC_H +#define _FS_CEPH_MDS_METRIC_H + +/* This is the global metrics */ +struct ceph_client_metric { +atomic64_t total_dentries; +struct percpu_counter d_lease_hit; +struct percpu_counter d_lease_mis; +}; +#endif /* _FS_CEPH_MDS_METRIC_H */
fs/ceph/super.h +1 -0
@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
Signature · 1 line
1.8.3.1
PATCH v9 2/5 ceph: add caps perf metric for each session
In reply to [email protected]From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)
fs/ceph/acl.c +1 -1
@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);
fs/ceph/caps.c +19 -0
@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */
@@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem); +if (!ret) +ceph_update_cap_mis (&mdsc->metric); +else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;
fs/ceph/debugfs.c +16 -0
@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i, nr_caps =0 ;seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
@@ -137,6 +138,21 @@ static int metric_show(struct seq_file *s, void *p)percpu_counter_sum (&mdsc->metric.d_lease_mis), percpu_counter_sum(&mdsc->metric.d_lease_hit)); +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *s; + + s =__ceph_lookup_mds_session (mdsc, i); +if (!s) +continue ; + nr_caps += s->s_nr_caps; +ceph_put_mds_session (s); + } +mutex_unlock (&mdsc->mutex); +seq_printf (s,"%-14s%-16d%-16lld%lld\n" ,"caps" , nr_caps, +percpu_counter_sum (&mdsc->metric.i_caps_mis), +percpu_counter_sum (&mdsc->metric.i_caps_hit)); +return 0 ; }
fs/ceph/dir.c +3 -2
@@ -349,8 +349,9 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + __ceph_caps_issued_mask_metric(ci, CEPH_CAP_FILE_SHARED, 1)) {int shared_gen =atomic_read (&ci->i_shared_gen); +spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )
@@ -767,7 +768,7 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, !is_root_ceph_dentry(dir, dentry) && ceph_test_mount_opt(fsc, DCACHE) && __ceph_dir_is_complete(ci) && - (__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) { + __ceph_caps_issued_mask_metric(ci, CEPH_CAP_FILE_SHARED, 1)) {__ceph_touch_fmode (ci, mdsc,CEPH_FILE_MODE_RD );spin_unlock (&ci->i_ceph_lock);dout (" dir %p complete, -ENOENT\n" , dir);
fs/ceph/inode.c +2 -2
@@ -2284,8 +2284,8 @@ int __ceph_do_getattr(struct inode *inode, struct page *locked_page,dout ("do_getattr inode %p mask %s mode 0%o\n" , inode,ceph_cap_string (mask), inode->i_mode); -if (!force &&ceph_caps_issued_mask (ceph_inode (inode), mask,1 )) -return 0 ; +if (!force &&ceph_caps_issued_mask_metric (ceph_inode (inode), mask,1 )) +return 0 ; mode = (mask &CEPH_STAT_RSTAT ) ?USE_AUTH_MDS :USE_ANY_MDS ; req =ceph_mdsc_create_request (mdsc,CEPH_MDS_OP_GETATTR , mode);
fs/ceph/mds_client.c +22 -4
@@ -4332,13 +4332,29 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric) ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret)return ret; + ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL ); -if (ret) { -percpu_counter_destroy (&metric->d_lease_hit); -return ret; - } +if (ret) + goto err_d_lease_mis; + + ret =percpu_counter_init (&metric->i_caps_hit,0 ,GFP_KERNEL ); +if (ret) + goto err_i_caps_hit; + + ret =percpu_counter_init (&metric->i_caps_mis,0 ,GFP_KERNEL ); +if (ret) + goto err_i_caps_mis;return 0 ; + +err_i_caps_mis: +percpu_counter_destroy (&metric->i_caps_hit); +err_i_caps_hit: +percpu_counter_destroy (&metric->d_lease_mis); +err_d_lease_mis: +percpu_counter_destroy (&metric->d_lease_hit); + +return ret; }int ceph_mdsc_init (struct ceph_fs_client *fsc)
@@ -4678,6 +4694,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.i_caps_mis); +percpu_counter_destroy (&mdsc->metric.i_caps_hit);percpu_counter_destroy (&mdsc->metric.d_lease_mis);percpu_counter_destroy (&mdsc->metric.d_lease_hit);
fs/ceph/metric.h +13 -0
@@ -7,5 +7,18 @@ struct ceph_client_metric {atomic64_t total_dentries;struct percpu_counter d_lease_hit;struct percpu_counter d_lease_mis; + +struct percpu_counter i_caps_hit; + struct percpu_counter i_caps_mis; }; + +static inline void ceph_update_cap_hit (struct ceph_client_metric *m) +{ +percpu_counter_inc (&m->i_caps_hit); +} + +static inline void ceph_update_cap_mis (struct ceph_client_metric *m) +{ +percpu_counter_inc (&m->i_caps_mis); +}#endif /* _FS_CEPH_MDS_METRIC_H */
fs/ceph/super.h +5 -3
@@ -645,6 +645,8 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci)extern int __ceph_caps_issued (struct ceph_inode_info *ci,int *implemented);extern int __ceph_caps_issued_mask (struct ceph_inode_info *ci,int mask,int t); +extern int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int t);extern int __ceph_caps_issued_other (struct ceph_inode_info *ci,struct ceph_cap *cap);
@@ -657,12 +659,12 @@ static inline int ceph_caps_issued(struct ceph_inode_info *ci)return issued; } -static inline int ceph_caps_issued_mask (struct ceph_inode_info *ci,int mask, -int touch) +static inline int ceph_caps_issued_mask_metric (struct ceph_inode_info *ci, +int mask,int touch) {int r;spin_lock (&ci->i_ceph_lock); - r =__ceph_caps_issued_mask (ci, mask, touch); + r =__ceph_caps_issued_mask_metric (ci, mask, touch);spin_unlock (&ci->i_ceph_lock);return r; }
fs/ceph/xattr.c +2 -2
@@ -856,7 +856,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (ci->i_xattrs.version ==0 || !((req_mask &CEPH_CAP_XATTR_SHARED ) || -__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,1 ))) {spin_unlock (&ci->i_ceph_lock);/* security module gets xattr while filling trace */
@@ -914,7 +914,7 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size) ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,1 )) {spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)
Signature · 1 line
1.8.3.1
Re: [PATCH v9 2/5] ceph: add caps perf metric for each session
Quoted 63 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)fs/ceph/acl.c +1
-1@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);fs/ceph/caps.c +14
-0@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */
fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2698
Quoted 5 lines
fs/ceph/caps.c +2
-0@@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem); +if (!ret) +ceph_update_cap_mis (&mdsc->metric);
Should a negative error code here also mean a miss?
Quoted 6 lines
fs/ceph/caps.c +3
-0+else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH v9 2/5] ceph: add caps perf metric for each session
Quoted 72 lines · Jeff Layton
Quoted 63 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)fs/ceph/acl.c +1
-1@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);fs/ceph/caps.c +14
-0@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */ fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2698
Quoted 5 lines
fs/ceph/caps.c +2
-0@@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem); +if (!ret) +ceph_update_cap_mis (&mdsc->metric);Should a negative error code here also mean a miss?
Quoted 6 lines
fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2704
Quoted 6 lines
fs/ceph/caps.c +3
-0+else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;
Here's what I'd propose on top. If this looks ok, then I can just fold this patch into yours before merging. No need to resend just for this.
----------------8<----------------
[PATCH] SQUASH: count negative error code as miss in try_get_cap_refs
--- fs/ceph/caps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
fs/ceph/caps.c +0 -0
@@ -2694,9 +2694,9 @@ static int try_get_cap_refs(struct inode *inode,
int need, int want,
if (snap_rwsem_locked)
up_read(&mdsc->snap_rwsem);
- if (!ret)
+ if (ret <= 0)
ceph_update_cap_mis(&mdsc->metric);
- else if (ret == 1)
+ else
ceph_update_cap_hit(&mdsc->metric);
dout("get_cap_refs %p ret %d got %s\n", inode,
Signature · 1 line
2.24.1
Re: [PATCH v9 2/5] ceph: add caps perf metric for each session
Quoted 106 lines · Jeff Layton
Quoted 71 lines · Jeff Layton
Quoted 63 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)fs/ceph/acl.c +1
-1@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);fs/ceph/caps.c +14
-0@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */ fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2698
Quoted 5 lines
fs/ceph/caps.c +2
-0@@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem); +if (!ret) +ceph_update_cap_mis (&mdsc->metric);Should a negative error code here also mean a miss?
Quoted 6 lines
fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2704
Quoted 6 lines
fs/ceph/caps.c +3
-0+else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;Here's what I'd propose on top. If this looks ok, then I can just fold this patch into yours before merging. No need to resend just for this.
----------------8<----------------
[PATCH] SQUASH: count negative error code as miss in
try_get_cap_refsSigned-off-by Jeff Layton <[email protected]>--- fs/ceph/caps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)fs/ceph/caps.c +2
-2@@ -2694,9 +2694,9 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, if (snap_rwsem_locked) up_read(&mdsc->snap_rwsem); -if (!ret) +if (ret <=0 )ceph_update_cap_mis (&mdsc->metric); -else if (ret ==1 ) +else ceph_update_cap_hit (&mdsc->metric);
Quoted 1 line
dout("get_cap_refs %p ret %d got %s\n", inode,
For the try_get_cap_refs(), maybe this is the correct approach to count hit/miss as the function comments.
Thanks
Brs
Re: [PATCH v9 2/5] ceph: add caps perf metric for each session
Quoted 107 lines · Xiubo Li
Quoted 106 lines · Jeff Layton
Quoted 71 lines · Jeff Layton
Quoted 63 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)fs/ceph/acl.c +1
-1@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);fs/ceph/caps.c +14
-0@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */ fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2698
Quoted 5 lines
fs/ceph/caps.c +2
-0@@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem); +if (!ret) +ceph_update_cap_mis (&mdsc->metric);Should a negative error code here also mean a miss?
Quoted 6 lines
fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2704
Quoted 6 lines
fs/ceph/caps.c +3
-0+else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;Here's what I'd propose on top. If this looks ok, then I can just fold this patch into yours before merging. No need to resend just for this.
----------------8<----------------
[PATCH] SQUASH: count negative error code as miss in
try_get_cap_refsSigned-off-by Jeff Layton <[email protected]>--- fs/ceph/caps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)fs/ceph/caps.c +2
-2@@ -2694,9 +2694,9 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, if (snap_rwsem_locked) up_read(&mdsc->snap_rwsem); -if (!ret) +if (ret <=0 )ceph_update_cap_mis (&mdsc->metric); -else if (ret ==1 ) +else ceph_update_cap_hit (&mdsc->metric);
Quoted 5 lines
Quoted 1 line
dout("get_cap_refs %p ret %d got %s\n", inode,
For the
try_get_cap_refs(), maybe this is the correct approach to count hit/miss as the function comments.
I decided to just merge your patches as-is. Given that those are error conditions, and some of them may occur before we ever check the caps, I think we should just opt to not count those cases.
I did clean up the changelogs a bit, so please have a look and let me know if they look ok to you.
Thanks!
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH v9 2/5] ceph: add caps perf metric for each session
Quoted 112 lines · Jeff Layton
Quoted 107 lines · Xiubo Li
Quoted 106 lines · Jeff Layton
Quoted 71 lines · Jeff Layton
Quoted 63 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)fs/ceph/acl.c +1
-1@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);fs/ceph/caps.c +14
-0@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */ fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2698
Quoted 5 lines
fs/ceph/caps.c +2
-0@@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem); +if (!ret) +ceph_update_cap_mis (&mdsc->metric);Should a negative error code here also mean a miss?
Quoted 6 lines
fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2704
Quoted 6 lines
fs/ceph/caps.c +3
-0+else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;Here's what I'd propose on top. If this looks ok, then I can just fold this patch into yours before merging. No need to resend just for this.
----------------8<----------------
[PATCH] SQUASH: count negative error code as miss in
try_get_cap_refsSigned-off-by Jeff Layton <[email protected]>--- fs/ceph/caps.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)fs/ceph/caps.c +2
-2@@ -2694,9 +2694,9 @@ static int try_get_cap_refs(struct inode *inode, int need, int want, if (snap_rwsem_locked) up_read(&mdsc->snap_rwsem); -if (!ret) +if (ret <=0 )ceph_update_cap_mis (&mdsc->metric); -else if (ret ==1 ) +else ceph_update_cap_hit (&mdsc->metric);Quoted 4 lines
Quoted 1 line
dout("get_cap_refs %p ret %d got %s\n", inode,
For the
try_get_cap_refs(), maybe this is the correct approach to count hit/miss as the function comments.
Quoted 6 lines
I decided to just merge your patches as-is. Given that those are error conditions, and some of them may occur before we ever check the caps, I think we should just opt to not count those cases.
I did clean up the changelogs a bit, so please have a look and let me know if they look ok to you.
Cool, it looks nice to me.
Thanks Jeff.
BRs
Quoted 1 line
Thanks!
Re: [PATCH v9 2/5] ceph: add caps perf metric for each session
In reply to Jeff LaytonQuoted 67 lines · Jeff Layton
Quoted 66 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the cap hit/mis metric stuff per-superblock, it will count the hit/mis counters based each inode, and if one inode's 'issued & ~revoking == mask' will mean a hit, or a miss.
item total miss hit
-------------------------------------------------
caps 295 107 4119URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 +- fs/ceph/caps.c | 19 +++++++++++++++++++ fs/ceph/debugfs.c | 16 ++++++++++++++++ fs/ceph/dir.c | 5 +++-- fs/ceph/inode.c | 4 ++-- fs/ceph/mds_client.c | 26 ++++++++++++++++++++++---- fs/ceph/metric.h | 13 +++++++++++++ fs/ceph/super.h | 8 +++++--- fs/ceph/xattr.c | 4 ++-- 9 files changed, 83 insertions(+), 14 deletions(-)fs/ceph/acl.c +1
-1@@ -22,7 +22,7 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); -if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 )) +if (__ceph_caps_issued_mask_metric (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else forget_cached_acl (inode, type);fs/ceph/caps.c +14
-0@@ -912,6 +912,20 @@ int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch)return 0 ; } +int __ceph_caps_issued_mask_metric (struct ceph_inode_info *ci,int mask, +int touch) +{ +struct ceph_fs_client *fsc =ceph_sb_to_client (ci->vfs_inode.i_sb); +int r; + + r =__ceph_caps_issued_mask (ci, mask, touch); +if (r) +ceph_update_cap_hit (&fsc->mdsc->metric); +else +ceph_update_cap_mis (&fsc->mdsc->metric); +return r; +} +/* * Return true if mask caps are currently being revoked by an MDS. */ @@ -2680,6 +2694,11 @@ static int try_get_cap_refs(struct inode *inode, int need, int want,if (snap_rwsem_locked)up_read (&mdsc->snap_rwsem);
Quoted 3 lines
fs/ceph/caps.c static int try_get_cap_refs(struct inode *inode, int need, int want, · line 2698
Quoted 2 lines
fs/ceph/caps.c +2
-0+if (!ret) +ceph_update_cap_mis (&mdsc->metric);Should a negative error code here also mean a miss?
A negative error code could also from the case that if have & need == need, but it may fail with ret = -EAGAIN.
Or maybe could we just move this to :
if ((have & need) == need){
hit()
} else {
miss()
}
Thanks
BRs
Quoted 6 lines
Quoted 6 lines
fs/ceph/caps.c +3
-0+else if (ret ==1 ) +ceph_update_cap_hit (&mdsc->metric); +dout ("get_cap_refs %p ret %d got %s\n" , inode, ret,ceph_cap_string (*got));return ret;
PATCH v9 3/5 ceph: add global read latency metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
It will calculate the latency for the read osd requests:
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
read 1036 848000 818
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/addr.c | 8 ++++++++ fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/file.c | 16 ++++++++++++++++ fs/ceph/mds_client.c | 14 ++++++++++++++ fs/ceph/metric.h | 15 +++++++++++++++ include/linux/ceph/osd_client.h | 1 + net/ceph/osd_client.c | 2 ++ 7 files changed, 67 insertions(+)
fs/ceph/addr.c +8 -0
@@ -216,6 +216,9 @@ static int ceph_sync_readpages(struct ceph_fs_client *fsc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +ceph_update_read_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, rc); +ceph_osdc_put_request (req);dout ("readpages result %d\n" , rc);return rc;
@@ -299,6 +302,7 @@ static int ceph_readpage(struct file *filp, struct page *page)static void finish_read (struct ceph_osd_request *req) {struct inode *inode = req->r_inode; +struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_osd_data *osd_data;int rc = req->r_result <=0 ? req->r_result :0 ;int bytes = req->r_result >=0 ? req->r_result :0 ;
@@ -336,6 +340,10 @@ static void finish_read(struct ceph_osd_request *req)put_page (page); bytes -=PAGE_SIZE ; } + +ceph_update_read_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, rc); +kfree (osd_data->pages); }
fs/ceph/debugfs.c +11 -0
@@ -129,7 +129,18 @@ static int metric_show(struct seq_file *s, void *p)struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc;int i, nr_caps =0 ; +s64 total, sum, avg =0 ; +seq_printf (s,"item total sum_lat(us) avg_lat(us)\n" ); +seq_printf (s,"-----------------------------------------------------\n" ); + + total =percpu_counter_sum (&mdsc->metric.total_reads); + sum =percpu_counter_sum (&mdsc->metric.read_latency_sum); + sum =jiffies_to_usecs (sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , total, sum, avg); + +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/file.c +16 -0
@@ -906,6 +906,10 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, ret =ceph_osdc_start_request (osdc, req, false);if (!ret) ret =ceph_osdc_wait_request (osdc, req); + +ceph_update_read_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, ret); +ceph_osdc_put_request (req); i_size =i_size_read (inode);
@@ -1044,6 +1048,8 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs);
@@ -1051,6 +1057,11 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)dout ("ceph_aio_complete_req %p rc %d bytes %u\n" , inode, rc, osd_data->bvec_pos.iter.bi_size); +/* r_start_stamp == 0 means the request was not submitted */ +if (req->r_start_stamp && !aio_req->write) +ceph_update_read_latency (metric, req->r_start_stamp, + req->r_end_stamp, rc); +if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;BUG_ON (!aio_req->write);
@@ -1179,6 +1190,7 @@ static void ceph_aio_retry_work(struct work_struct *work)struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct bio_vec *bvecs;
@@ -1295,6 +1307,10 @@ static void ceph_aio_retry_work(struct work_struct *work)if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if (!write) +ceph_update_read_latency (metric, req->r_start_stamp, + req->r_end_stamp, ret); + size =i_size_read (inode);if (!write) {if (ret == -ENOENT )
fs/ceph/mds_client.c +14 -0
@@ -4345,8 +4345,20 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)if (ret) goto err_i_caps_mis; + ret =percpu_counter_init (&metric->total_reads,0 ,GFP_KERNEL ); +if (ret) + goto err_total_reads; + + ret =percpu_counter_init (&metric->read_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_read_latency_sum; +return 0 ; +err_read_latency_sum: +percpu_counter_destroy (&metric->total_reads); +err_total_reads: +percpu_counter_destroy (&metric->i_caps_mis); err_i_caps_mis:percpu_counter_destroy (&metric->i_caps_hit); err_i_caps_hit:
@@ -4694,6 +4706,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.read_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_reads);percpu_counter_destroy (&mdsc->metric.i_caps_mis);percpu_counter_destroy (&mdsc->metric.i_caps_hit);percpu_counter_destroy (&mdsc->metric.d_lease_mis);
fs/ceph/metric.h +15 -0
@@ -10,6 +10,9 @@ struct ceph_client_metric {struct percpu_counter i_caps_hit;struct percpu_counter i_caps_mis; + +struct percpu_counter total_reads; +struct percpu_counter read_latency_sum; };static inline void ceph_update_cap_hit (struct ceph_client_metric *m)
@@ -21,4 +24,16 @@ static inline void ceph_update_cap_mis(struct ceph_client_metric *m) {percpu_counter_inc (&m->i_caps_mis); } + +static inline void ceph_update_read_latency (struct ceph_client_metric *m, +unsigned long r_start, +unsigned long r_end, +int rc) +{ +if (rc <0 && rc != -ENOENT && rc != -ETIMEDOUT ) +return ; + +percpu_counter_inc (&m->total_reads); +percpu_counter_add (&m->read_latency_sum, r_end - r_start); +}#endif /* _FS_CEPH_MDS_METRIC_H */
include/linux/ceph/osd_client.h +1 -0
@@ -213,6 +213,7 @@ struct ceph_osd_request {/* internal */ unsigned long r_stamp;/* jiffies, send or check time */ unsigned long r_start_stamp;/* jiffies */ +unsigned long r_end_stamp;/* jiffies */ int r_attempts;u32 r_map_dne_bound;
net/ceph/osd_client.c +2 -0
@@ -2389,6 +2389,8 @@ static void finish_request(struct ceph_osd_request *req)WARN_ON (lookup_request_mc (&osdc->map_checks, req->r_tid));dout ("%s req %p tid %llu\n" , __func__, req, req->r_tid); + req->r_end_stamp = jiffies; +if (req->r_osd)unlink_request (req->r_osd, req);atomic_dec (&osdc->num_requests);
Signature · 1 line
1.8.3.1
PATCH v9 4/5 ceph: add global write latency metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
It will calculate the latency for the write osd requests:
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
write 1048 8778000 8375
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/addr.c | 10 ++++++++++ fs/ceph/debugfs.c | 6 ++++++ fs/ceph/file.c | 18 ++++++++++++++---- fs/ceph/mds_client.c | 14 ++++++++++++++ fs/ceph/metric.h | 15 +++++++++++++++ 5 files changed, 59 insertions(+), 4 deletions(-)
fs/ceph/addr.c +10 -0
@@ -651,6 +651,9 @@ static int ceph_sync_writepages(struct ceph_fs_client *fsc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +ceph_update_write_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, rc); +ceph_osdc_put_request (req);if (rc ==0 ) rc = len;
@@ -802,6 +805,9 @@ static void writepages_finish(struct ceph_osd_request *req)ceph_clear_error_write (ci); } +ceph_update_write_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, rc); +/* * We lost the cache cap, need to truncate the page before * it is unlocked, otherwise we'd truncate it later in the
@@ -1860,6 +1866,10 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err =ceph_osdc_start_request (&fsc->client->osdc, req, false);if (!err) err =ceph_osdc_wait_request (&fsc->client->osdc, req); + +ceph_update_write_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, err); + out_put:ceph_osdc_put_request (req);if (err == -ECANCELED )
fs/ceph/debugfs.c +6 -0
@@ -140,6 +140,12 @@ static int metric_show(struct seq_file *s, void *p) avg = total ? sum / total :0 ;seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , total, sum, avg); + total =percpu_counter_sum (&mdsc->metric.total_writes); + sum =percpu_counter_sum (&mdsc->metric.write_latency_sum); + sum =jiffies_to_usecs (sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"write" , total, sum, avg); +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/file.c +14 -4
@@ -1058,9 +1058,14 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req) inode, rc, osd_data->bvec_pos.iter.bi_size);/* r_start_stamp == 0 means the request was not submitted */ -if (req->r_start_stamp && !aio_req->write) -ceph_update_read_latency (metric, req->r_start_stamp, - req->r_end_stamp, rc); +if (req->r_start_stamp) { +if (aio_req->write) +ceph_update_write_latency (metric, req->r_start_stamp, + req->r_end_stamp, rc); +else +ceph_update_read_latency (metric, req->r_start_stamp, + req->r_end_stamp, rc); + }if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;
@@ -1307,7 +1312,10 @@ static void ceph_aio_retry_work(struct work_struct *work)if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); -if (!write) +if (write) +ceph_update_write_latency (metric, req->r_start_stamp, + req->r_end_stamp, ret); +else ceph_update_read_latency (metric, req->r_start_stamp, req->r_end_stamp, ret);
@@ -1482,6 +1490,8 @@ static void ceph_aio_retry_work(struct work_struct *work)if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +ceph_update_write_latency (&fsc->mdsc->metric, req->r_start_stamp, + req->r_end_stamp, ret); out:ceph_osdc_put_request (req);if (ret !=0 ) {
fs/ceph/mds_client.c +14 -0
@@ -4353,8 +4353,20 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)if (ret) goto err_read_latency_sum; + ret =percpu_counter_init (&metric->total_writes,0 ,GFP_KERNEL ); +if (ret) + goto err_total_writes; + + ret =percpu_counter_init (&metric->write_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_write_latency_sum; +return 0 ; +err_write_latency_sum: +percpu_counter_destroy (&metric->total_writes); +err_total_writes: +percpu_counter_destroy (&metric->read_latency_sum); err_read_latency_sum:percpu_counter_destroy (&metric->total_reads); err_total_reads:
@@ -4706,6 +4718,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.write_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_writes);percpu_counter_destroy (&mdsc->metric.read_latency_sum);percpu_counter_destroy (&mdsc->metric.total_reads);percpu_counter_destroy (&mdsc->metric.i_caps_mis);
fs/ceph/metric.h +15 -0
@@ -13,6 +13,9 @@ struct ceph_client_metric {struct percpu_counter total_reads;struct percpu_counter read_latency_sum; + +struct percpu_counter total_writes; +struct percpu_counter write_latency_sum; };static inline void ceph_update_cap_hit (struct ceph_client_metric *m)
@@ -36,4 +39,16 @@ static inline void ceph_update_read_latency(struct ceph_client_metric *m,percpu_counter_inc (&m->total_reads); percpu_counter_add(&m->read_latency_sum, r_end - r_start); } + +static inline void ceph_update_write_latency (struct ceph_client_metric *m, +unsigned long r_start, +unsigned long r_end, +int rc) +{ +if (rc && rc != -ETIMEDOUT ) +return ; + +percpu_counter_inc (&m->total_writes); +percpu_counter_add (&m->write_latency_sum, r_end - r_start); +}#endif /* _FS_CEPH_MDS_METRIC_H */
Signature · 1 line
1.8.3.1
PATCH v9 5/5 ceph: add global metadata perf metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
It will calculate the latency for the metedata requests:
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
metadata 113 220000 1946
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 6 ++++++ fs/ceph/mds_client.c | 19 +++++++++++++++++++ fs/ceph/mds_client.h | 3 ++- fs/ceph/metric.h | 15 +++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-)
fs/ceph/debugfs.c +6 -0
@@ -146,6 +146,12 @@ static int metric_show(struct seq_file *s, void *p) avg = total ? sum / total :0 ;seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"write" , total, sum, avg); + total =percpu_counter_sum (&mdsc->metric.total_metadatas); + sum =percpu_counter_sum (&mdsc->metric.metadata_latency_sum); + sum =jiffies_to_usecs (sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"metadata" , total, sum, avg); +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/mds_client.c +19 -0
@@ -2547,6 +2547,8 @@ static struct ceph_msg *create_request_message(struct ceph_mds_client *mdsc,static void complete_request (struct ceph_mds_client *mdsc,struct ceph_mds_request *req) { + req->r_ended = jiffies; +if (req->r_callback) req->r_callback (mdsc, req);complete_all (&req->r_completion);
@@ -3155,6 +3157,9 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg)/* kick calling process */ complete_request (mdsc, req); + +ceph_update_metadata_latency (&mdsc->metric, req->r_started, + req->r_ended, err); out:ceph_mdsc_put_request (req);return ;
@@ -4361,8 +4366,20 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)if (ret) goto err_write_latency_sum; + ret =percpu_counter_init (&metric->total_metadatas,0 ,GFP_KERNEL ); +if (ret) + goto err_total_metadatas; + + ret =percpu_counter_init (&metric->metadata_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_metadata_latency_sum; +return 0 ; +err_metadata_latency_sum: +percpu_counter_destroy (&metric->total_metadatas); +err_total_metadatas: +percpu_counter_destroy (&metric->write_latency_sum); err_write_latency_sum:percpu_counter_destroy (&metric->total_writes); err_total_writes:
@@ -4718,6 +4735,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.metadata_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_metadatas);percpu_counter_destroy (&mdsc->metric.write_latency_sum);percpu_counter_destroy (&mdsc->metric.total_writes);percpu_counter_destroy (&mdsc->metric.read_latency_sum);
fs/ceph/mds_client.h +2 -1
@@ -298,7 +298,8 @@ struct ceph_mds_request {u32 r_readdir_offset;unsigned long r_timeout;/* optional. jiffies, 0 is "wait forever" */ -unsigned long r_started;/* start time to measure timeout against */ +unsigned long r_started;/* start time to measure timeout against and latency */ +unsigned long r_ended;/* finish time to measure latency */ unsigned long r_request_started;/* start time for mds request only, used to measure lease durations */
fs/ceph/metric.h +15 -0
@@ -16,6 +16,9 @@ struct ceph_client_metric {struct percpu_counter total_writes;struct percpu_counter write_latency_sum; + +struct percpu_counter total_metadatas; +struct percpu_counter metadata_latency_sum; };static inline void ceph_update_cap_hit (struct ceph_client_metric *m)
@@ -51,4 +54,16 @@ static inline void ceph_update_write_latency(struct ceph_client_metric *m,percpu_counter_inc (&m->total_writes); percpu_counter_add(&m->write_latency_sum, r_end - r_start); } + +static inline void ceph_update_metadata_latency (struct ceph_client_metric *m, +unsigned long r_start, +unsigned long r_end, +int rc) +{ +if (rc && rc != -ENOENT ) +return ; + +percpu_counter_inc (&m->total_metadatas); +percpu_counter_add (&m->metadata_latency_sum, r_end - r_start); +}#endif /* _FS_CEPH_MDS_METRIC_H */
Signature · 1 line
1.8.3.1
Quoted 15 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
Changed in V9:
- add anr_endedfield to the mds request struct and use that to calculate the metric
- fix some commit commentsWe can get the metrics from the debugfs:
$ cat /sys/kernel/debug/ceph/0c93a60d-5645-4c46-8568-4c8f63db4c7f.client4267/metrics item total sum_lat(us) avg_lat(us) ----------------------------------------------------- read 13 417000 32076 write 42 131205000 3123928 metadata 104 493000 4740
Quoted 5 lines
item total miss hit
-------------------------------------------------d_lease 204 0 918 caps 204 213 368218
Thanks Xiubo! This looks good. One minor issue with the cap patch, but I can just fix that up before merging if you're ok with my proposed change.
Beyond this...while average latency is a good metric, it's often not enough to help diagnose problems. I wonder if we ought to be at least tracking min/max latency for all calls too. I wonder if there's way to track standard deviation too? That would be really nice to have.
Cheers,
Signature · 1 line
Jeff Layton <[email protected]>
Quoted 21 lines · Jeff Layton
Quoted 15 lines · On Mon, 2020-03-09 at 03:37 -0400, [email protected] wrote:
From: Xiubo Li <[email protected]>
Changed in V9:
- add anr_endedfield to the mds request struct and use that to calculate the metric
- fix some commit commentsWe can get the metrics from the debugfs:
$ cat /sys/kernel/debug/ceph/0c93a60d-5645-4c46-8568-4c8f63db4c7f.client4267/metrics item total sum_lat(us) avg_lat(us) ----------------------------------------------------- read 13 417000 32076 write 42 131205000 3123928 metadata 104 493000 4740Quoted 5 lines
item total miss hit
-------------------------------------------------d_lease 204 0 918 caps 204 213 368218
Quoted 8 lines
Thanks Xiubo! This looks good. One minor issue with the cap patch, but I can just fix that up before merging if you're ok with my proposed change.
Beyond this...while average latency is a good metric, it's often not enough to help diagnose problems. I wonder if we ought to be at least tracking min/max latency for all calls too. I wonder if there's way to track standard deviation too? That would be really nice to have.
yeah, the min/max latencies here make sense, it is on my todo list and I will do it after this patch series.
And for the standard deviation I will try to have a investigate of it.
Thanks
Quoted 1 line
Cheers,