ceph: add perf metrics support
Subsystems (2)
v4 cover ceph: add perf metrics support
From: Xiubo Li <[email protected]>
Changed in V2:
- add read/write/metadata latency metric support.
- add and send client provided metric flags in client metadata
- addressed the comments from Ilya and merged the 4/4 patch into 3/4.
- addressed all the other comments in v1 series.
Changed in V3:
- addressed Jeff's comments and let's the callers do the metric counting.
- with some small fixes for the read/write latency
- tested based on the latest testing branch
Changed in V4:
- fix the lock issue
It will send the metrics to the MDSs every second if sending_metrics is enabled, disable as default.
We can get the metrics from the debugfs:
$ cat /sys/kernel/debug/ceph/0c93a60d-5645-4c46-8568-4c8f63db4c7f.client4267/metrics item total sum_lat(us) avg_lat(us) ----------------------------------------------------- read 13 417000 32076 write 42 131205000 3123928 metadata 104 493000 4740
item total miss hit
-------------------------------------------------d_lease 204 0 918
session caps miss hit
-------------------------------------------------
0 204 213 368218
In the MDS side, we can get the metrics(NOTE: the latency is in nanosecond):
$ ./bin/ceph fs perf stats | python -m json.tool
{
"client_metadata": {
"client.4267": {
"IP": "v1:192.168.195.165",
"hostname": "fedora1",
"mount_point": "N/A",
"root": "/"
}
},
"counters": [
"cap_hit"
],
"global_counters": [
"read_latency",
"write_latency",
"metadata_latency",
"dentry_lease_hit"
],
"global_metrics": {
"client.4267": [
[
0,
32076923
],
[
3,
123928571
],
[
0,
4740384
],
[
918,
0
]
]
},
"metrics": {
"delayed_ranks": [],
"mds.0": {
"client.4267": [
[
368218,
213
]
]
}
}
}The provided metric flags in client metadata
$./bin/cephfs-journal-tool --rank=1:0 event get --type=SESSION json Wrote output to JSON file 'dump'
$ cat dump
[
{
"client instance": "client.4275 v1:192.168.195.165:0/461391971",
"open": "true",
"client map version": 1,
"inos": "[]",
"inotable version": 0,
"client_metadata": {
"client_features": {
"feature_bits": "0000000000001bff"
},
"metric_spec": {
"metric_flags": {
"feature_bits": "000000000000001f"
}
},
"entity_id": "",
"hostname": "fedora1",
"kernel_version": "5.5.0-rc2+",
"root": "/"
}
},
[...]Xiubo Li (8):
ceph: add global dentry lease metric support
ceph: add caps perf metric for each session
ceph: add global read latency metric support
ceph: add global write latency metric support
ceph: add global metadata perf metric support
ceph: periodically send perf metrics to MDS
ceph: add reset metrics support
ceph: send client provided metric flags in client metadata
fs/ceph/acl.c | 2 + fs/ceph/addr.c | 40 +++- fs/ceph/caps.c | 74 +++++++ fs/ceph/debugfs.c | 182 +++++++++++++++- fs/ceph/dir.c | 27 ++- fs/ceph/file.c | 37 ++++ fs/ceph/mds_client.c | 369 ++++++++++++++++++++++++++++++-- fs/ceph/mds_client.h | 48 +++++ fs/ceph/quota.c | 9 +- fs/ceph/super.h | 14 ++ fs/ceph/xattr.c | 17 +- include/linux/ceph/ceph_fs.h | 77 +++++++ include/linux/ceph/osd_client.h | 5 +- net/ceph/osd_client.c | 18 +- 14 files changed, 878 insertions(+), 41 deletions(-)
Signature · 1 line
2.21.0
PATCH v4 1/8 ceph: add global dentry lease metric support
From: Xiubo Li <[email protected]>
For the dentry lease we will only count the hit/miss info triggered from the vfs calls, for the cases like request reply handling and perodically ceph_trim_dentries() we will ignore them.
Currently only the debugfs is support:
The output will be:
item total miss hit
-------------------------------------------------d_lease 11 7 141
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 32 ++++++++++++++++++++++++++++---- fs/ceph/dir.c | 18 ++++++++++++++++-- fs/ceph/mds_client.c | 37 +++++++++++++++++++++++++++++++++++-- fs/ceph/mds_client.h | 9 +++++++++ fs/ceph/super.h | 1 + 5 files changed, 89 insertions(+), 8 deletions(-)
fs/ceph/debugfs.c +28 -4
@@ -124,6 +124,22 @@ static int mdsc_show(struct seq_file *s, void *p)return 0 ; } +static int metric_show (struct seq_file *s,void *p) +{ +struct ceph_fs_client *fsc = s->private; +struct ceph_mds_client *mdsc = fsc->mdsc; + +seq_printf (s,"item total miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +seq_printf (s,"%-14s%-16lld%-16lld%lld\n" ,"d_lease" , +atomic64_read (&mdsc->metric.total_dentries), +percpu_counter_sum (&mdsc->metric.d_lease_mis), +percpu_counter_sum (&mdsc->metric.d_lease_hit)); + +return 0 ; +} +static int caps_show_cb (struct inode *inode,struct ceph_cap *cap,void *p) {struct seq_file *s = p;
@@ -220,6 +236,7 @@ static int mds_sessions_show(struct seq_file *s, void *ptr) CEPH_DEFINE_SHOW_FUNC(mdsmap_show) CEPH_DEFINE_SHOW_FUNC(mdsc_show) +CEPH_DEFINE_SHOW_FUNC(metric_show) CEPH_DEFINE_SHOW_FUNC(caps_show) CEPH_DEFINE_SHOW_FUNC(mds_sessions_show)
@@ -255,6 +272,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_mdsmap);debugfs_remove (fsc->debugfs_mds_sessions);debugfs_remove (fsc->debugfs_caps); +debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_mdsc); }
@@ -295,11 +313,17 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &mdsc_show_fops); + fsc->debugfs_metric =debugfs_create_file ("metrics" , +0400 , + fsc->client->debugfs_dir, + fsc, + &metric_show_fops); + fsc->debugfs_caps =debugfs_create_file ("caps" , -0400 , - fsc->client->debugfs_dir, - fsc, - &caps_show_fops); +0400 , + fsc->client->debugfs_dir, + fsc, + &caps_show_fops); }
fs/ceph/dir.c +16 -2
@@ -38,6 +38,8 @@ static int __dir_lease_try_check(const struct dentry *dentry);static int ceph_d_init (struct dentry *dentry) {struct ceph_dentry_info *di; +struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb); +struct ceph_mds_client *mdsc = fsc->mdsc; di =kmem_cache_zalloc (ceph_dentry_cachep,GFP_KERNEL );if (!di)
@@ -48,6 +50,9 @@ static int ceph_d_init(struct dentry *dentry) di->time = jiffies; dentry->d_fsdata = di;INIT_LIST_HEAD (&di->lease_list); + +atomic64_inc (&mdsc->metric.total_dentries); +return 0 ; }
@@ -1613,6 +1618,7 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry)*/ static int ceph_d_revalidate (struct dentry *dentry,unsigned int flags) { +struct ceph_mds_client *mdsc;int valid =0 ;struct dentry *parent;struct inode *dir, *inode;
@@ -1651,9 +1657,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) } } + mdsc =ceph_sb_to_client (dir->i_sb)->mdsc;if (!valid) { -struct ceph_mds_client *mdsc = -ceph_sb_to_client (dir->i_sb)->mdsc;struct ceph_mds_request *req;int op, err;u32 mask;
@@ -1661,6 +1666,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)if (flags &LOOKUP_RCU )return -ECHILD ; +percpu_counter_inc (&mdsc->metric.d_lease_mis); + op =ceph_snap (dir) ==CEPH_SNAPDIR ?CEPH_MDS_OP_LOOKUPSNAP :CEPH_MDS_OP_LOOKUP ; req =ceph_mdsc_create_request (mdsc, op,USE_ANY_MDS );
@@ -1692,6 +1699,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)dout ("d_revalidate %p lookup result=%d\n" , dentry, err); } + }else { +percpu_counter_inc (&mdsc->metric.d_lease_hit); }dout ("d_revalidate %p %s\n" , dentry, valid ?"valid" :"invalid" );
@@ -1700,6 +1709,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)if (!(flags &LOOKUP_RCU ))dput (parent); +return valid; }
@@ -1734,9 +1744,13 @@ static int ceph_d_delete(const struct dentry *dentry)static void ceph_d_release (struct dentry *dentry) {struct ceph_dentry_info *di =ceph_dentry (dentry); +struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb); +struct ceph_mds_client *mdsc = fsc->mdsc;dout ("d_release %p\n" , dentry); +atomic64_dec (&mdsc->metric.total_dentries); +spin_lock (&dentry->d_lock);__dentry_lease_unlist (di); dentry->d_fsdata =NULL ;
fs/ceph/mds_client.c +35 -2
@@ -4158,10 +4158,31 @@ static void delayed_work(struct work_struct *work)schedule_delayed (mdsc); } +static int ceph_mdsc_metric_init (struct ceph_client_metric *metric) +{ +int ret; + +if (!metric) +return -EINVAL ; + +atomic64_set (&metric->total_dentries,0 ); + ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL ); +if (ret) +return ret; + ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL ); +if (ret) { +percpu_counter_destroy (&metric->d_lease_hit); +return ret; + } + +return 0 ; +} +int ceph_mdsc_init (struct ceph_fs_client *fsc) {struct ceph_mds_client *mdsc; +int err; mdsc =kzalloc (sizeof (struct ceph_mds_client ),GFP_NOFS );if (!mdsc)
@@ -4170,8 +4191,8 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)mutex_init (&mdsc->mutex); mdsc->mdsmap =kzalloc (sizeof (*mdsc->mdsmap),GFP_NOFS );if (!mdsc->mdsmap) { -kfree (mdsc); -return -ENOMEM ; + err = -ENOMEM ; + goto err_mdsc; } fsc->mdsc = mdsc;
@@ -4210,6 +4231,9 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)init_waitqueue_head (&mdsc->cap_flushing_wq);INIT_WORK (&mdsc->cap_reclaim_work, ceph_cap_reclaim_work);atomic_set (&mdsc->cap_reclaim_pending,0 ); + err =ceph_mdsc_metric_init (&mdsc->metric); +if (err) + goto err_mdsmap;spin_lock_init (&mdsc->dentry_list_lock);INIT_LIST_HEAD (&mdsc->dentry_leases);
@@ -4228,6 +4252,12 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)strscpy (mdsc->nodename,utsname ()->nodename,sizeof (mdsc->nodename));return 0 ; + +err_mdsmap: +kfree (mdsc->mdsmap); +err_mdsc: +kfree (mdsc); +return err; }/*
@@ -4485,6 +4515,9 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.d_lease_mis); +percpu_counter_destroy (&mdsc->metric.d_lease_hit); + fsc->mdsc =NULL ;kfree (mdsc);dout ("mdsc_destroy %p done\n" , mdsc);
fs/ceph/mds_client.h +9 -0
@@ -358,6 +358,13 @@ struct cap_wait {int want; }; +/* This is the global metrics */ +struct ceph_client_metric { +atomic64_t total_dentries; +struct percpu_counter d_lease_hit; +struct percpu_counter d_lease_mis; +}; +/* * mds client state */
@@ -446,6 +453,8 @@ struct ceph_mds_client {struct list_head dentry_leases;/* fifo list */ struct list_head dentry_dir_leases;/* lru list */ +struct ceph_client_metric metric; +spinlock_t snapid_map_lock;struct rb_root snapid_map_tree;struct list_head snapid_map_lru;
fs/ceph/super.h +1 -0
@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
Signature · 1 line
2.21.0
PATCH v4 2/8 ceph: add caps perf metric for each session
In reply to [email protected]From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need" mask it means hit, or missed.
item total miss hit
-------------------------------------------------d_lease 295 0 993
session caps miss hit
-------------------------------------------------
0 295 107 4119 1 1 107 9
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/acl.c | 2 ++ fs/ceph/addr.c | 2 ++ fs/ceph/caps.c | 74 ++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/debugfs.c | 20 ++++++++++++ fs/ceph/dir.c | 9 ++++-- fs/ceph/file.c | 3 ++ fs/ceph/mds_client.c | 16 +++++++++- fs/ceph/mds_client.h | 3 ++ fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)
fs/ceph/acl.c +2 -0
@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); +if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else
fs/ceph/addr.c +2 -0
@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM ; goto out; } + +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); err =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , true);if (err <0 ) {
fs/ceph/caps.c +74 -0
104 lines not on this page · raw message
fs/ceph/debugfs.c +20 -0
@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i;seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p)percpu_counter_sum (&mdsc->metric.d_lease_mis), percpu_counter_sum(&mdsc->metric.d_lease_hit)); +seq_printf (s,"\n" ); +seq_printf (s,"session caps miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +seq_printf (s,"%-14d%-16d%-16lld%lld\n" , i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); +ceph_put_mds_session (session); + } +mutex_unlock (&mdsc->mutex); +return 0 ; }
fs/ceph/dir.c +7 -2
@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_mds_client *mdsc = fsc->mdsc;int i; -int err; +int err, ret =-1 ;unsigned frag =-1 ;struct ceph_mds_reply_info_parsed *rinfo;
@@ -346,13 +346,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {int shared_gen =atomic_read (&ci->i_shared_gen); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )return err; } else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); }
@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry,struct ceph_dentry_info *di =ceph_dentry (dentry);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_FILE_SHARED ); +dout (" dir %p flags are %d\n" , dir, ci->i_ceph_flags); if (strncmp(dentry->d_name.name, fsc->mount_options->snapdir_name,
fs/ceph/file.c +3 -0
@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, struct file *file)* asynchronously. */ spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, wanted); +if (__ceph_is_any_real_caps (ci) && (((fmode &CEPH_FILE_MODE_WR ) ==0 ) || ci->i_auth_cap)) {int mds_wanted =__ceph_caps_mds_wanted (ci, true);
@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)return -ENOMEM ; } +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); statret =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , !!page);if (statret <0 ) {
fs/ceph/mds_client.c +15 -1
@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s)if (refcount_dec_and_test (&s->s_ref)) {if (s->s_auth.authorizer)ceph_auth_destroy_authorizer (s->s_auth.authorizer); +percpu_counter_destroy (&s->i_caps_hit); +percpu_counter_destroy (&s->i_caps_mis);kfree (s); } }
@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) {struct ceph_mds_session *s; +int err;if (mds >= mdsc->mdsmap->possible_max_rank)return ERR_PTR (-EINVAL );
@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,dout ("%s: realloc to %d\n" , __func__, newmax); sa =kcalloc (newmax,sizeof (void *),GFP_NOFS ); -if (!sa) +if (!sa) { + err = -ENOMEM ; goto fail_realloc; + }if (mdsc->sessions) {memcpy (sa, mdsc->sessions, mdsc->max_sessions *sizeof (void *));
@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,INIT_LIST_HEAD (&s->s_cap_flushing); + err =percpu_counter_init (&s->i_caps_hit,0 ,GFP_NOFS ); +if (err) + goto fail_realloc; + err =percpu_counter_init (&s->i_caps_mis,0 ,GFP_NOFS ); +if (err) + goto fail_init; + mdsc->sessions[mds] = s;atomic_inc (&mdsc->num_sessions);refcount_inc (&s->s_ref);/* one ref to sessions[], one to caller */
@@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,return s; +fail_init: +percpu_counter_destroy (&s->i_caps_hit); fail_realloc:kfree (s);return ERR_PTR (-ENOMEM );
fs/ceph/mds_client.h +3 -0
@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting;/* waiting requests */ struct list_head s_unsafe;/* unsafe requests */ + +struct percpu_counter i_caps_hit; +struct percpu_counter i_caps_mis; };/*
fs/ceph/quota.c +7 -2
@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc,return NULL ; }if (qri->inode) { +struct ceph_inode_info *ci =ceph_inode (qri->inode); +int ret; + +ceph_caps_metric (ci,CEPH_STAT_CAP_INODE ); +/* get caps */ -int ret =__ceph_do_getattr (qri->inode,NULL , -CEPH_STAT_CAP_INODE , true); + ret =__ceph_do_getattr (qri->inode,NULL , +CEPH_STAT_CAP_INODE , true);if (ret >=0 ) in = qri->inode;else
fs/ceph/super.h +11 -0
@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci)return !RB_EMPTY_ROOT (&ci->i_caps); } +extern void __ceph_caps_metric (struct ceph_inode_info *ci,int mask); +static inline void ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, mask); +spin_unlock (&ci->i_ceph_lock); +} +extern int __ceph_caps_issued (struct ceph_inode_info *ci,int *implemented);extern int __ceph_caps_issued_mask (struct ceph_inode_info *ci,int mask,int t); extern int __ceph_caps_issued_other(struct ceph_inode_info *ci,
@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int ceph_do_getattr (struct inode *inode,int mask,bool force) { +struct ceph_inode_info *ci =ceph_inode (inode); + +ceph_caps_metric (ci, mask);return __ceph_do_getattr (inode,NULL , mask, force); }extern int ceph_permission (struct inode *inode,int mask);
fs/ceph/xattr.c +14 -3
@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,struct ceph_vxattr *vxattr =NULL ;int req_mask;ssize_t err; +int ret =-1 ;/* let's see if a virtual xattr was requested */ vxattr =ceph_match_vxattr (inode, name);
@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (ci->i_xattrs.version ==0 || !((req_mask &CEPH_CAP_XATTR_SHARED ) || -__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { + (ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock);/* security module gets xattr while filling trace */
@@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);
@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)struct ceph_inode_info *ci =ceph_inode (inode);bool len_only = (size ==0 );u32 namelen; -int err; +int err, ret =-1 ;spin_lock (&ci->i_ceph_lock);dout ("listxattr %p ver=%lld index_ver=%lld\n" , inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !(ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);
Signature · 1 line
2.21.0
PATCH v4 3/8 ceph: add global read latency metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
read 73 3590000 49178082
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/addr.c | 15 ++++++++++++++- fs/ceph/debugfs.c | 13 +++++++++++++ fs/ceph/file.c | 25 +++++++++++++++++++++++++ fs/ceph/mds_client.c | 25 ++++++++++++++++++++++++- fs/ceph/mds_client.h | 7 +++++++ include/linux/ceph/osd_client.h | 2 +- net/ceph/osd_client.c | 9 ++++++++- 7 files changed, 92 insertions(+), 4 deletions(-)
fs/ceph/addr.c +14 -1
@@ -190,6 +190,8 @@ static int ceph_do_readpage(struct file *filp, struct page *page)struct inode *inode =file_inode (filp);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric; +s64 latency;int err =0 ;u64 off =page_offset (page);u64 len =PAGE_SIZE ;
@@ -221,7 +223,7 @@ static int ceph_do_readpage(struct file *filp, struct page *page) err =ceph_osdc_readpages (&fsc->client->osdc,ceph_vino (inode), &ci->i_layout, off, &len, ci->i_truncate_seq, ci->i_truncate_size, - &page,1 ,0 ); + &page,1 ,0 , &latency);if (err == -ENOENT ) err =0 ;if (err <0 ) {
@@ -241,6 +243,9 @@ static int ceph_do_readpage(struct file *filp, struct page *page)ceph_readpage_to_fscache (inode, page); out: +if (latency) +ceph_mdsc_update_read_latency (metric, latency); +return err <0 ? err :0 ; }
@@ -260,6 +265,8 @@ static int ceph_readpage(struct file *filp, struct page *page)static void finish_read (struct ceph_osd_request *req) {struct inode *inode = req->r_inode; +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_data *osd_data;int rc = req->r_result <=0 ? req->r_result :0 ;int bytes = req->r_result >=0 ? req->r_result :0 ;
@@ -297,6 +304,12 @@ static void finish_read(struct ceph_osd_request *req)put_page (page); bytes -=PAGE_SIZE ; } + +if (rc >=0 || rc == -ENOENT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_read_latency (metric, latency); + } +kfree (osd_data->pages); }
fs/ceph/debugfs.c +13 -0
@@ -128,8 +128,21 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +s64 total, sum, avg =0 ;int i; +seq_printf (s,"item total sum_lat(us) avg_lat(us)\n" ); +seq_printf (s,"-----------------------------------------------------\n" ); + +spin_lock (&mdsc->metric.read_lock); + total =atomic64_read (&mdsc->metric.total_reads), + sum =timespec64_to_ns (&mdsc->metric.read_latency_sum); +spin_unlock (&mdsc->metric.read_lock); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/file.c +25 -0
@@ -588,6 +588,7 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_client *osdc = &fsc->client->osdc;ssize_t ret;u64 off = iocb->ki_pos;
@@ -660,6 +661,11 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, ret =ceph_osdc_start_request (osdc, req, false);if (!ret) ret =ceph_osdc_wait_request (osdc, req); + +if (ret >=0 || ret == -ENOENT || ret == -ETIMEDOUT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_read_latency (metric, latency); + }ceph_osdc_put_request (req); i_size =i_size_read (inode);
@@ -798,13 +804,24 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs); +BUG_ON (!aio_req);dout ("ceph_aio_complete_req %p rc %d bytes %u\n" , inode, rc, osd_data->bvec_pos.iter.bi_size); +/* r_start_stamp == 0 means the request was not submitted */ +if (req->r_start_stamp && (rc >=0 || rc == -ENOENT )) { +s64 latency = jiffies - req->r_start_stamp; + +if (!aio_req->write) +ceph_mdsc_update_read_latency (metric, latency); + } +if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;BUG_ON (!aio_req->write);
@@ -933,6 +950,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct bio_vec *bvecs;
@@ -1049,6 +1067,13 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if ((ret >=0 || ret == -ENOENT || ret == -ETIMEDOUT )) { +s64 latency = jiffies - req->r_start_stamp; + +if (!write) +ceph_mdsc_update_read_latency (metric, latency); + } + size =i_size_read (inode);if (!write) {if (ret == -ENOENT )
fs/ceph/mds_client.c +24 -1
@@ -4093,6 +4093,25 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)ceph_force_reconnect (fsc->sb); } +/* +* metric helpers +*/ +void ceph_mdsc_update_read_latency (struct ceph_client_metric *m, +s64 latency) +{ +struct timespec64 ts; + +if (!m) +return ; + +jiffies_to_timespec64 (latency, &ts); + +spin_lock (&m->read_lock); +atomic64_inc (&m->total_reads); + m->read_latency_sum =timespec64_add (m->read_latency_sum, ts); +spin_unlock (&m->read_lock); +} +/* * delayed work -- periodically trim expired leases, renew caps with mds */
@@ -4182,13 +4201,17 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)atomic64_set (&metric->total_dentries,0 ); ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret) -return ret; +return ret;; ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL );if (ret) {percpu_counter_destroy (&metric->d_lease_hit);return ret; } +spin_lock_init (&metric->read_lock); +memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_reads,0 ); +return 0 ; }
fs/ceph/mds_client.h +7 -0
@@ -366,6 +366,10 @@ struct ceph_client_metric {atomic64_t total_dentries;struct percpu_counter d_lease_hit;struct percpu_counter d_lease_mis; + +spinlock_t read_lock; +atomic64_t total_reads; +struct timespec64 read_latency_sum; };/*
@@ -549,4 +553,7 @@ extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc,extern int ceph_trim_caps (struct ceph_mds_client *mdsc,struct ceph_mds_session *session,int max_caps); + +extern void ceph_mdsc_update_read_latency (struct ceph_client_metric *m, +s64 latency);#endif
include/linux/ceph/osd_client.h +1 -1
@@ -515,7 +515,7 @@ extern int ceph_osdc_readpages(struct ceph_osd_client *osdc, u64 off, u64 *plen, u32 truncate_seq, u64 truncate_size, struct page **pages, int nr_pages, - int page_align); + int page_align, s64 *latency); extern int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino,
net/ceph/osd_client.c +8 -1
@@ -5238,11 +5238,15 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc, struct ceph_vino vino, struct ceph_file_layout *layout, u64 off, u64 *plen, u32 truncate_seq, u64 truncate_size, - struct page **pages, int num_pages, int page_align) + struct page **pages, int num_pages, int page_align, + s64 *latency) { struct ceph_osd_request *req;int rc =0 ; +if (latency) + *latency =0 ; +dout ("readpages on ino %llx.%llx on %llu~%llu\n" , vino.ino, vino.snap, off, *plen); req = ceph_osdc_new_request(osdc, layout, vino, off, plen, 0, 1,
@@ -5263,6 +5267,9 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +if (latency && (rc >=0 || rc == -ENOENT || rc == -ETIMEDOUT )) + *latency = jiffies - req->r_start_stamp; +ceph_osdc_put_request (req);dout ("readpages result %d\n" , rc);return rc;
Signature · 1 line
2.21.0
Re: [PATCH v4 3/8] ceph: add global read latency metric support
Quoted 276 lines · On Thu, 2020-01-16 at 05:38 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
item total
sum_lat(us)avg_lat(us)
-----------------------------------------------------
read 73 3590000 49178082URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/addr.c | 15 ++++++++++++++- fs/ceph/debugfs.c | 13 +++++++++++++ fs/ceph/file.c | 25 +++++++++++++++++++++++++ fs/ceph/mds_client.c | 25 ++++++++++++++++++++++++- fs/ceph/mds_client.h | 7 +++++++ include/linux/ceph/osd_client.h | 2 +- net/ceph/osd_client.c | 9 ++++++++- 7 files changed, 92 insertions(+), 4 deletions(-)fs/ceph/addr.c +14
-1@@ -190,6 +190,8 @@ static int ceph_do_readpage(struct file *filp, struct page *page)struct inode *inode =file_inode (filp);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric; +s64 latency;int err =0 ;u64 off =page_offset (page);u64 len =PAGE_SIZE ;@@ -221,7 +223,7 @@ static int ceph_do_readpage(struct file *filp, struct page *page) err =ceph_osdc_readpages (&fsc->client->osdc,ceph_vino (inode), &ci->i_layout, off, &len, ci->i_truncate_seq, ci->i_truncate_size, - &page,1 ,0 ); + &page,1 ,0 , &latency);if (err == -ENOENT ) err =0 ;if (err <0 ) {@@ -241,6 +243,9 @@ static int ceph_do_readpage(struct file *filp, struct page *page)ceph_readpage_to_fscache (inode, page); out: +if (latency) +ceph_mdsc_update_read_latency (metric, latency); +return err <0 ? err :0 ; }@@ -260,6 +265,8 @@ static int ceph_readpage(struct file *filp, struct page *page)static void finish_read (struct ceph_osd_request *req) {struct inode *inode = req->r_inode; +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_data *osd_data;int rc = req->r_result <=0 ? req->r_result :0 ;int bytes = req->r_result >=0 ? req->r_result :0 ;@@ -297,6 +304,12 @@ static void finish_read(struct ceph_osd_request *req)put_page (page); bytes -=PAGE_SIZE ; } + +if (rc >=0 || rc == -ENOENT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_read_latency (metric, latency); + } +kfree (osd_data->pages); }fs/ceph/debugfs.c +13
-0@@ -128,8 +128,21 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +s64 total, sum, avg =0 ;int i; +seq_printf (s,"item total sum_lat(us) avg_lat(us)\n" ); +seq_printf (s,"-----------------------------------------------------\n" ); + +spin_lock (&mdsc->metric.read_lock); + total =atomic64_read (&mdsc->metric.total_reads), + sum =timespec64_to_ns (&mdsc->metric.read_latency_sum); +spin_unlock (&mdsc->metric.read_lock); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );fs/ceph/file.c +25
-0@@ -588,6 +588,7 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_client *osdc = &fsc->client->osdc;ssize_t ret;u64 off = iocb->ki_pos;@@ -660,6 +661,11 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, ret =ceph_osdc_start_request (osdc, req, false);if (!ret) ret =ceph_osdc_wait_request (osdc, req); + +if (ret >=0 || ret == -ENOENT || ret == -ETIMEDOUT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_read_latency (metric, latency); + }ceph_osdc_put_request (req); i_size =i_size_read (inode);@@ -798,13 +804,24 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs); +BUG_ON (!aio_req);dout ("ceph_aio_complete_req %p rc %d bytes %u\n" , inode, rc, osd_data->bvec_pos.iter.bi_size); +/* r_start_stamp == 0 means the request was not submitted */ +if (req->r_start_stamp && (rc >=0 || rc == -ENOENT )) { +s64 latency = jiffies - req->r_start_stamp; + +if (!aio_req->write) +ceph_mdsc_update_read_latency (metric, latency); + } +if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;BUG_ON (!aio_req->write);@@ -933,6 +950,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct bio_vec *bvecs;@@ -1049,6 +1067,13 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if ((ret >=0 || ret == -ENOENT || ret == -ETIMEDOUT )) { +s64 latency = jiffies - req->r_start_stamp; + +if (!write) +ceph_mdsc_update_read_latency (metric, latency); + } + size =i_size_read (inode);if (!write) {if (ret == -ENOENT )fs/ceph/mds_client.c +24
-1@@ -4093,6 +4093,25 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)ceph_force_reconnect (fsc->sb); } +/* +* metric helpers +*/ +void ceph_mdsc_update_read_latency (struct ceph_client_metric *m, +s64 latency) +{ +struct timespec64 ts; + +if (!m) +return ; + +jiffies_to_timespec64 (latency, &ts); + +spin_lock (&m->read_lock); +atomic64_inc (&m->total_reads); + m->read_latency_sum =timespec64_add (m->read_latency_sum, ts); +spin_unlock (&m->read_lock); +} +/* * delayed work -- periodically trim expired leases, renew caps with mds */ @@ -4182,13 +4201,17 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)atomic64_set (&metric->total_dentries,0 ); ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret) -return ret; +return ret;; ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL );if (ret) {percpu_counter_destroy (&metric->d_lease_hit);return ret; } +spin_lock_init (&metric->read_lock); +memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_reads,0 ); +return 0 ; }fs/ceph/mds_client.h +7
-0@@ -366,6 +366,10 @@ struct ceph_client_metric {atomic64_t total_dentries;struct percpu_counter d_lease_hit;struct percpu_counter d_lease_mis; + +spinlock_t read_lock; +atomic64_t total_reads; +struct timespec64 read_latency_sum; };/* @@ -549,4 +553,7 @@ extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc,extern int ceph_trim_caps (struct ceph_mds_client *mdsc,struct ceph_mds_session *session,int max_caps); + +extern void ceph_mdsc_update_read_latency (struct ceph_client_metric *m, +s64 latency);#endif include/linux/ceph/osd_client.h +1
-1@@ -515,7 +515,7 @@ extern int ceph_osdc_readpages(struct ceph_osd_client *osdc, u64 off, u64 *plen, u32 truncate_seq, u64 truncate_size, struct page **pages, int nr_pages, - int page_align); + int page_align, s64 *latency); extern int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino,net/ceph/osd_client.c +5
-1@@ -5238,11 +5238,15 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc, struct ceph_vino vino, struct ceph_file_layout *layout, u64 off, u64 *plen, u32 truncate_seq, u64 truncate_size, - struct page **pages, int num_pages, int page_align) + struct page **pages, int num_pages, int page_align, + s64 *latency) { struct ceph_osd_request *req;int rc =0 ; +if (latency) + *latency =0 ; +dout ("readpages on ino %llx.%llx on %llu~%llu\n" , vino.ino, vino.snap, off, *plen); req = ceph_osdc_new_request(osdc, layout, vino, off, plen, 0, 1,
net/ceph/osd_client.c int ceph_osdc_readpages(struct ceph_osd_client *osdc, · line 5275
Quoted 9 lines
net/ceph/osd_client.c +3
-0@@ -5263,6 +5267,9 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +if (latency && (rc >=0 || rc == -ENOENT || rc == -ETIMEDOUT )) + *latency = jiffies - req->r_start_stamp; +ceph_osdc_put_request (req);dout ("readpages result %d\n" , rc);return rc;
This function is only called from ceph_do_readpage().
I think it'd be better to just turn ceph_osdc_readpages into ceph_osdc_readpages_submit, make it not wait on the req and have it return a pointer to the req with a reference held.
Then the caller could then handle the waiting and do the latency calculation afterward and we wouldn't need to add a new pointer argument here and push these details into libcephfs.
Signature · 1 line
Jeff Layton <[email protected]>
Ilya Dryomov
maintainerRe: [PATCH v4 3/8] ceph: add global read latency metric support
Quoted 290 lines · Jeff Layton
Quoted 276 lines · On Thu, 2020-01-16 at 05:38 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
item total
sum_lat(us)avg_lat(us)
-----------------------------------------------------
read 73 3590000 49178082URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/addr.c | 15 ++++++++++++++- fs/ceph/debugfs.c | 13 +++++++++++++ fs/ceph/file.c | 25 +++++++++++++++++++++++++ fs/ceph/mds_client.c | 25 ++++++++++++++++++++++++- fs/ceph/mds_client.h | 7 +++++++ include/linux/ceph/osd_client.h | 2 +- net/ceph/osd_client.c | 9 ++++++++- 7 files changed, 92 insertions(+), 4 deletions(-)fs/ceph/addr.c +14
-1@@ -190,6 +190,8 @@ static int ceph_do_readpage(struct file *filp, struct page *page)struct inode *inode =file_inode (filp);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric; +s64 latency;int err =0 ;u64 off =page_offset (page);u64 len =PAGE_SIZE ;@@ -221,7 +223,7 @@ static int ceph_do_readpage(struct file *filp, struct page *page) err =ceph_osdc_readpages (&fsc->client->osdc,ceph_vino (inode), &ci->i_layout, off, &len, ci->i_truncate_seq, ci->i_truncate_size, - &page,1 ,0 ); + &page,1 ,0 , &latency);if (err == -ENOENT ) err =0 ;if (err <0 ) {@@ -241,6 +243,9 @@ static int ceph_do_readpage(struct file *filp, struct page *page)ceph_readpage_to_fscache (inode, page); out: +if (latency) +ceph_mdsc_update_read_latency (metric, latency); +return err <0 ? err :0 ; }@@ -260,6 +265,8 @@ static int ceph_readpage(struct file *filp, struct page *page)static void finish_read (struct ceph_osd_request *req) {struct inode *inode = req->r_inode; +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_data *osd_data;int rc = req->r_result <=0 ? req->r_result :0 ;int bytes = req->r_result >=0 ? req->r_result :0 ;@@ -297,6 +304,12 @@ static void finish_read(struct ceph_osd_request *req)put_page (page); bytes -=PAGE_SIZE ; } + +if (rc >=0 || rc == -ENOENT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_read_latency (metric, latency); + } +kfree (osd_data->pages); }fs/ceph/debugfs.c +13
-0@@ -128,8 +128,21 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +s64 total, sum, avg =0 ;int i; +seq_printf (s,"item total sum_lat(us) avg_lat(us)\n" ); +seq_printf (s,"-----------------------------------------------------\n" ); + +spin_lock (&mdsc->metric.read_lock); + total =atomic64_read (&mdsc->metric.total_reads), + sum =timespec64_to_ns (&mdsc->metric.read_latency_sum); +spin_unlock (&mdsc->metric.read_lock); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );fs/ceph/file.c +25
-0@@ -588,6 +588,7 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_client *osdc = &fsc->client->osdc;ssize_t ret;u64 off = iocb->ki_pos;@@ -660,6 +661,11 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, ret =ceph_osdc_start_request (osdc, req, false);if (!ret) ret =ceph_osdc_wait_request (osdc, req); + +if (ret >=0 || ret == -ENOENT || ret == -ETIMEDOUT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_read_latency (metric, latency); + }ceph_osdc_put_request (req); i_size =i_size_read (inode);@@ -798,13 +804,24 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs); +BUG_ON (!aio_req);dout ("ceph_aio_complete_req %p rc %d bytes %u\n" , inode, rc, osd_data->bvec_pos.iter.bi_size); +/* r_start_stamp == 0 means the request was not submitted */ +if (req->r_start_stamp && (rc >=0 || rc == -ENOENT )) { +s64 latency = jiffies - req->r_start_stamp; + +if (!aio_req->write) +ceph_mdsc_update_read_latency (metric, latency); + } +if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;BUG_ON (!aio_req->write);@@ -933,6 +950,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct bio_vec *bvecs;@@ -1049,6 +1067,13 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if ((ret >=0 || ret == -ENOENT || ret == -ETIMEDOUT )) { +s64 latency = jiffies - req->r_start_stamp; + +if (!write) +ceph_mdsc_update_read_latency (metric, latency); + } + size =i_size_read (inode);if (!write) {if (ret == -ENOENT )fs/ceph/mds_client.c +24
-1@@ -4093,6 +4093,25 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)ceph_force_reconnect (fsc->sb); } +/* +* metric helpers +*/ +void ceph_mdsc_update_read_latency (struct ceph_client_metric *m, +s64 latency) +{ +struct timespec64 ts; + +if (!m) +return ; + +jiffies_to_timespec64 (latency, &ts); + +spin_lock (&m->read_lock); +atomic64_inc (&m->total_reads); + m->read_latency_sum =timespec64_add (m->read_latency_sum, ts); +spin_unlock (&m->read_lock); +} +/* * delayed work -- periodically trim expired leases, renew caps with mds */ @@ -4182,13 +4201,17 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)atomic64_set (&metric->total_dentries,0 ); ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret) -return ret; +return ret;; ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL );if (ret) {percpu_counter_destroy (&metric->d_lease_hit);return ret; } +spin_lock_init (&metric->read_lock); +memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_reads,0 ); +return 0 ; }fs/ceph/mds_client.h +7
-0@@ -366,6 +366,10 @@ struct ceph_client_metric {atomic64_t total_dentries;struct percpu_counter d_lease_hit;struct percpu_counter d_lease_mis; + +spinlock_t read_lock; +atomic64_t total_reads; +struct timespec64 read_latency_sum; };/* @@ -549,4 +553,7 @@ extern void ceph_mdsc_open_export_target_sessions(struct ceph_mds_client *mdsc,extern int ceph_trim_caps (struct ceph_mds_client *mdsc,struct ceph_mds_session *session,int max_caps); + +extern void ceph_mdsc_update_read_latency (struct ceph_client_metric *m, +s64 latency);#endif include/linux/ceph/osd_client.h +1
-1@@ -515,7 +515,7 @@ extern int ceph_osdc_readpages(struct ceph_osd_client *osdc, u64 off, u64 *plen, u32 truncate_seq, u64 truncate_size, struct page **pages, int nr_pages, - int page_align); + int page_align, s64 *latency); extern int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino,net/ceph/osd_client.c +5
-1@@ -5238,11 +5238,15 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc, struct ceph_vino vino, struct ceph_file_layout *layout, u64 off, u64 *plen, u32 truncate_seq, u64 truncate_size, - struct page **pages, int num_pages, int page_align) + struct page **pages, int num_pages, int page_align, + s64 *latency) { struct ceph_osd_request *req;int rc =0 ; +if (latency) + *latency =0 ; +dout ("readpages on ino %llx.%llx on %llu~%llu\n" , vino.ino, vino.snap, off, *plen); req = ceph_osdc_new_request(osdc, layout, vino, off, plen, 0, 1,net/ceph/osd_client.c int ceph_osdc_readpages(struct ceph_osd_client *osdc, · line 5275
Quoted 9 lines
net/ceph/osd_client.c +3
-0@@ -5263,6 +5267,9 @@ int ceph_osdc_readpages(struct ceph_osd_client *osdc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +if (latency && (rc >=0 || rc == -ENOENT || rc == -ETIMEDOUT )) + *latency = jiffies - req->r_start_stamp; +ceph_osdc_put_request (req);dout ("readpages result %d\n" , rc);return rc;This function is only called from
ceph_do_readpage().
Quoted 7 lines
I think it'd be better to just turn
ceph_osdc_readpagesintoceph_osdc_readpages_submit, make it not wait on the req and have it return a pointer to the req with a reference held.Then the caller could then handle the waiting and do the latency calculation afterward and we wouldn't need to add a new pointer argument here and push these details into libcephfs.
Same as for ceph_osdc_writepages(), let's keep the existing name.
Thanks,
Ilya
PATCH v4 4/8 ceph: add global write latency metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
write 222 5287750000 23818693
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/addr.c | 23 +++++++++++++++++++++-- fs/ceph/debugfs.c | 8 ++++++++ fs/ceph/file.c | 11 ++++++++++- fs/ceph/mds_client.c | 20 ++++++++++++++++++++ fs/ceph/mds_client.h | 6 ++++++ include/linux/ceph/osd_client.h | 3 ++- net/ceph/osd_client.c | 9 ++++++++- 7 files changed, 75 insertions(+), 5 deletions(-)
fs/ceph/addr.c +21 -2
@@ -598,12 +598,15 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc)loff_t page_off =page_offset (page);int err, len =PAGE_SIZE ;struct ceph_writeback_ctl ceph_wbc; +struct ceph_client_metric *metric; +s64 latency;dout ("writepage %p idx %lu\n" , page, page->index); inode = page->mapping->host; ci =ceph_inode (inode); fsc =ceph_inode_to_client (inode); + metric = &fsc->mdsc->metric;/* verify this is a writeable snap context */ snapc =page_snap_context (page);
@@ -645,7 +648,11 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc) &ci->i_layout, snapc, page_off, len, ceph_wbc.truncate_seq, ceph_wbc.truncate_size, - &inode->i_mtime, &page, 1); + &inode->i_mtime, &page,1 , + &latency); +if (latency) +ceph_mdsc_update_write_latency (metric, latency); +if (err <0 ) {struct writeback_control tmp_wbc;if (!wbc)
@@ -707,6 +714,8 @@ static void writepages_finish(struct ceph_osd_request *req) {struct inode *inode = req->r_inode;struct ceph_inode_info *ci =ceph_inode (inode); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_data *osd_data;struct page *page;int num_pages, total_pages =0 ;
@@ -714,7 +723,6 @@ static void writepages_finish(struct ceph_osd_request *req)int rc = req->r_result;struct ceph_snap_context *snapc = req->r_snapc;struct address_space *mapping = inode->i_mapping; -struct ceph_fs_client *fsc =ceph_inode_to_client (inode);bool remove_page;dout ("writepages_finish %p rc %d\n" , inode, rc);
@@ -783,6 +791,11 @@ static void writepages_finish(struct ceph_osd_request *req) ceph_sb_to_client(inode->i_sb)->wb_pagevec_pool); else kfree(osd_data->pages); + +if (!rc) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_write_latency (metric, latency); + }ceph_osdc_put_request (req); }
@@ -1675,6 +1688,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)struct inode *inode =file_inode (filp);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_request *req;struct page *page =NULL ;u64 len, inline_version;
@@ -1787,6 +1801,11 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err =ceph_osdc_start_request (&fsc->client->osdc, req, false);if (!err) err =ceph_osdc_wait_request (&fsc->client->osdc, req); + +if (!err || err == -ETIMEDOUT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_write_latency (metric, latency); + } out_put:ceph_osdc_put_request (req);if (err == -ECANCELED )
fs/ceph/debugfs.c +8 -0
@@ -142,6 +142,14 @@ static int metric_show(struct seq_file *s, void *p)seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); +spin_lock (&mdsc->metric.write_lock); + total =atomic64_read (&mdsc->metric.total_writes), + sum =timespec64_to_ns (&mdsc->metric.write_latency_sum); +spin_unlock (&mdsc->metric.write_lock); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"write" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/file.c +10 -1
@@ -818,7 +818,9 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)if (req->r_start_stamp && (rc >=0 || rc == -ENOENT )) {s64 latency = jiffies - req->r_start_stamp; -if (!aio_req->write) +if (aio_req->write) +ceph_mdsc_update_write_latency (metric, latency); +else ceph_mdsc_update_read_latency (metric, latency); }
@@ -1072,6 +1074,8 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!write)ceph_mdsc_update_read_latency (metric, latency); +else if (write && ret != -ENOENT ) +ceph_mdsc_update_write_latency (metric, latency); } size =i_size_read (inode);
@@ -1160,6 +1164,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct page **pages;
@@ -1245,6 +1250,10 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if (!ret || ret == -ETIMEDOUT ) { +s64 latency = jiffies - req->r_start_stamp; +ceph_mdsc_update_write_latency (metric, latency); + } out:ceph_osdc_put_request (req);if (ret !=0 ) {
fs/ceph/mds_client.c +20 -0
@@ -4112,6 +4112,22 @@ void ceph_mdsc_update_read_latency(struct ceph_client_metric *m,spin_unlock (&m->read_lock); } +void ceph_mdsc_update_write_latency (struct ceph_client_metric *m, +s64 latency) +{ +struct timespec64 ts; + +if (!m) +return ; + +jiffies_to_timespec64 (latency, &ts); + +spin_lock (&m->write_lock); +atomic64_inc (&m->total_writes); + m->write_latency_sum =timespec64_add (m->write_latency_sum, ts); +spin_unlock (&m->write_lock); +} +/* * delayed work -- periodically trim expired leases, renew caps with mds */
@@ -4212,6 +4228,10 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 ));atomic64_set (&metric->total_reads,0 ); +spin_lock_init (&metric->write_lock); +memset (&metric->write_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_writes,0 ); +return 0 ; }
fs/ceph/mds_client.h +6 -0
@@ -370,6 +370,10 @@ struct ceph_client_metric {spinlock_t read_lock;atomic64_t total_reads;struct timespec64 read_latency_sum; + +spinlock_t write_lock; +atomic64_t total_writes; +struct timespec64 write_latency_sum; };/*
@@ -556,4 +560,6 @@ extern int ceph_trim_caps(struct ceph_mds_client *mdsc,extern void ceph_mdsc_update_read_latency (struct ceph_client_metric *m,s64 latency); +extern void ceph_mdsc_update_write_latency (struct ceph_client_metric *m, +s64 latency);#endif
include/linux/ceph/osd_client.h +2 -1
@@ -524,7 +524,8 @@ extern int ceph_osdc_writepages(struct ceph_osd_client *osdc, u64 off, u64 len, u32 truncate_seq, u64 truncate_size, struct timespec64 *mtime, - struct page **pages, int nr_pages); + struct page **pages, int nr_pages, + s64 *latency); int ceph_osdc_copy_from(struct ceph_osd_client *osdc, u64 src_snapid, u64 src_version,
net/ceph/osd_client.c +8 -1
@@ -5285,12 +5285,16 @@ int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino, u64 off, u64 len, u32 truncate_seq, u64 truncate_size, struct timespec64 *mtime, - struct page **pages, int num_pages) + struct page **pages, int num_pages, + s64 *latency) { struct ceph_osd_request *req;int rc =0 ;int page_align = off & ~PAGE_MASK ; +if (latency) + *latency =0 ; + req = ceph_osdc_new_request(osdc, layout, vino, off, &len, 0, 1, CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, snapc, truncate_seq, truncate_size,
@@ -5308,6 +5312,9 @@ int ceph_osdc_writepages(struct ceph_osd_client *osdc, struct ceph_vino vino,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +if (latency && (!rc || rc == -ETIMEDOUT )) + *latency = jiffies - req->r_start_stamp; +ceph_osdc_put_request (req);if (rc ==0 ) rc = len;
Signature · 1 line
2.21.0
PATCH v4 5/8 ceph: add global metadata perf metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
metadata 1288 24506000 19026
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 8 ++++++++ fs/ceph/mds_client.c | 25 +++++++++++++++++++++++++ fs/ceph/mds_client.h | 6 ++++++ 3 files changed, 39 insertions(+)
fs/ceph/debugfs.c +8 -0
@@ -150,6 +150,14 @@ static int metric_show(struct seq_file *s, void *p)seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"write" , total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); +spin_lock (&mdsc->metric.metadata_lock); + total =atomic64_read (&mdsc->metric.total_metadatas), + sum =timespec64_to_ns (&mdsc->metric.metadata_latency_sum); +spin_unlock (&mdsc->metric.metadata_lock); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"metadata" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/mds_client.c +25 -0
@@ -2903,6 +2903,11 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) result =le32_to_cpu (head->result); +if (!result || result == -ENOENT ) { +s64 latency = jiffies - req->r_started; +ceph_mdsc_update_metadata_latency (&mdsc->metric, latency); + } +/* * Handle an ESTALE * if we're not talking to the authority, send to them
@@ -4128,6 +4133,22 @@ void ceph_mdsc_update_write_latency(struct ceph_client_metric *m,spin_unlock (&m->write_lock); } +void ceph_mdsc_update_metadata_latency (struct ceph_client_metric *m, +s64 latency) +{ +struct timespec64 ts; + +if (!m) +return ; + +jiffies_to_timespec64 (latency, &ts); + +spin_lock (&m->metadata_lock); +atomic64_inc (&m->total_metadatas); + m->metadata_latency_sum =timespec64_add (m->metadata_latency_sum, ts); +spin_unlock (&m->metadata_lock); +} +/* * delayed work -- periodically trim expired leases, renew caps with mds */
@@ -4232,6 +4253,10 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)memset (&metric->write_latency_sum,0 ,sizeof (struct timespec64 ));atomic64_set (&metric->total_writes,0 ); +spin_lock_init (&metric->metadata_lock); +memset (&metric->metadata_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_metadatas,0 ); +return 0 ; }
fs/ceph/mds_client.h +6 -0
@@ -374,6 +374,10 @@ struct ceph_client_metric {spinlock_t write_lock;atomic64_t total_writes;struct timespec64 write_latency_sum; + +spinlock_t metadata_lock; +atomic64_t total_metadatas; +struct timespec64 metadata_latency_sum; };/*
@@ -562,4 +566,6 @@ extern void ceph_mdsc_update_read_latency(struct ceph_client_metric *m, s64 latency);extern void ceph_mdsc_update_write_latency (struct ceph_client_metric *m,s64 latency); +extern void ceph_mdsc_update_metadata_latency (struct ceph_client_metric *m, +s64 latency);#endif
Signature · 1 line
2.21.0
PATCH v4 6/8 ceph: periodically send perf metrics to MDS
In reply to [email protected]From: Xiubo Li <[email protected]>
Add enable/disable sending metrics to MDS debugfs and disabled as default, if it's enabled the kclient will send metrics every second.
This will send global dentry lease hit/miss and read/write/metadata latency metrics and each session's caps hit/miss metric to MDS.
Every time only sends the global metrics once via any availible session.
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 44 +++++++- fs/ceph/mds_client.c | 205 ++++++++++++++++++++++++++++++++--- fs/ceph/mds_client.h | 3 + fs/ceph/super.h | 1 + include/linux/ceph/ceph_fs.h | 77 +++++++++++++ 5 files changed, 312 insertions(+), 18 deletions(-)
fs/ceph/debugfs.c +42 -2
@@ -124,6 +124,40 @@ static int mdsc_show(struct seq_file *s, void *p)return 0 ; } +/* +* metrics debugfs +*/ +static int sending_metrics_set (void *data,u64 val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +if (val >1 ) { +pr_err ("Invalid sending metrics set value %llu\n" , val); +return -EINVAL ; + } + +mutex_lock (&mdsc->mutex); + mdsc->sending_metrics = (unsigned int )val; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +static int metric_show (struct seq_file *s,void *p) {struct ceph_fs_client *fsc = s->private;
@@ -308,11 +342,9 @@ static int congestion_kb_get(void *data, u64 *val) *val = (u64 )fsc->mount_options->congestion_kb;return 0 ; } -DEFINE_SIMPLE_ATTRIBUTE (congestion_kb_fops, congestion_kb_get, congestion_kb_set,"%llu\n" ); -void ceph_fs_debugfs_cleanup (struct ceph_fs_client *fsc) {dout ("ceph_fs_debugfs_cleanup\n" );
@@ -322,6 +354,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_mds_sessions);debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric); +debugfs_remove (fsc->debugfs_sending_metrics);debugfs_remove (fsc->debugfs_mdsc); }
@@ -362,6 +395,13 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &mdsc_show_fops); + fsc->debugfs_sending_metrics = +debugfs_create_file ("sending_metrics" , +0600 , + fsc->client->debugfs_dir, + fsc, + &sending_metrics_fops); + fsc->debugfs_metric = debugfs_create_file("metrics", 0400, fsc->client->debugfs_dir,
fs/ceph/mds_client.c +189 -16
263 lines not on this page · raw message
fs/ceph/mds_client.h +3 -0
@@ -468,6 +468,9 @@ struct ceph_mds_client {struct list_head dentry_leases;/* fifo list */ struct list_head dentry_dir_leases;/* lru list */ +/* metrics */ +unsigned int sending_metrics; +unsigned int ticks;struct ceph_client_metric metric;spinlock_t snapid_map_lock;
fs/ceph/super.h +1 -0
@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_sending_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
include/linux/ceph/ceph_fs.h +77 -0
89 lines not on this page · raw message
Signature · 1 line
2.21.0
PATCH v4 7/8 ceph: add reset metrics support
In reply to [email protected]From: Xiubo Li <[email protected]>
This will reset the most metric counters, except the cap and dentry total numbers.
Sometimes we need to discard the old metrics and start to get new metrics.
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/super.h | 1 + 2 files changed, 58 insertions(+)
fs/ceph/debugfs.c +57 -0
@@ -158,6 +158,55 @@ static int sending_metrics_get(void *data, u64 *val)DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, sending_metrics_set,"%llu\n" ); +static int reset_metrics_set (void *data,u64 val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_client_metric *metric = &mdsc->metric; +int i; + +if (val !=1 ) { +pr_err ("Invalid reset metrics set value %llu\n" , val); +return -EINVAL ; + } + +percpu_counter_set (&metric->d_lease_hit,0 ); +percpu_counter_set (&metric->d_lease_mis,0 ); + +spin_lock (&metric->read_lock); +memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_reads,0 ), +spin_unlock (&metric->read_lock); + +spin_lock (&metric->write_lock); +memset (&metric->write_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_writes,0 ), +spin_unlock (&metric->write_lock); + +spin_lock (&metric->metadata_lock); +memset (&metric->metadata_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_metadatas,0 ), +spin_unlock (&metric->metadata_lock); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +percpu_counter_set (&session->i_caps_hit,0 ); +percpu_counter_set (&session->i_caps_mis,0 ); +ceph_put_mds_session (session); + } + +mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +DEFINE_SIMPLE_ATTRIBUTE (reset_metrics_fops,NULL , reset_metrics_set,"%llu\n" ); +static int metric_show (struct seq_file *s,void *p) {struct ceph_fs_client *fsc = s->private;
@@ -355,6 +404,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_sending_metrics); +debugfs_remove (fsc->debugfs_reset_metrics);debugfs_remove (fsc->debugfs_mdsc); }
@@ -402,6 +452,13 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &sending_metrics_fops); + fsc->debugfs_reset_metrics = +debugfs_create_file ("reset_metrics" , +0600 , + fsc->client->debugfs_dir, + fsc, + &reset_metrics_fops); + fsc->debugfs_metric = debugfs_create_file("metrics", 0400, fsc->client->debugfs_dir,
fs/ceph/super.h +1 -0
@@ -129,6 +129,7 @@ struct ceph_fs_client {struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap;struct dentry *debugfs_sending_metrics; +struct dentry *debugfs_reset_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
Signature · 1 line
2.21.0
Ilya Dryomov
maintainerRe: [PATCH v4 7/8] ceph: add reset metrics support
Quoted 103 lines · On Thu, Jan 16, 2020 at 11:39 AM <[email protected]> wrote:
From: Xiubo Li <[email protected]>
This will reset the most metric counters, except the cap and dentry total numbers.
Sometimes we need to discard the old metrics and start to get new metrics.
URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/debugfs.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/super.h | 1 + 2 files changed, 58 insertions(+)fs/ceph/debugfs.c +57
-0@@ -158,6 +158,55 @@ static int sending_metrics_get(void *data, u64 *val)DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, sending_metrics_set,"%llu\n" ); +static int reset_metrics_set (void *data,u64 val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_client_metric *metric = &mdsc->metric; +int i; + +if (val !=1 ) { +pr_err ("Invalid reset metrics set value %llu\n" , val); +return -EINVAL ; + } + +percpu_counter_set (&metric->d_lease_hit,0 ); +percpu_counter_set (&metric->d_lease_mis,0 ); + +spin_lock (&metric->read_lock); +memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_reads,0 ), +spin_unlock (&metric->read_lock); + +spin_lock (&metric->write_lock); +memset (&metric->write_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_writes,0 ), +spin_unlock (&metric->write_lock); + +spin_lock (&metric->metadata_lock); +memset (&metric->metadata_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_metadatas,0 ), +spin_unlock (&metric->metadata_lock); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +percpu_counter_set (&session->i_caps_hit,0 ); +percpu_counter_set (&session->i_caps_mis,0 ); +ceph_put_mds_session (session); + } + +mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +DEFINE_SIMPLE_ATTRIBUTE (reset_metrics_fops,NULL , reset_metrics_set,"%llu\n" ); +static int metric_show (struct seq_file *s,void *p) {struct ceph_fs_client *fsc = s->private;@@ -355,6 +404,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_sending_metrics); +debugfs_remove (fsc->debugfs_reset_metrics);debugfs_remove (fsc->debugfs_mdsc); }@@ -402,6 +452,13 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &sending_metrics_fops); + fsc->debugfs_reset_metrics = +debugfs_create_file ("reset_metrics" , +0600 , + fsc->client->debugfs_dir, + fsc, + &reset_metrics_fops); + fsc->debugfs_metric = debugfs_create_file("metrics", 0400, fsc->client->debugfs_dir,fs/ceph/super.h +0
-0
fs/ceph/super.h struct ceph_fs_client { · line 135
Quoted 7 lines
fs/ceph/super.h +1
-0@@ -129,6 +129,7 @@ struct ceph_fs_client {struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap;struct dentry *debugfs_sending_metrics; +struct dentry *debugfs_reset_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
Do we need a separate attribute for this? Did you think about making metrics attribute writeable and accepting some string, e.g. "reset"?
Thanks,
Ilya
Re: [PATCH v4 7/8] ceph: add reset metrics support
Quoted 104 lines · Ilya Dryomov
Quoted 103 lines · On Thu, Jan 16, 2020 at 11:39 AM <[email protected]> wrote:
From: Xiubo Li <[email protected]>
This will reset the most metric counters, except the cap and dentry total numbers.
Sometimes we need to discard the old metrics and start to get new metrics.
URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/debugfs.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/super.h | 1 + 2 files changed, 58 insertions(+)fs/ceph/debugfs.c +57
-0@@ -158,6 +158,55 @@ static int sending_metrics_get(void *data, u64 *val)DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, sending_metrics_set,"%llu\n" ); +static int reset_metrics_set (void *data,u64 val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_client_metric *metric = &mdsc->metric; +int i; + +if (val !=1 ) { +pr_err ("Invalid reset metrics set value %llu\n" , val); +return -EINVAL ; + } + +percpu_counter_set (&metric->d_lease_hit,0 ); +percpu_counter_set (&metric->d_lease_mis,0 ); + +spin_lock (&metric->read_lock); +memset (&metric->read_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_reads,0 ), +spin_unlock (&metric->read_lock); + +spin_lock (&metric->write_lock); +memset (&metric->write_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_writes,0 ), +spin_unlock (&metric->write_lock); + +spin_lock (&metric->metadata_lock); +memset (&metric->metadata_latency_sum,0 ,sizeof (struct timespec64 )); +atomic64_set (&metric->total_metadatas,0 ), +spin_unlock (&metric->metadata_lock); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +percpu_counter_set (&session->i_caps_hit,0 ); +percpu_counter_set (&session->i_caps_mis,0 ); +ceph_put_mds_session (session); + } + +mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +DEFINE_SIMPLE_ATTRIBUTE (reset_metrics_fops,NULL , reset_metrics_set,"%llu\n" ); +static int metric_show (struct seq_file *s,void *p) {struct ceph_fs_client *fsc = s->private;@@ -355,6 +404,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_sending_metrics); +debugfs_remove (fsc->debugfs_reset_metrics);debugfs_remove (fsc->debugfs_mdsc); }@@ -402,6 +452,13 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &sending_metrics_fops); + fsc->debugfs_reset_metrics = +debugfs_create_file ("reset_metrics" , +0600 , + fsc->client->debugfs_dir, + fsc, + &reset_metrics_fops); + fsc->debugfs_metric = debugfs_create_file("metrics", 0400, fsc->client->debugfs_dir,fs/ceph/super.h +0
-0@@ -129,6 +129,7 @@ struct ceph_fs_client {struct dentry *debugfs_bdi;
Quoted 8 lines
fs/ceph/super.h struct ceph_fs_client { · line 135
Quoted 6 lines
fs/ceph/super.h +1
-0struct dentry *debugfs_mdsc, *debugfs_mdsmap;struct dentry *debugfs_sending_metrics; +struct dentry *debugfs_reset_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif Do we need a separate attribute for this? Did you think about making metrics attribute writeable and accepting some string, e.g. "reset"?
Let's make the "metrics" writeable, which will means reset.
Thanks.
Quoted 4 lines
Thanks,
Ilya
PATCH v4 8/8 ceph: send client provided metric flags in client metadata
In reply to [email protected]From: Xiubo Li <[email protected]>
Will send the metric flags to MDS, currently it supports the cap, dentry lease, read latency, write latency and metadata latency.
URL: https://tracker.ceph.com/issues/43435
--- fs/ceph/mds_client.c | 47 ++++++++++++++++++++++++++++++++++++++++++-- fs/ceph/mds_client.h | 14 +++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-)
fs/ceph/mds_client.c +45 -2
@@ -1096,6 +1096,41 @@ static void encode_supported_features(void **p, void *end) } } +static const unsigned char metric_bits[] =CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED ; +#define METRIC_BYTES (cnt) (DIV_ROUND_UP((size_t)metric_bits[cnt - 1] + 1, 64) * 8) +static void encode_metric_spec (void **p,void *end) +{ +static const size_t count =ARRAY_SIZE (metric_bits); + +/* header */ +BUG_ON (*p +2 > end); +ceph_encode_8 (p,1 );/* version */ +ceph_encode_8 (p,1 );/* compat */ + +if (count >0 ) { +size_t i; +size_t size =METRIC_BYTES (count); + +BUG_ON (*p +4 +4 + size > end); + +/* metric spec info length */ +ceph_encode_32 (p,4 + size); + +/* metric spec */ +ceph_encode_32 (p, size); +memset (*p,0 , size); +for (i =0 ; i < count; i++) + ((unsigned char *)(*p))[i /8 ] |=BIT (metric_bits[i] %8 ); + *p += size; + }else { +BUG_ON (*p +4 +4 > end); +/* metric spec info length */ +ceph_encode_32 (p,4 ); +/* metric spec */ +ceph_encode_32 (p,0 ); + } +} +/* * session message, specialization for CEPH_SESSION_REQUEST_OPEN * to include additional client metadata fields.
@@ -1135,6 +1170,13 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6 size =FEATURE_BYTES (count); extra_bytes +=4 + size; +/* metric spec */ + size =0 ; + count =ARRAY_SIZE (metric_bits); +if (count >0 ) + size =METRIC_BYTES (count); + extra_bytes +=2 +4 +4 + size; +/* Allocate the message */ msg =ceph_msg_new (CEPH_MSG_CLIENT_SESSION ,sizeof (*h) + extra_bytes,GFP_NOFS , false);
@@ -1153,9 +1195,9 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6* Serialize client metadata into waiting buffer space, using * the format that userspace expects for map<string, string> * -* ClientSession messages with metadata are v3 +* ClientSession messages with metadata are v4 */ - msg->hdr.version =cpu_to_le16 (3 ); + msg->hdr.version =cpu_to_le16 (4 ); msg->hdr.compat_version =cpu_to_le16 (1 );/* The write pointer, following the session_head structure */
@@ -1178,6 +1220,7 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6 }encode_supported_features (&p, end); +encode_metric_spec (&p, end); msg->front.iov_len = p - msg->front.iov_base; msg->hdr.front_len =cpu_to_le32 (msg->front.iov_len);
fs/ceph/mds_client.h +14 -0
@@ -42,6 +42,20 @@ enum ceph_feature_type { }#define CEPHFS_FEATURES_CLIENT_REQUIRED {} +/* +* This will always have the highest metric bit value +* as the last element of the array. +*/ +#define CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED { \ + CLIENT_METRIC_TYPE_CAP_INFO, \ + CLIENT_METRIC_TYPE_READ_LATENCY, \ + CLIENT_METRIC_TYPE_WRITE_LATENCY, \ + CLIENT_METRIC_TYPE_METADATA_LATENCY, \ + CLIENT_METRIC_TYPE_DENTRY_LEASE, \ + \ + CLIENT_METRIC_TYPE_MAX, \ +} +/* * Some lock dependencies: *
Signature · 1 line
2.21.0