ceph: add perf metrics support
Subsystems (2)
v5 cover ceph: add perf metrics support
From: Xiubo Li <[email protected]>
Changed in V2:
- add read/write/metadata latency metric support.
- add and send client provided metric flags in client metadata
- addressed the comments from Ilya and merged the 4/4 patch into 3/4.
- addressed all the other comments in v1 series.
Changed in V3:
- addressed Jeff's comments and let's the callers do the metric counting.
- with some small fixes for the read/write latency
- tested based on the latest testing branch
Changed in V4:
- fix the lock issue
Changed in V5:
- add r_end_stamp for the osdc request
- delete reset metric and move it to metric sysfs
- move ceph_osdc_{read,write}pages to ceph.ko
- use percpu counters instead for read/write/metadata latencies
Changed in V5 resend:
- add the lost patch.
It will send the metrics to the MDSs every second if sending_metrics is enabled, disable as default.
We can get the metrics from the debugfs:
$ cat /sys/kernel/debug/ceph/0c93a60d-5645-4c46-8568-4c8f63db4c7f.client4267/metrics item total sum_lat(us) avg_lat(us) ----------------------------------------------------- read 13 417000 32076 write 42 131205000 3123928 metadata 104 493000 4740
item total miss hit
-------------------------------------------------d_lease 204 0 918
session caps miss hit
-------------------------------------------------
0 204 213 368218
In the MDS side, we can get the metrics(NOTE: the latency is in nanosecond):
$ ./bin/ceph fs perf stats | python -m json.tool
{
"client_metadata": {
"client.4267": {
"IP": "v1:192.168.195.165",
"hostname": "fedora1",
"mount_point": "N/A",
"root": "/"
}
},
"counters": [
"cap_hit"
],
"global_counters": [
"read_latency",
"write_latency",
"metadata_latency",
"dentry_lease_hit"
],
"global_metrics": {
"client.4267": [
[
0,
32076923
],
[
3,
123928571
],
[
0,
4740384
],
[
918,
0
]
]
},
"metrics": {
"delayed_ranks": [],
"mds.0": {
"client.4267": [
[
368218,
213
]
]
}
}
}The provided metric flags in client metadata
$./bin/cephfs-journal-tool --rank=1:0 event get --type=SESSION json Wrote output to JSON file 'dump'
$ cat dump
[
{
"client instance": "client.4275 v1:192.168.195.165:0/461391971",
"open": "true",
"client map version": 1,
"inos": "[]",
"inotable version": 0,
"client_metadata": {
"client_features": {
"feature_bits": "0000000000001bff"
},
"metric_spec": {
"metric_flags": {
"feature_bits": "000000000000001f"
}
},
"entity_id": "",
"hostname": "fedora1",
"kernel_version": "5.5.0-rc2+",
"root": "/"
}
},
[...]*** BLURB HERE ***
Xiubo Li (11):
ceph: add global dentry lease metric support
ceph: add caps perf metric for each session
ceph: move ceph_osdc_{read,write}pages to ceph.ko
ceph: add r_end_stamp for the osdc request
ceph: add global read latency metric support
ceph: add global write latency metric support
ceph: add global metadata perf metric support
ceph: periodically send perf metrics to MDS
ceph: add CEPH_DEFINE_RW_FUNC helper support
ceph: add reset metrics support
ceph: send client provided metric flags in client metadata
fs/ceph/acl.c | 2 + fs/ceph/addr.c | 106 +++++++++- fs/ceph/caps.c | 74 +++++++ fs/ceph/debugfs.c | 168 ++++++++++++++- fs/ceph/dir.c | 27 ++- fs/ceph/file.c | 26 +++ fs/ceph/mds_client.c | 350 ++++++++++++++++++++++++++++++-- fs/ceph/mds_client.h | 10 + fs/ceph/metric.h | 150 ++++++++++++++ fs/ceph/quota.c | 9 +- fs/ceph/super.h | 14 ++ fs/ceph/xattr.c | 17 +- include/linux/ceph/ceph_fs.h | 1 + include/linux/ceph/debugfs.h | 14 ++ include/linux/ceph/osd_client.h | 18 +- net/ceph/osd_client.c | 81 +------- 16 files changed, 935 insertions(+), 132 deletions(-) create mode 100644 fs/ceph/metric.h
Signature · 1 line
2.21.0
PATCH RESEND v5 1/11 ceph: add global dentry lease metric support
From: Xiubo Li <[email protected]>
For the dentry lease we will only count the hit/miss info triggered from the vfs calls, for the cases like request reply handling and perodically ceph_trim_dentries() we will ignore them.
Currently only the debugfs is support:
The output will be:
item total miss hit
-------------------------------------------------d_lease 11 7 141
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 32 ++++++++++++++++++++++++++++---- fs/ceph/dir.c | 18 ++++++++++++++++-- fs/ceph/mds_client.c | 37 +++++++++++++++++++++++++++++++++++-- fs/ceph/mds_client.h | 9 +++++++++ fs/ceph/super.h | 1 + 5 files changed, 89 insertions(+), 8 deletions(-)
fs/ceph/debugfs.c +28 -4
@@ -124,6 +124,22 @@ static int mdsc_show(struct seq_file *s, void *p)return 0 ; } +static int metric_show (struct seq_file *s,void *p) +{ +struct ceph_fs_client *fsc = s->private; +struct ceph_mds_client *mdsc = fsc->mdsc; + +seq_printf (s,"item total miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +seq_printf (s,"%-14s%-16lld%-16lld%lld\n" ,"d_lease" , +atomic64_read (&mdsc->metric.total_dentries), +percpu_counter_sum (&mdsc->metric.d_lease_mis), +percpu_counter_sum (&mdsc->metric.d_lease_hit)); + +return 0 ; +} +static int caps_show_cb (struct inode *inode,struct ceph_cap *cap,void *p) {struct seq_file *s = p;
@@ -220,6 +236,7 @@ static int mds_sessions_show(struct seq_file *s, void *ptr) CEPH_DEFINE_SHOW_FUNC(mdsmap_show) CEPH_DEFINE_SHOW_FUNC(mdsc_show) +CEPH_DEFINE_SHOW_FUNC(metric_show) CEPH_DEFINE_SHOW_FUNC(caps_show) CEPH_DEFINE_SHOW_FUNC(mds_sessions_show)
@@ -255,6 +272,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_mdsmap);debugfs_remove (fsc->debugfs_mds_sessions);debugfs_remove (fsc->debugfs_caps); +debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_mdsc); }
@@ -295,11 +313,17 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &mdsc_show_fops); + fsc->debugfs_metric =debugfs_create_file ("metrics" , +0400 , + fsc->client->debugfs_dir, + fsc, + &metric_show_fops); + fsc->debugfs_caps =debugfs_create_file ("caps" , -0400 , - fsc->client->debugfs_dir, - fsc, - &caps_show_fops); +0400 , + fsc->client->debugfs_dir, + fsc, + &caps_show_fops); }
fs/ceph/dir.c +16 -2
@@ -38,6 +38,8 @@ static int __dir_lease_try_check(const struct dentry *dentry);static int ceph_d_init (struct dentry *dentry) {struct ceph_dentry_info *di; +struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb); +struct ceph_mds_client *mdsc = fsc->mdsc; di =kmem_cache_zalloc (ceph_dentry_cachep,GFP_KERNEL );if (!di)
@@ -48,6 +50,9 @@ static int ceph_d_init(struct dentry *dentry) di->time = jiffies; dentry->d_fsdata = di;INIT_LIST_HEAD (&di->lease_list); + +atomic64_inc (&mdsc->metric.total_dentries); +return 0 ; }
@@ -1613,6 +1618,7 @@ static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry)*/ static int ceph_d_revalidate (struct dentry *dentry,unsigned int flags) { +struct ceph_mds_client *mdsc;int valid =0 ;struct dentry *parent;struct inode *dir, *inode;
@@ -1651,9 +1657,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags) } } + mdsc =ceph_sb_to_client (dir->i_sb)->mdsc;if (!valid) { -struct ceph_mds_client *mdsc = -ceph_sb_to_client (dir->i_sb)->mdsc;struct ceph_mds_request *req;int op, err;u32 mask;
@@ -1661,6 +1666,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)if (flags &LOOKUP_RCU )return -ECHILD ; +percpu_counter_inc (&mdsc->metric.d_lease_mis); + op =ceph_snap (dir) ==CEPH_SNAPDIR ?CEPH_MDS_OP_LOOKUPSNAP :CEPH_MDS_OP_LOOKUP ; req =ceph_mdsc_create_request (mdsc, op,USE_ANY_MDS );
@@ -1692,6 +1699,8 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)dout ("d_revalidate %p lookup result=%d\n" , dentry, err); } + }else { +percpu_counter_inc (&mdsc->metric.d_lease_hit); }dout ("d_revalidate %p %s\n" , dentry, valid ?"valid" :"invalid" );
@@ -1700,6 +1709,7 @@ static int ceph_d_revalidate(struct dentry *dentry, unsigned int flags)if (!(flags &LOOKUP_RCU ))dput (parent); +return valid; }
@@ -1734,9 +1744,13 @@ static int ceph_d_delete(const struct dentry *dentry)static void ceph_d_release (struct dentry *dentry) {struct ceph_dentry_info *di =ceph_dentry (dentry); +struct ceph_fs_client *fsc =ceph_sb_to_client (dentry->d_sb); +struct ceph_mds_client *mdsc = fsc->mdsc;dout ("d_release %p\n" , dentry); +atomic64_dec (&mdsc->metric.total_dentries); +spin_lock (&dentry->d_lock);__dentry_lease_unlist (di); dentry->d_fsdata =NULL ;
fs/ceph/mds_client.c +35 -2
@@ -4158,10 +4158,31 @@ static void delayed_work(struct work_struct *work)schedule_delayed (mdsc); } +static int ceph_mdsc_metric_init (struct ceph_client_metric *metric) +{ +int ret; + +if (!metric) +return -EINVAL ; + +atomic64_set (&metric->total_dentries,0 ); + ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL ); +if (ret) +return ret; + ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL ); +if (ret) { +percpu_counter_destroy (&metric->d_lease_hit); +return ret; + } + +return 0 ; +} +int ceph_mdsc_init (struct ceph_fs_client *fsc) {struct ceph_mds_client *mdsc; +int err; mdsc =kzalloc (sizeof (struct ceph_mds_client ),GFP_NOFS );if (!mdsc)
@@ -4170,8 +4191,8 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)mutex_init (&mdsc->mutex); mdsc->mdsmap =kzalloc (sizeof (*mdsc->mdsmap),GFP_NOFS );if (!mdsc->mdsmap) { -kfree (mdsc); -return -ENOMEM ; + err = -ENOMEM ; + goto err_mdsc; } fsc->mdsc = mdsc;
@@ -4210,6 +4231,9 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)init_waitqueue_head (&mdsc->cap_flushing_wq);INIT_WORK (&mdsc->cap_reclaim_work, ceph_cap_reclaim_work);atomic_set (&mdsc->cap_reclaim_pending,0 ); + err =ceph_mdsc_metric_init (&mdsc->metric); +if (err) + goto err_mdsmap;spin_lock_init (&mdsc->dentry_list_lock);INIT_LIST_HEAD (&mdsc->dentry_leases);
@@ -4228,6 +4252,12 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)strscpy (mdsc->nodename,utsname ()->nodename,sizeof (mdsc->nodename));return 0 ; + +err_mdsmap: +kfree (mdsc->mdsmap); +err_mdsc: +kfree (mdsc); +return err; }/*
@@ -4485,6 +4515,9 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.d_lease_mis); +percpu_counter_destroy (&mdsc->metric.d_lease_hit); + fsc->mdsc =NULL ;kfree (mdsc);dout ("mdsc_destroy %p done\n" , mdsc);
fs/ceph/mds_client.h +9 -0
@@ -358,6 +358,13 @@ struct cap_wait {int want; }; +/* This is the global metrics */ +struct ceph_client_metric { +atomic64_t total_dentries; +struct percpu_counter d_lease_hit; +struct percpu_counter d_lease_mis; +}; +/* * mds client state */
@@ -446,6 +453,8 @@ struct ceph_mds_client {struct list_head dentry_leases;/* fifo list */ struct list_head dentry_dir_leases;/* lru list */ +struct ceph_client_metric metric; +spinlock_t snapid_map_lock;struct rb_root snapid_map_tree;struct list_head snapid_map_lru;
fs/ceph/super.h +1 -0
@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
Signature · 1 line
2.21.0
PATCH RESEND v5 2/11 ceph: add caps perf metric for each session
In reply to [email protected]From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need" mask it means hit, or missed.
item total miss hit
-------------------------------------------------d_lease 295 0 993
session caps miss hit
-------------------------------------------------
0 295 107 4119 1 1 107 9
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/acl.c | 2 ++ fs/ceph/addr.c | 2 ++ fs/ceph/caps.c | 74 ++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/debugfs.c | 20 ++++++++++++ fs/ceph/dir.c | 9 ++++-- fs/ceph/file.c | 3 ++ fs/ceph/mds_client.c | 16 +++++++++- fs/ceph/mds_client.h | 3 ++ fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)
fs/ceph/acl.c +2 -0
@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); +if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else
fs/ceph/addr.c +2 -0
@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM ; goto out; } + +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); err =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , true);if (err <0 ) {
fs/ceph/caps.c +74 -0
104 lines not on this page · raw message
fs/ceph/debugfs.c +20 -0
@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i;seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p)percpu_counter_sum (&mdsc->metric.d_lease_mis), percpu_counter_sum(&mdsc->metric.d_lease_hit)); +seq_printf (s,"\n" ); +seq_printf (s,"session caps miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +seq_printf (s,"%-14d%-16d%-16lld%lld\n" , i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); +ceph_put_mds_session (session); + } +mutex_unlock (&mdsc->mutex); +return 0 ; }
fs/ceph/dir.c +7 -2
@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_mds_client *mdsc = fsc->mdsc;int i; -int err; +int err, ret =-1 ;unsigned frag =-1 ;struct ceph_mds_reply_info_parsed *rinfo;
@@ -346,13 +346,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {int shared_gen =atomic_read (&ci->i_shared_gen); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )return err; } else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); }
@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry,struct ceph_dentry_info *di =ceph_dentry (dentry);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_FILE_SHARED ); +dout (" dir %p flags are %d\n" , dir, ci->i_ceph_flags); if (strncmp(dentry->d_name.name, fsc->mount_options->snapdir_name,
fs/ceph/file.c +3 -0
@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, struct file *file)* asynchronously. */ spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, wanted); +if (__ceph_is_any_real_caps (ci) && (((fmode &CEPH_FILE_MODE_WR ) ==0 ) || ci->i_auth_cap)) {int mds_wanted =__ceph_caps_mds_wanted (ci, true);
@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)return -ENOMEM ; } +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); statret =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , !!page);if (statret <0 ) {
fs/ceph/mds_client.c +15 -1
@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s)if (refcount_dec_and_test (&s->s_ref)) {if (s->s_auth.authorizer)ceph_auth_destroy_authorizer (s->s_auth.authorizer); +percpu_counter_destroy (&s->i_caps_hit); +percpu_counter_destroy (&s->i_caps_mis);kfree (s); } }
@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) {struct ceph_mds_session *s; +int err;if (mds >= mdsc->mdsmap->possible_max_rank)return ERR_PTR (-EINVAL );
@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,dout ("%s: realloc to %d\n" , __func__, newmax); sa =kcalloc (newmax,sizeof (void *),GFP_NOFS ); -if (!sa) +if (!sa) { + err = -ENOMEM ; goto fail_realloc; + }if (mdsc->sessions) {memcpy (sa, mdsc->sessions, mdsc->max_sessions *sizeof (void *));
@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,INIT_LIST_HEAD (&s->s_cap_flushing); + err =percpu_counter_init (&s->i_caps_hit,0 ,GFP_NOFS ); +if (err) + goto fail_realloc; + err =percpu_counter_init (&s->i_caps_mis,0 ,GFP_NOFS ); +if (err) + goto fail_init; + mdsc->sessions[mds] = s;atomic_inc (&mdsc->num_sessions);refcount_inc (&s->s_ref);/* one ref to sessions[], one to caller */
@@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,return s; +fail_init: +percpu_counter_destroy (&s->i_caps_hit); fail_realloc:kfree (s);return ERR_PTR (-ENOMEM );
fs/ceph/mds_client.h +3 -0
@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting;/* waiting requests */ struct list_head s_unsafe;/* unsafe requests */ + +struct percpu_counter i_caps_hit; +struct percpu_counter i_caps_mis; };/*
fs/ceph/quota.c +7 -2
@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc,return NULL ; }if (qri->inode) { +struct ceph_inode_info *ci =ceph_inode (qri->inode); +int ret; + +ceph_caps_metric (ci,CEPH_STAT_CAP_INODE ); +/* get caps */ -int ret =__ceph_do_getattr (qri->inode,NULL , -CEPH_STAT_CAP_INODE , true); + ret =__ceph_do_getattr (qri->inode,NULL , +CEPH_STAT_CAP_INODE , true);if (ret >=0 ) in = qri->inode;else
fs/ceph/super.h +11 -0
@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci)return !RB_EMPTY_ROOT (&ci->i_caps); } +extern void __ceph_caps_metric (struct ceph_inode_info *ci,int mask); +static inline void ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, mask); +spin_unlock (&ci->i_ceph_lock); +} +extern int __ceph_caps_issued (struct ceph_inode_info *ci,int *implemented);extern int __ceph_caps_issued_mask (struct ceph_inode_info *ci,int mask,int t); extern int __ceph_caps_issued_other(struct ceph_inode_info *ci,
@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int ceph_do_getattr (struct inode *inode,int mask,bool force) { +struct ceph_inode_info *ci =ceph_inode (inode); + +ceph_caps_metric (ci, mask);return __ceph_do_getattr (inode,NULL , mask, force); }extern int ceph_permission (struct inode *inode,int mask);
fs/ceph/xattr.c +14 -3
@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,struct ceph_vxattr *vxattr =NULL ;int req_mask;ssize_t err; +int ret =-1 ;/* let's see if a virtual xattr was requested */ vxattr =ceph_match_vxattr (inode, name);
@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (ci->i_xattrs.version ==0 || !((req_mask &CEPH_CAP_XATTR_SHARED ) || -__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { + (ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock);/* security module gets xattr while filling trace */
@@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);
@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)struct ceph_inode_info *ci =ceph_inode (inode);bool len_only = (size ==0 );u32 namelen; -int err; +int err, ret =-1 ;spin_lock (&ci->i_ceph_lock);dout ("listxattr %p ver=%lld index_ver=%lld\n" , inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !(ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);
Signature · 1 line
2.21.0
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 49 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need" mask it means hit, or missed.
item total miss hit
-------------------------------------------------d_lease295 0 993session caps miss hit
-------------------------------------------------0 295 107 4119 1 1 107 9URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 ++ fs/ceph/addr.c | 2 ++ fs/ceph/caps.c | 74 ++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/debugfs.c | 20 ++++++++++++ fs/ceph/dir.c | 9 ++++-- fs/ceph/file.c | 3 ++ fs/ceph/mds_client.c | 16 +++++++++- fs/ceph/mds_client.h | 3 ++ fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)fs/ceph/acl.c +2
-0@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); +if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else fs/ceph/addr.c +0
-0
fs/ceph/addr.c int ceph_uninline_data(struct file *filp, struct page *locked_page) · line 1710
Quoted 5 lines
fs/ceph/addr.c +2
-0@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM ; goto out; } + +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA );
Should a check for inline data really count here?
Quoted 8 lines
fs/ceph/addr.c +0
-0err =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , true);if (err <0 ) {fs/ceph/caps.c +0
-0
fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 788
Quoted 6 lines
fs/ceph/caps.c +3
-0@@ -783,6 +783,75 @@ static int __cap_is_valid(struct ceph_cap *cap)return 1 ; } +/* +* Counts the cap metric. +*/
This needs some comments. Specifically, what should this be counting and how?
Quoted 56 lines
fs/ceph/caps.c +56
-0+void __ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +int have = ci->i_snap_caps; +struct ceph_mds_session *s; +struct ceph_cap *cap; +struct rb_node *p; +bool skip_auth = false; + +lockdep_assert_held (&ci->i_ceph_lock); + +if (mask <=0 ) +return ; + +/* Counts the snap caps metric in the auth cap */ +if (ci->i_auth_cap) { + cap = ci->i_auth_cap; +if (have) { + have |= cap->issued; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap->issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & have) +percpu_counter_inc (&s->i_caps_hit); +else +percpu_counter_inc (&s->i_caps_mis); +ceph_put_mds_session (s); + } + skip_auth = true; + } + } + +if ((mask & have) == mask) +return ; + +/* Checks others */ +for (p =rb_first (&ci->i_caps); p; p =rb_next (p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); +if (!__cap_is_valid (cap)) +continue ; + +if (skip_auth && cap == ci->i_auth_cap) +continue ; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap->issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & cap->issued) +percpu_counter_inc (&s->i_caps_hit); +else
fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 854
Quoted 10 lines
fs/ceph/caps.c +10
-0+percpu_counter_inc (&s->i_caps_mis); +ceph_put_mds_session (s); + } + + have |= cap->issued; +if ((mask & have) == mask) +return ; + } +} +
I'm trying to understand what happens with the above when more than one ceph_cap has the same bit set in "issued". For instance:
Suppose we're doing the check for a statx call, and we're trying to get caps for pAsFsLs. We have two MDS's and they've each granted us caps for the inode, say:
MDS 0: pAs
MDS 1: pAsLsFs
We check the cap0 first, and consider it a hit, and then we check cap1 and consider it a hit as well. So that seems like it's being double- counted.
ISTM, that what you really want to do here is logically or all of the cap->issued fields together, and then check that vs. the mask value, and count only one hit or miss per inode.
That said, it's not 100% clear what you're counting as a hit or miss here, so please let me know if I have that wrong.
Quoted 315 lines
fs/ceph/caps.c +5
-0/* * Return set of valid cap bits issued to us. Note that caps time * out, and may be invalidated in bulk if the client session times out @@ -2746,6 +2815,7 @@ static void check_max_size(struct inode *inode, loff_t endoff)int ceph_try_get_caps (struct inode *inode,int need,int want,bool nonblock,int *got) { +struct ceph_inode_info *ci =ceph_inode (inode);int ret;BUG_ON (need & ~CEPH_CAP_FILE_RD );@@ -2758,6 +2828,7 @@ int ceph_try_get_caps(struct inode *inode, int need, int want,BUG_ON (want & ~(CEPH_CAP_FILE_CACHE |CEPH_CAP_FILE_LAZYIO |CEPH_CAP_FILE_SHARED |CEPH_CAP_FILE_EXCL |CEPH_CAP_ANY_DIR_OPS )); +ceph_caps_metric (ci, need | want); ret =try_get_cap_refs (inode, need, want,0 , nonblock, got);return ret == -EAGAIN ?0 : ret; }@@ -2784,6 +2855,8 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen))return -EBADF ; +ceph_caps_metric (ci, need | want); +while (true) {if (endoff >0 )check_max_size (inode, endoff);@@ -2871,6 +2944,7 @@ int ceph_get_caps(struct file *filp, int need, int want,* getattr request will bring inline data into * page cache */ +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); ret =__ceph_do_getattr (inode,NULL ,CEPH_STAT_CAP_INLINE_DATA , true);fs/ceph/debugfs.c +20
-0@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i;seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p)percpu_counter_sum (&mdsc->metric.d_lease_mis), percpu_counter_sum(&mdsc->metric.d_lease_hit)); +seq_printf (s,"\n" ); +seq_printf (s,"session caps miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +seq_printf (s,"%-14d%-16d%-16lld%lld\n" , i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); +ceph_put_mds_session (session); + } +mutex_unlock (&mdsc->mutex); +return 0 ; }fs/ceph/dir.c +7
-2@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_mds_client *mdsc = fsc->mdsc;int i; -int err; +int err, ret =-1 ;unsigned frag =-1 ;struct ceph_mds_reply_info_parsed *rinfo;@@ -346,13 +346,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {int shared_gen =atomic_read (&ci->i_shared_gen); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )return err; } else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); }@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry,struct ceph_dentry_info *di =ceph_dentry (dentry);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_FILE_SHARED ); +dout (" dir %p flags are %d\n" , dir, ci->i_ceph_flags); if (strncmp(dentry->d_name.name, fsc->mount_options->snapdir_name,fs/ceph/file.c +3
-0@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, struct file *file)* asynchronously. */ spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, wanted); +if (__ceph_is_any_real_caps (ci) && (((fmode &CEPH_FILE_MODE_WR ) ==0 ) || ci->i_auth_cap)) {int mds_wanted =__ceph_caps_mds_wanted (ci, true);@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)return -ENOMEM ; } +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); statret =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , !!page);if (statret <0 ) {fs/ceph/mds_client.c +15
-1@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s)if (refcount_dec_and_test (&s->s_ref)) {if (s->s_auth.authorizer)ceph_auth_destroy_authorizer (s->s_auth.authorizer); +percpu_counter_destroy (&s->i_caps_hit); +percpu_counter_destroy (&s->i_caps_mis);kfree (s); } }@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) {struct ceph_mds_session *s; +int err;if (mds >= mdsc->mdsmap->possible_max_rank)return ERR_PTR (-EINVAL );@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,dout ("%s: realloc to %d\n" , __func__, newmax); sa =kcalloc (newmax,sizeof (void *),GFP_NOFS ); -if (!sa) +if (!sa) { + err = -ENOMEM ; goto fail_realloc; + }if (mdsc->sessions) {memcpy (sa, mdsc->sessions, mdsc->max_sessions *sizeof (void *));@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,INIT_LIST_HEAD (&s->s_cap_flushing); + err =percpu_counter_init (&s->i_caps_hit,0 ,GFP_NOFS ); +if (err) + goto fail_realloc; + err =percpu_counter_init (&s->i_caps_mis,0 ,GFP_NOFS ); +if (err) + goto fail_init; + mdsc->sessions[mds] = s;atomic_inc (&mdsc->num_sessions);refcount_inc (&s->s_ref);/* one ref to sessions[], one to caller */ @@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,return s; +fail_init: +percpu_counter_destroy (&s->i_caps_hit); fail_realloc:kfree (s);return ERR_PTR (-ENOMEM );fs/ceph/mds_client.h +3
-0@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting;/* waiting requests */ struct list_head s_unsafe;/* unsafe requests */ + +struct percpu_counter i_caps_hit; +struct percpu_counter i_caps_mis; };/* fs/ceph/quota.c +7
-2@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc,return NULL ; }if (qri->inode) { +struct ceph_inode_info *ci =ceph_inode (qri->inode); +int ret; + +ceph_caps_metric (ci,CEPH_STAT_CAP_INODE ); +/* get caps */ -int ret =__ceph_do_getattr (qri->inode,NULL , -CEPH_STAT_CAP_INODE , true); + ret =__ceph_do_getattr (qri->inode,NULL , +CEPH_STAT_CAP_INODE , true);if (ret >=0 ) in = qri->inode;else fs/ceph/super.h +11
-0@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci)return !RB_EMPTY_ROOT (&ci->i_caps); } +extern void __ceph_caps_metric (struct ceph_inode_info *ci,int mask); +static inline void ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, mask); +spin_unlock (&ci->i_ceph_lock); +} +extern int __ceph_caps_issued (struct ceph_inode_info *ci,int *implemented);extern int __ceph_caps_issued_mask (struct ceph_inode_info *ci,int mask,int t); extern int __ceph_caps_issued_other(struct ceph_inode_info *ci,@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int ceph_do_getattr (struct inode *inode,int mask,bool force) { +struct ceph_inode_info *ci =ceph_inode (inode); + +ceph_caps_metric (ci, mask);return __ceph_do_getattr (inode,NULL , mask, force); }extern int ceph_permission (struct inode *inode,int mask);fs/ceph/xattr.c +14
-3@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,struct ceph_vxattr *vxattr =NULL ;int req_mask;ssize_t err; +int ret =-1 ;/* let's see if a virtual xattr was requested */ vxattr =ceph_match_vxattr (inode, name);@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (ci->i_xattrs.version ==0 || !((req_mask &CEPH_CAP_XATTR_SHARED ) || -__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { + (ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock);/* security module gets xattr while filling trace */ @@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)struct ceph_inode_info *ci =ceph_inode (inode);bool len_only = (size ==0 );u32 namelen; -int err; +int err, ret =-1 ;spin_lock (&ci->i_ceph_lock);dout ("listxattr %p ver=%lld index_ver=%lld\n" , inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !(ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 48 lines · Jeff Layton
Quoted 47 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need" mask it means hit, or missed.
item total miss hit
-------------------------------------------------d_lease295 0 993session caps miss hit
-------------------------------------------------0 295 107 4119 1 1 107 9URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 ++ fs/ceph/addr.c | 2 ++ fs/ceph/caps.c | 74 ++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/debugfs.c | 20 ++++++++++++ fs/ceph/dir.c | 9 ++++-- fs/ceph/file.c | 3 ++ fs/ceph/mds_client.c | 16 +++++++++- fs/ceph/mds_client.h | 3 ++ fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)fs/ceph/acl.c +2
-0@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(struct inode *inode,struct ceph_inode_info *ci =ceph_inode (inode);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); +if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl (inode, type, acl);else fs/ceph/addr.c +0
-1--- a/fs/ceph/addr.c
Quoted 8 lines
fs/ceph/addr.c int ceph_uninline_data(struct file *filp, struct page *locked_page) · line 1710
Quoted 7 lines
+++
b/fs/ceph/addr.cfs/ceph/addr.c +2
-0@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM ; goto out; } + +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA );Should a check for inline data really count here?
Currently all the INLINE_DATA is in 'force' mode, so we can ignore it.
Quoted 11 lines
Quoted 11 lines
fs/ceph/addr.c +0
-0err =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , true);if (err <0 ) {fs/ceph/caps.c +0
-0@@ -783,6 +783,75 @@ static int __cap_is_valid(struct ceph_cap *cap)return 1 ; }
Quoted 5 lines
fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 788
Quoted 3 lines
fs/ceph/caps.c +3
-0+/* +* Counts the cap metric. +*/ This needs some comments. Specifically, what should this be counting and how?
Will add it.
The __ceph_caps_metric() will traverse the inode's i_caps try to get the 'issued' excepting the 'invoking' caps until it get enough caps in 'mask'. The i_caps traverse logic is following __ceph_caps_issued_mask().
Quoted 73 lines
Quoted 56 lines
fs/ceph/caps.c +56
-0+void __ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +int have = ci->i_snap_caps; +struct ceph_mds_session *s; +struct ceph_cap *cap; +struct rb_node *p; +bool skip_auth = false; + +lockdep_assert_held (&ci->i_ceph_lock); + +if (mask <=0 ) +return ; + +/* Counts the snap caps metric in the auth cap */ +if (ci->i_auth_cap) { + cap = ci->i_auth_cap; +if (have) { + have |= cap->issued; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap->issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & have) +percpu_counter_inc (&s->i_caps_hit); +else +percpu_counter_inc (&s->i_caps_mis); +ceph_put_mds_session (s); + } + skip_auth = true; + } + } + +if ((mask & have) == mask) +return ; + +/* Checks others */ +for (p =rb_first (&ci->i_caps); p; p =rb_next (p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); +if (!__cap_is_valid (cap)) +continue ; + +if (skip_auth && cap == ci->i_auth_cap) +continue ; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap->issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & cap->issued) +percpu_counter_inc (&s->i_caps_hit); +else fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 854
Quoted 10 lines
fs/ceph/caps.c +10
-0+percpu_counter_inc (&s->i_caps_mis); +ceph_put_mds_session (s); + } + + have |= cap->issued; +if ((mask & have) == mask) +return ; + } +} +I'm trying to understand what happens with the above when more than one
ceph_caphas the same bit set in "issued". For instance:Suppose we're doing the check for a statx call, and we're trying to get caps for pAsFsLs. We have two MDS's and they've each granted us caps for the inode, say:
Quoted 6 lines
MDS 0: pAs
MDS 1: pAsLsFsWe check the cap0 first, and consider it a hit, and then we check cap1 and consider it a hit as well. So that seems like it's being double- counted.
Yeah, it will.
In case2:
MDS 0: pAsFs
MDS 1: pAsLs
For this case and yours both the i_cap0 and i_cap1 are 'hit'.
In case3 :
MDS0: pAsFsLs
MDS1: pAs
Only the i_cap0 is 'hit'. i_cap1 will not be counted any 'hit' or 'mis'.
In case4:
MDS0: p
MDS1: pAsLsFs
i_cap0 will 'mis' and i_cap1 will 'hit'.
All the logic are the same with __ceph_caps_issued_mask() does.
The 'hit' means to get all the caps in 'mask we have checked the i_cap[0~N] and if they have a subset in 'mask', and the 'mis' means we have checked the i_cap[0~N] but they do not.
For the i_cap[N+1 ~ M], if we won't touch them because we have already gotten enough caps needed in 'mask', they won't count any 'hit' or 'mis'
All in all, the current logic is that the 'hit' means 'touched' and it have some of what we needed, and the 'mis' means 'touched' and missed any of what we needed.
Quoted 323 lines
ISTM, that what you really want to do here is logically or all of the
cap->issuedfields together, and then check that vs. the mask value, and count only one hit or miss per inode.That said, it's not 100% clear what you're counting as a hit or miss here, so please let me know if I have that wrong.
Quoted 315 lines
fs/ceph/caps.c +5
-0/* * Return set of valid cap bits issued to us. Note that caps time * out, and may be invalidated in bulk if the client session times out @@ -2746,6 +2815,7 @@ static void check_max_size(struct inode *inode, loff_t endoff)int ceph_try_get_caps (struct inode *inode,int need,int want,bool nonblock,int *got) { +struct ceph_inode_info *ci =ceph_inode (inode);int ret;BUG_ON (need & ~CEPH_CAP_FILE_RD );@@ -2758,6 +2828,7 @@ int ceph_try_get_caps(struct inode *inode, int need, int want,BUG_ON (want & ~(CEPH_CAP_FILE_CACHE |CEPH_CAP_FILE_LAZYIO |CEPH_CAP_FILE_SHARED |CEPH_CAP_FILE_EXCL |CEPH_CAP_ANY_DIR_OPS )); +ceph_caps_metric (ci, need | want); ret =try_get_cap_refs (inode, need, want,0 , nonblock, got);return ret == -EAGAIN ?0 : ret; }@@ -2784,6 +2855,8 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen))return -EBADF ; +ceph_caps_metric (ci, need | want); +while (true) {if (endoff >0 )check_max_size (inode, endoff);@@ -2871,6 +2944,7 @@ int ceph_get_caps(struct file *filp, int need, int want,* getattr request will bring inline data into * page cache */ +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); ret =__ceph_do_getattr (inode,NULL ,CEPH_STAT_CAP_INLINE_DATA , true);fs/ceph/debugfs.c +20
-0@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i;seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p)percpu_counter_sum (&mdsc->metric.d_lease_mis), percpu_counter_sum(&mdsc->metric.d_lease_hit)); +seq_printf (s,"\n" ); +seq_printf (s,"session caps miss hit\n" ); +seq_printf (s,"-------------------------------------------------\n" ); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +seq_printf (s,"%-14d%-16d%-16lld%lld\n" , i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); +ceph_put_mds_session (session); + } +mutex_unlock (&mdsc->mutex); +return 0 ; }fs/ceph/dir.c +7
-2@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_mds_client *mdsc = fsc->mdsc;int i; -int err; +int err, ret =-1 ;unsigned frag =-1 ;struct ceph_mds_reply_info_parsed *rinfo;@@ -346,13 +346,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {int shared_gen =atomic_read (&ci->i_shared_gen); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )return err; } else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); }@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry,struct ceph_dentry_info *di =ceph_dentry (dentry);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_FILE_SHARED ); +dout (" dir %p flags are %d\n" , dir, ci->i_ceph_flags); if (strncmp(dentry->d_name.name, fsc->mount_options->snapdir_name,fs/ceph/file.c +3
-0@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, struct file *file)* asynchronously. */ spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, wanted); +if (__ceph_is_any_real_caps (ci) && (((fmode &CEPH_FILE_MODE_WR ) ==0 ) || ci->i_auth_cap)) {int mds_wanted =__ceph_caps_mds_wanted (ci, true);@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)return -ENOMEM ; } +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); statret =__ceph_do_getattr (inode, page,CEPH_STAT_CAP_INLINE_DATA , !!page);if (statret <0 ) {fs/ceph/mds_client.c +15
-1@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s)if (refcount_dec_and_test (&s->s_ref)) {if (s->s_auth.authorizer)ceph_auth_destroy_authorizer (s->s_auth.authorizer); +percpu_counter_destroy (&s->i_caps_hit); +percpu_counter_destroy (&s->i_caps_mis);kfree (s); } }@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) {struct ceph_mds_session *s; +int err;if (mds >= mdsc->mdsmap->possible_max_rank)return ERR_PTR (-EINVAL );@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,dout ("%s: realloc to %d\n" , __func__, newmax); sa =kcalloc (newmax,sizeof (void *),GFP_NOFS ); -if (!sa) +if (!sa) { + err = -ENOMEM ; goto fail_realloc; + }if (mdsc->sessions) {memcpy (sa, mdsc->sessions, mdsc->max_sessions *sizeof (void *));@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,INIT_LIST_HEAD (&s->s_cap_flushing); + err =percpu_counter_init (&s->i_caps_hit,0 ,GFP_NOFS ); +if (err) + goto fail_realloc; + err =percpu_counter_init (&s->i_caps_mis,0 ,GFP_NOFS ); +if (err) + goto fail_init; + mdsc->sessions[mds] = s;atomic_inc (&mdsc->num_sessions);refcount_inc (&s->s_ref);/* one ref to sessions[], one to caller */ @@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc,return s; +fail_init: +percpu_counter_destroy (&s->i_caps_hit); fail_realloc:kfree (s);return ERR_PTR (-ENOMEM );fs/ceph/mds_client.h +3
-0@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting;/* waiting requests */ struct list_head s_unsafe;/* unsafe requests */ + +struct percpu_counter i_caps_hit; +struct percpu_counter i_caps_mis; };/* fs/ceph/quota.c +7
-2@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc,return NULL ; }if (qri->inode) { +struct ceph_inode_info *ci =ceph_inode (qri->inode); +int ret; + +ceph_caps_metric (ci,CEPH_STAT_CAP_INODE ); +/* get caps */ -int ret =__ceph_do_getattr (qri->inode,NULL , -CEPH_STAT_CAP_INODE , true); + ret =__ceph_do_getattr (qri->inode,NULL , +CEPH_STAT_CAP_INODE , true);if (ret >=0 ) in = qri->inode;else fs/ceph/super.h +11
-0@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci)return !RB_EMPTY_ROOT (&ci->i_caps); } +extern void __ceph_caps_metric (struct ceph_inode_info *ci,int mask); +static inline void ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, mask); +spin_unlock (&ci->i_ceph_lock); +} +extern int __ceph_caps_issued (struct ceph_inode_info *ci,int *implemented);extern int __ceph_caps_issued_mask (struct ceph_inode_info *ci,int mask,int t); extern int __ceph_caps_issued_other(struct ceph_inode_info *ci,@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int ceph_do_getattr (struct inode *inode,int mask,bool force) { +struct ceph_inode_info *ci =ceph_inode (inode); + +ceph_caps_metric (ci, mask);return __ceph_do_getattr (inode,NULL , mask, force); }extern int ceph_permission (struct inode *inode,int mask);fs/ceph/xattr.c +14
-3@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,struct ceph_vxattr *vxattr =NULL ;int req_mask;ssize_t err; +int ret =-1 ;/* let's see if a virtual xattr was requested */ vxattr =ceph_match_vxattr (inode, name);@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (ci->i_xattrs.version ==0 || !((req_mask &CEPH_CAP_XATTR_SHARED ) || -__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { + (ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock);/* security module gets xattr while filling trace */ @@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)struct ceph_inode_info *ci =ceph_inode (inode);bool len_only = (size ==0 );u32 namelen; -int err; +int err, ret =-1 ;spin_lock (&ci->i_ceph_lock);dout ("listxattr %p ver=%lld index_ver=%lld\n" , inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !(ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); } err =__build_xattrs (inode);
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 224 lines · Xiubo Li
Quoted 53 lines · Jeff Layton
Quoted 52 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need"
mask it means hit, or missed.item total miss hit
-------------------------------------------------d_lease295 0 993session caps miss hit
-------------------------------------------------0 295 107 4119 1 1 107 9URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 ++ fs/ceph/addr.c | 2 ++ fs/ceph/caps.c | 74 ++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/debugfs.c | 20 ++++++++++++ fs/ceph/dir.c | 9 ++++-- fs/ceph/file.c | 3 ++ fs/ceph/mds_client.c | 16 +++++++++- fs/ceph/mds_client.h | 3 ++ fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)fs/ceph/acl.c +2
-0@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(structinode *inode, struct ceph_inode_info *ci = ceph_inode(inode);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); +if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl(inode, type, acl); elsefs/ceph/addr.c +0
-0Quoted 8 lines
fs/ceph/addr.c int ceph_uninline_data(struct file *filp, · line 1711
Quoted 7 lines
fs/ceph/addr.c +2
-0@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM ; goto out; } + +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA );Should a check for inline data really count here?
Currently all the
INLINE_DATAis in 'force' mode, so we can ignore it.Quoted 13 lines
Quoted 13 lines
fs/ceph/addr.c +0
-0err =__ceph_do_getattr (inode, page, CEPH_STAT_CAP_INLINE_DATA, true);
if (err < 0) {fs/ceph/caps.c +0
-0@@ -783,6 +783,75 @@ static int __cap_is_valid(struct ceph_cap *cap)return 1 ; }Quoted 6 lines
fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 788
Quoted 3 lines
fs/ceph/caps.c +3
-0+/* +* Counts the cap metric. +*/ This needs some comments. Specifically, what should this be counting and
how?Will add it.
The
__ceph_caps_metric()will traverse the inode'si_capstry to get the
'issued' excepting the 'invoking' caps until it get enough caps in 'mask'. Thei_capstraverse logic is following__ceph_caps_issued_mask().Quoted 80 lines
Quoted 60 lines
fs/ceph/caps.c +27
-0+void __ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +int have = ci->i_snap_caps; +struct ceph_mds_session *s; +struct ceph_cap *cap; +struct rb_node *p; +bool skip_auth = false; + +lockdep_assert_held (&ci->i_ceph_lock); + +if (mask <=0 ) +return ; + +/* Counts the snap caps metric in the auth cap */ +if (ci->i_auth_cap) { + cap = ci->i_auth_cap; +if (have) { + have |= cap->issued; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap- >issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & have) + percpu_counter_inc(&s-Quoted 1 line
i_caps_hit);fs/ceph/caps.c +2
-0+else + percpu_counter_inc(&s-Quoted 1 line
i_caps_mis);fs/ceph/caps.c +27
-0+ceph_put_mds_session (s); + } + skip_auth = true; + } + } + +if ((mask & have) == mask) +return ; + +/* Checks others */ +for (p =rb_first (&ci->i_caps); p; p =rb_next (p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); +if (!__cap_is_valid (cap)) +continue ; + +if (skip_auth && cap == ci->i_auth_cap) +continue ; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap->issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & cap->issued) +percpu_counter_inc (&s->i_caps_hit); +else fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 854
Quoted 10 lines
fs/ceph/caps.c +10
-0+percpu_counter_inc (&s->i_caps_mis); +ceph_put_mds_session (s); + } + + have |= cap->issued; +if ((mask & have) == mask) +return ; + } +} +I'm trying to understand what happens with the above when more than one
ceph_caphas the same bit set in "issued". For instance:Suppose we're doing the check for a statx call, and we're trying to get
caps for pAsFsLs. We have two MDS's and they've each granted us caps for
the inode, say:Quoted 8 lines
MDS 0: pAs
MDS 1: pAsLsFsWe check the cap0 first, and consider it a hit, and then we check cap1
and consider it a hit as well. So that seems like it's being double-
counted.Yeah, it will.
In case2:
MDS 0: pAsFs
MDS 1: pAsLs
For this case and yours both the
i_cap0andi_cap1are 'hit'.In case3 :
MDS0: pAsFsLs
MDS1: pAs
Only the
i_cap0is 'hit'.i_cap1will not be counted any 'hit' or 'mis'.In case4:
MDS0: p
MDS1: pAsLsFs
i_cap0will 'mis' andi_cap1will 'hit'.All the logic are the same with
__ceph_caps_issued_mask()does.The 'hit' means to get all the caps in 'mask we have checked the i_cap[0~N] and if they have a subset in 'mask', and the 'mis' means we
have checked the i_cap[0~N] but they do not.For the i_cap[N+1 ~ M], if we won't touch them because we have already
gotten enough caps needed in 'mask', they won't count any 'hit' or 'mis'
Quoted 7 lines
All in all, the current logic is that the 'hit' means 'touched' and it
have some of what we needed, and the 'mis' means 'touched' and missed
any of what we needed.
That seems sort of arbitrary, given that you're going to get different results depending on the index of the MDS with the caps. For instance:
MDS0: pAsLsFs
MDS1: pAs
...vs...
MDS0: pAs
MDS1: pAsLsFs
If we assume we're looking for pAsLsFs, then the first scenario will just end up with 1 hit and the second will give you 2. AFAIU, the two MDSs are peers, so it really seems like the index should not matter here.
I'm really struggling to understand how these numbers will be useful. What, specifically, are we trying to count here and why?
Quoted 385 lines
Quoted 383 lines
ISTM, that what you really want to do here is logically or all of the
cap->issuedfields together, and then check that vs. the mask value, and
count only one hit or miss per inode.That said, it's not 100% clear what you're counting as a hit or miss
here, so please let me know if I have that wrong.Quoted 372 lines
/*
* Return set of valid cap bits issued to us. Note that caps
time
* out, and may be invalidated in bulk if the client session
times outfs/ceph/caps.c +1
-0@@ -2746,6 +2815,7 @@ static void check_max_size(struct inode *inode, loff_t endoff)int ceph_try_get_caps (struct inode *inode,int need,int want,bool nonblock,int *got) { +struct ceph_inode_info *ci =ceph_inode (inode);int ret;BUG_ON(need & ~CEPH_CAP_FILE_RD);fs/ceph/caps.c +0
-0@@ -2758,6 +2828,7 @@ int ceph_try_get_caps(struct inode *inode, int need, int want, BUG_ON(want & ~(CEPH_CAP_FILE_CACHE |
CEPH_CAP_FILE_LAZYIO|
CEPH_CAP_FILE_SHARED|CEPH_CAP_FILE_EXCL|
CEPH_CAP_ANY_DIR_OPS));fs/ceph/caps.c +1
-0+ceph_caps_metric (ci, need | want); ret = try_get_cap_refs(inode, need, want, 0, nonblock,got);
return ret ==-EAGAIN? 0 : ret;
}fs/ceph/caps.c +2
-0@@ -2784,6 +2855,8 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen))return -EBADF ; +ceph_caps_metric (ci, need | want); +while (true) {if (endoff >0 )check_max_size(inode, endoff);fs/ceph/caps.c +0
-0@@ -2871,6 +2944,7 @@ int ceph_get_caps(struct file *filp, int need, int want, * getattr request will bring inlinedata into
* page cache
*/fs/ceph/caps.c +1
-0+ ceph_caps_metric(ci,
CEPH_STAT_CAP_INLINE_DATA);
ret = __ceph_do_getattr(inode, NULL,
CEPH_STAT_CAP_INLINE_DATA,
true);fs/ceph/debugfs.c +1
-0@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i; seq_printf(s,"item total miss hit\n");
seq_printf(s, "--------------------------------------
-----------\n");fs/ceph/debugfs.c +2
-0@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p) percpu_counter_sum(&mdsc- >metric.d_lease_mis),percpu_counter_sum (&mdsc- >metric.d_lease_hit)); +seq_printf (s,"\n" ); + seq_printf(s,"session caps miss hit\n");
fs/ceph/debugfs.c +17
-1+ seq_printf(s, "---------------------------------------------- ---\n"); + + mutex_lock(&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +seq_printf (s,"%-14d%-16d%-16lld%lld\n" , i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); +ceph_put_mds_session (session); + } +mutex_unlock (&mdsc->mutex); +return 0 ; }fs/ceph/dir.c +1
-1@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_mds_client *mdsc = fsc->mdsc;int i; -int err; +int err, ret =-1 ;unsigned frag =-1 ;struct
ceph_mds_reply_info_parsed*rinfo;fs/ceph/dir.c +4
-1@@ -346,13 +346,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {int shared_gen =atomic_read (&ci- >i_shared_gen); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )return err; } else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock(&ci->i_ceph_lock); }fs/ceph/dir.c +2
-0@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, struct ceph_dentry_info *di =ceph_dentry (dentry);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_FILE_SHARED ); + dout(" dir %p flags are %d\n", dir, ci-Quoted 1 line
i_ceph_flags);if (strncmp(dentry->d_name.name,
fsc->mount_options->snapdir_name,fs/ceph/file.c +2
-0@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, structfile *file) * asynchronously. */ spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, wanted); + if (__ceph_is_any_real_caps(ci) && (((fmode & CEPH_FILE_MODE_WR) == 0) || ci-Quoted 1 line
i_auth_cap)) {int
mds_wanted= __ceph_caps_mds_wanted(ci,
true);fs/ceph/file.c +1
-0@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)return -ENOMEM ; } +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); statret =__ceph_do_getattr (inode, page, CEPH_STAT_CAP_INLIN
E_DATA, !!page);
if (statret < 0) {fs/ceph/mds_client.c +2
-0@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s)if (refcount_dec_and_test (&s->s_ref)) {if (s->s_auth.authorizer)ceph_auth_destroy_authorizer (s- >s_auth.authorizer); +percpu_counter_destroy (&s->i_caps_hit); +percpu_counter_destroy (&s->i_caps_mis);kfree (s);} }fs/ceph/mds_client.c +1
-0@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) {struct ceph_mds_session *s; +int err;if (mds >= mdsc->mdsmap->possible_max_rank)return ERR_PTR(-EINVAL);fs/ceph/mds_client.c +3
-1@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, dout("%s: realloc to %d\n", __func__, newmax); sa =kcalloc (newmax,sizeof (void *),GFP_NOFS ); -if (!sa) +if (!sa) { + err = -ENOMEM ; goto fail_realloc; + }if (mdsc->sessions) { memcpy(sa, mdsc->sessions,
mdsc->max_sessions* sizeof(void
*));fs/ceph/mds_client.c +7
-0@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, INIT_LIST_HEAD(&s->s_cap_flushing); + err =percpu_counter_init (&s->i_caps_hit,0 ,GFP_NOFS ); +if (err) + goto fail_realloc; + err =percpu_counter_init (&s->i_caps_mis,0 ,GFP_NOFS ); +if (err) + goto fail_init; + mdsc->sessions[mds] = s;atomic_inc (&mdsc->num_sessions);
refcount_inc(&s->s_ref); /* one ref to sessions[], one
to caller */fs/ceph/mds_client.c +2
-0@@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, return s; +fail_init: + percpu_counter_destroy(&s->i_caps_hit); fail_realloc:kfree (s);return ERR_PTR(-ENOMEM);fs/ceph/mds_client.h +3
-0@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting;/* waiting requests */ struct list_head s_unsafe;/* unsafe requests */ + +struct percpu_counter i_caps_hit; +struct percpu_counter i_caps_mis; };/* fs/ceph/quota.c +7
-2@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc, return NULL; } if (qri->inode) { + struct ceph_inode_info *ci = ceph_inode(qri->inode); + int ret; + + ceph_caps_metric(ci, CEPH_STAT_CAP_INODE); + /* get caps */ - int ret = __ceph_do_getattr(qri->inode, NULL, - CEPH_STAT_CAP_INODE, true); + ret = __ceph_do_getattr(qri->inode, NULL, + CEPH_STAT_CAP_INODE, true); if (ret >= 0) in = qri->inode;else
fs/ceph/super.h +8
-0@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci) return !RB_EMPTY_ROOT(&ci->i_caps); } +extern void __ceph_caps_metric (struct ceph_inode_info *ci,int mask); +static inline void ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, mask); +spin_unlock (&ci->i_ceph_lock); +} +extern int __ceph_caps_issued(struct
ceph_inode_info*ci, int
*implemented);
extern int __ceph_caps_issued_mask(structceph_inode_info*ci,
int mask, int t);
extern int __ceph_caps_issued_other(structceph_inode_info*ci,fs/ceph/super.h +3
-0@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int ceph_do_getattr (struct inode *inode,int mask,bool force) { +struct ceph_inode_info *ci =ceph_inode (inode); + +ceph_caps_metric (ci, mask);return __ceph_do_getattr (inode,NULL , mask, force);}
extern intceph_permission(struct inode *inode, int mask);fs/ceph/xattr.c +1
-0@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, struct ceph_vxattr *vxattr = NULL;int req_mask;ssize_t err; +int ret =-1 ;/* let's see if a virtual xattr was requested */ vxattr = ceph_match_vxattr(inode, name);fs/ceph/xattr.c +3
-1@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, if (ci->i_xattrs.version == 0 || !((req_mask & CEPH_CAP_XATTR_SHARED) || - __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1))) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1)))) { + if (ret != -1) + __ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED); spin_unlock(&ci->i_ceph_lock);/* security module gets xattr while filling
trace */fs/ceph/xattr.c +3
-0@@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, if (err) return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); }err = __build_xattrs(inode);fs/ceph/xattr.c +7
-2@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)struct ceph_inode_info *ci =ceph_inode (inode);bool len_only = (size ==0 );u32 namelen; -int err; +int err, ret =-1 ;spin_lock (&ci->i_ceph_lock);dout ("listxattr %p ver=%lld index_ver=%lld\n" , inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !(ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );} err = __build_xattrs(inode);
Signature · 1 line
Jeffrey Layton <[email protected]>
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 242 lines · Jeffrey Layton
Quoted 222 lines · Xiubo Li
Quoted 53 lines · Jeff Layton
Quoted 52 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need"
mask it means hit, or missed.item total miss hit
-------------------------------------------------d_lease295 0 993session caps miss hit
-------------------------------------------------0 295 107 4119 1 1 107 9URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/acl.c | 2 ++ fs/ceph/addr.c | 2 ++ fs/ceph/caps.c | 74 ++++++++++++++++++++++++++++++++++++++++++++ fs/ceph/debugfs.c | 20 ++++++++++++ fs/ceph/dir.c | 9 ++++-- fs/ceph/file.c | 3 ++ fs/ceph/mds_client.c | 16 +++++++++- fs/ceph/mds_client.h | 3 ++ fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)fs/ceph/acl.c +2
-0@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(structinode *inode, struct ceph_inode_info *ci = ceph_inode(inode);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); +if (__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,0 ))set_cached_acl(inode, type, acl); elsefs/ceph/addr.c +0
-0Quoted 8 lines
fs/ceph/addr.c int ceph_uninline_data(struct file *filp, · line 1711
Quoted 7 lines
fs/ceph/addr.c +2
-0@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM ; goto out; } + +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA );Should a check for inline data really count here?
Currently all the
INLINE_DATAis in 'force' mode, so we can ignore it.Quoted 13 lines
Quoted 13 lines
fs/ceph/addr.c +0
-0err =__ceph_do_getattr (inode, page, CEPH_STAT_CAP_INLINE_DATA, true); if (err < 0) {fs/ceph/caps.c +0
-0@@ -783,6 +783,75 @@ static int __cap_is_valid(struct ceph_cap *cap)return 1 ; }Quoted 6 lines
fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 788
Quoted 3 lines
fs/ceph/caps.c +3
-0+/* +* Counts the cap metric. +*/ This needs some comments. Specifically, what should this be counting and
how?Will add it.
The
__ceph_caps_metric()will traverse the inode'si_capstry to get the
'issued' excepting the 'invoking' caps until it get enough caps in 'mask'. Thei_capstraverse logic is following__ceph_caps_issued_mask().Quoted 80 lines
Quoted 60 lines
fs/ceph/caps.c +27
-0+void __ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +int have = ci->i_snap_caps; +struct ceph_mds_session *s; +struct ceph_cap *cap; +struct rb_node *p; +bool skip_auth = false; + +lockdep_assert_held (&ci->i_ceph_lock); + +if (mask <=0 ) +return ; + +/* Counts the snap caps metric in the auth cap */ +if (ci->i_auth_cap) { + cap = ci->i_auth_cap; +if (have) { + have |= cap->issued; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap- > issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & have) + percpu_counter_inc(&s-Quoted 1 line
i_caps_hit);fs/ceph/caps.c +2
-0+else + percpu_counter_inc(&s-Quoted 1 line
i_caps_mis);fs/ceph/caps.c +27
-0+ceph_put_mds_session (s); + } + skip_auth = true; + } + } + +if ((mask & have) == mask) +return ; + +/* Checks others */ +for (p =rb_first (&ci->i_caps); p; p =rb_next (p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); +if (!__cap_is_valid (cap)) +continue ; + +if (skip_auth && cap == ci->i_auth_cap) +continue ; + +dout ("%s %p cap %p issued %s, mask %s\n" , __func__, + &ci->vfs_inode, cap,ceph_cap_string (cap->issued), +ceph_cap_string (mask)); + + s =ceph_get_mds_session (cap->session); +if (s) { +if (mask & cap->issued) +percpu_counter_inc (&s->i_caps_hit); +else fs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 854
Quoted 10 lines
fs/ceph/caps.c +10
-0+percpu_counter_inc (&s->i_caps_mis); +ceph_put_mds_session (s); + } + + have |= cap->issued; +if ((mask & have) == mask) +return ; + } +} +I'm trying to understand what happens with the above when more than one
ceph_caphas the same bit set in "issued". For instance:Suppose we're doing the check for a statx call, and we're trying to get
caps for pAsFsLs. We have two MDS's and they've each granted us caps for
the inode, say:Quoted 8 lines
MDS 0: pAs
MDS 1: pAsLsFsWe check the cap0 first, and consider it a hit, and then we check cap1
and consider it a hit as well. So that seems like it's being double-
counted.Yeah, it will.
In case2:
MDS 0: pAsFs
MDS 1: pAsLs
For this case and yours both the
i_cap0andi_cap1are 'hit'.In case3 :
MDS0: pAsFsLs
MDS1: pAs
Only the
i_cap0is 'hit'.i_cap1will not be counted any 'hit' or 'mis'.In case4:
MDS0: p
MDS1: pAsLsFs
i_cap0will 'mis' andi_cap1will 'hit'.All the logic are the same with
__ceph_caps_issued_mask()does.The 'hit' means to get all the caps in 'mask we have checked the i_cap[0~N] and if they have a subset in 'mask', and the 'mis' means we
have checked the i_cap[0~N] but they do not.For the i_cap[N+1 ~ M], if we won't touch them because we have already
gotten enough caps needed in 'mask', they won't count any 'hit' or 'mis'Quoted 7 lines
All in all, the current logic is that the 'hit' means 'touched' and it
have some of what we needed, and the 'mis' means 'touched' and missed
any of what we needed.That seems sort of arbitrary, given that you're going to get different results depending on the index of the MDS with the caps. For instance:
MDS0: pAsLsFs
MDS1: pAs...vs...
MDS0: pAs
MDS1: pAsLsFs
Quoted 7 lines
If we assume we're looking for pAsLsFs, then the first scenario will just end up with 1 hit and the second will give you 2. AFAIU, the two MDSs are peers, so it really seems like the index should not matter here.
I'm really struggling to understand how these numbers will be useful. What, specifically, are we trying to count here and why?
Maybe we need count the hit/mis only once, the fake code like:
// Case1: check the auth caps first
if (auth_cap & mask == mask) {
s->hit++;
return;
}
// Case2: check all the other one by one
for (caps : i_caps) {
if (caps & mask == mask) {
s->hit++;
return;
}
c |= caps;
}
// Case3:
if (c & mask == mask)
s->hit++;
else
s->mis++;
return;
....
And for the session 's->' here, if one i_cap can hold all the 'mask' requested, like the Case1 and Case2 it will be i_cap's corresponding session. Or for Case3 we could choose any session.
But the above is still not very graceful of counting the cap metrics too.
IMO, for the cap hit/miss counter should be a global one just like the dentry_lease does in [PATCH 01/11], will this make sense ?
Thanks,
Quoted 383 lines
Quoted 383 lines
Quoted 382 lines
ISTM, that what you really want to do here is logically or all of the
cap->issuedfields together, and then check that vs. the mask value, and
count only one hit or miss per inode.That said, it's not 100% clear what you're counting as a hit or miss
here, so please let me know if I have that wrong.Quoted 372 lines
/*
* Return set of valid cap bits issued to us. Note that caps
time
* out, and may be invalidated in bulk if the client session
times outfs/ceph/caps.c +1
-0@@ -2746,6 +2815,7 @@ static void check_max_size(struct inode *inode, loff_t endoff)int ceph_try_get_caps (struct inode *inode,int need,int want,bool nonblock,int *got) { +struct ceph_inode_info *ci =ceph_inode (inode);int ret;BUG_ON(need & ~CEPH_CAP_FILE_RD);fs/ceph/caps.c +0
-0@@ -2758,6 +2828,7 @@ int ceph_try_get_caps(struct inode *inode, int need, int want, BUG_ON(want & ~(CEPH_CAP_FILE_CACHE |
CEPH_CAP_FILE_LAZYIO|
CEPH_CAP_FILE_SHARED|CEPH_CAP_FILE_EXCL|
CEPH_CAP_ANY_DIR_OPS));fs/ceph/caps.c +1
-0+ceph_caps_metric (ci, need | want); ret = try_get_cap_refs(inode, need, want, 0, nonblock,got); return ret == -EAGAIN ? 0 : ret; }fs/ceph/caps.c +2
-0@@ -2784,6 +2855,8 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen))return -EBADF ; +ceph_caps_metric (ci, need | want); +while (true) {if (endoff >0 )check_max_size(inode, endoff);fs/ceph/caps.c +0
-0@@ -2871,6 +2944,7 @@ int ceph_get_caps(struct file *filp, int need, int want, * getattr request will bring inlinedata into
* page cache
*/fs/ceph/caps.c +1
-0+ ceph_caps_metric(ci,
CEPH_STAT_CAP_INLINE_DATA);
ret = __ceph_do_getattr(inode, NULL,
CEPH_STAT_CAP_INLINE_DATA,
true);fs/ceph/debugfs.c +1
-0@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +int i; seq_printf(s,"item total miss hit\n"); seq_printf(s, "-------------------------------------------------\n");
fs/ceph/debugfs.c +2
-0@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p) percpu_counter_sum(&mdsc- > metric.d_lease_mis),percpu_counter_sum (&mdsc- > metric.d_lease_hit)); +seq_printf (s,"\n" ); + seq_printf(s,"session caps miss hit\n");
fs/ceph/debugfs.c +17
-1+ seq_printf(s, "---------------------------------------------- ---\n"); + + mutex_lock(&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +seq_printf (s,"%-14d%-16d%-16lld%lld\n" , i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); +ceph_put_mds_session (session); + } +mutex_unlock (&mdsc->mutex); +return 0 ; }fs/ceph/dir.c +1
-1@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct ceph_fs_client *fsc =ceph_inode_to_client (inode);struct ceph_mds_client *mdsc = fsc->mdsc;int i; -int err; +int err, ret =-1 ;unsigned frag =-1 ;struct ceph_mds_reply_info_parsed *rinfo;fs/ceph/dir.c +4
-1@@ -346,13 +346,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && ceph_snap(inode) != CEPH_SNAPDIR && __ceph_dir_is_complete_ordered(ci) && - __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {int shared_gen =atomic_read (&ci- > i_shared_gen); +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =__dcache_readdir (file, ctx, shared_gen);if (err != -EAGAIN )return err; } else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock(&ci->i_ceph_lock); }fs/ceph/dir.c +2
-0@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, struct ceph_dentry_info *di =ceph_dentry (dentry);spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci,CEPH_CAP_FILE_SHARED ); + dout(" dir %p flags are %d\n", dir, ci-Quoted 1 line
i_ceph_flags);if (strncmp(dentry->d_name.name,
fsc->mount_options->snapdir_name,fs/ceph/file.c +2
-0@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, structfile *file) * asynchronously. */ spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, wanted); + if (__ceph_is_any_real_caps(ci) && (((fmode & CEPH_FILE_MODE_WR) == 0) || ci-Quoted 1 line
i_auth_cap)) {int
mds_wanted= __ceph_caps_mds_wanted(ci,
true);fs/ceph/file.c +1
-0@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)return -ENOMEM ; } +ceph_caps_metric (ci,CEPH_STAT_CAP_INLINE_DATA ); statret =__ceph_do_getattr (inode, page, CEPH_STAT_CAP_INLINE_DATA, !!page); if (statret < 0) {fs/ceph/mds_client.c +2
-0@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s)if (refcount_dec_and_test (&s->s_ref)) {if (s->s_auth.authorizer)ceph_auth_destroy_authorizer (s- > s_auth.authorizer); +percpu_counter_destroy (&s->i_caps_hit); +percpu_counter_destroy (&s->i_caps_mis);kfree (s);} }fs/ceph/mds_client.c +1
-0@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) {struct ceph_mds_session *s; +int err;if (mds >= mdsc->mdsmap->possible_max_rank)return ERR_PTR(-EINVAL);fs/ceph/mds_client.c +3
-1@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, dout("%s: realloc to %d\n", __func__, newmax); sa =kcalloc (newmax,sizeof (void *),GFP_NOFS ); -if (!sa) +if (!sa) { + err = -ENOMEM ; goto fail_realloc; + }if (mdsc->sessions) { memcpy(sa, mdsc->sessions,mdsc->max_sessions * sizeof(void*));
fs/ceph/mds_client.c +7
-0@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, INIT_LIST_HEAD(&s->s_cap_flushing); + err =percpu_counter_init (&s->i_caps_hit,0 ,GFP_NOFS ); +if (err) + goto fail_realloc; + err =percpu_counter_init (&s->i_caps_mis,0 ,GFP_NOFS ); +if (err) + goto fail_init; + mdsc->sessions[mds] = s;atomic_inc (&mdsc->num_sessions);refcount_inc(&s->s_ref); /* one ref to sessions[], oneto caller */
fs/ceph/mds_client.c +2
-0@@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, return s; +fail_init: + percpu_counter_destroy(&s->i_caps_hit); fail_realloc:kfree (s);return ERR_PTR(-ENOMEM);fs/ceph/mds_client.h +3
-0@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting;/* waiting requests */ struct list_head s_unsafe;/* unsafe requests */ + +struct percpu_counter i_caps_hit; +struct percpu_counter i_caps_mis; };/* fs/ceph/quota.c +7
-2@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc, return NULL; } if (qri->inode) { + struct ceph_inode_info *ci = ceph_inode(qri->inode); + int ret; + + ceph_caps_metric(ci, CEPH_STAT_CAP_INODE); + /* get caps */ - int ret = __ceph_do_getattr(qri->inode, NULL, - CEPH_STAT_CAP_INODE, true); + ret = __ceph_do_getattr(qri->inode, NULL, + CEPH_STAT_CAP_INODE, true); if (ret >= 0) in = qri->inode;else
fs/ceph/super.h +8
-0@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci) return !RB_EMPTY_ROOT(&ci->i_caps); } +extern void __ceph_caps_metric (struct ceph_inode_info *ci,int mask); +static inline void ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ +spin_lock (&ci->i_ceph_lock); +__ceph_caps_metric (ci, mask); +spin_unlock (&ci->i_ceph_lock); +} +extern int __ceph_caps_issued(struct
ceph_inode_info*ci, int
*implemented);
extern int __ceph_caps_issued_mask(structceph_inode_info*ci,
int mask, int t);
extern int __ceph_caps_issued_other(structceph_inode_info*ci,fs/ceph/super.h +3
-0@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int ceph_do_getattr (struct inode *inode,int mask,bool force) { +struct ceph_inode_info *ci =ceph_inode (inode); + +ceph_caps_metric (ci, mask);return __ceph_do_getattr (inode,NULL , mask, force);} extern int ceph_permission(struct inode *inode, int mask);fs/ceph/xattr.c +1
-0@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, struct ceph_vxattr *vxattr = NULL;int req_mask;ssize_t err; +int ret =-1 ;/* let's see if a virtual xattr was requested */ vxattr = ceph_match_vxattr(inode, name);fs/ceph/xattr.c +3
-1@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, if (ci->i_xattrs.version == 0 || !((req_mask & CEPH_CAP_XATTR_SHARED) || - __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1))) { + (ret = __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1)))) { + if (ret != -1) + __ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED); spin_unlock(&ci->i_ceph_lock);/* security module gets xattr while fillingtrace */
fs/ceph/xattr.c +3
-0@@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, if (err) return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED ); }err = __build_xattrs(inode);fs/ceph/xattr.c +7
-2@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size)struct ceph_inode_info *ci =ceph_inode (inode);bool len_only = (size ==0 );u32 namelen; -int err; +int err, ret =-1 ;spin_lock (&ci->i_ceph_lock);dout ("listxattr %p ver=%lld index_ver=%lld\n" , inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (ci->i_xattrs.version ==0 || - !__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 )) { + !(ret =__ceph_caps_issued_mask (ci,CEPH_CAP_XATTR_SHARED ,1 ))) { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );spin_unlock (&ci->i_ceph_lock); err =ceph_do_getattr (inode,CEPH_STAT_CAP_XATTR , true);if (err)return err;spin_lock (&ci->i_ceph_lock); + }else { +if (ret !=-1 ) +__ceph_caps_metric (ci,CEPH_CAP_XATTR_SHARED );} err = __build_xattrs(inode);
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 289 lines · Xiubo Li
Quoted 240 lines · Jeffrey Layton
Quoted 220 lines · Xiubo Li
Quoted 52 lines · Jeff Layton
Quoted 51 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
This will fulfill the caps hit/miss metric for each session. When checking the "need" mask and if one cap has the subset of the "need"
mask it means hit, or missed.item total miss hit
-------------------------------------------------
d_lease 295 0 993session caps miss hit
-------------------------------------------------
0 295 107 4119 1 1 107 9URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>---
fs/ceph/acl.c | 2 ++
fs/ceph/addr.c | 2 ++
fs/ceph/caps.c | 74
++++++++++++++++++++++++++++++++++++++++++++
fs/ceph/debugfs.c | 20 ++++++++++++
fs/ceph/dir.c | 9 ++++--
fs/ceph/file.c | 3 ++
fs/ceph/mds_client.c| 16 +++++++++-
fs/ceph/mds_client.h| 3 ++fs/ceph/quota.c | 9 ++++-- fs/ceph/super.h | 11 +++++++ fs/ceph/xattr.c | 17 ++++++++-- 11 files changed, 158 insertions(+), 8 deletions(-)fs/ceph/acl.c +2
-0@@ -22,6 +22,8 @@ static inline void ceph_set_cached_acl(struct inode *inode, struct ceph_inode_info *ci = ceph_inode(inode); spin_lock(&ci->i_ceph_lock); + __ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED); + if (__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED,0 )) set_cached_acl(inode, type, acl);else
fs/ceph/addr.c +0
-0Quoted 8 lines
fs/ceph/addr.c int ceph_uninline_data(struct file *filp, · line 1711
Quoted 7 lines
fs/ceph/addr.c +2
-0@@ -1706,6 +1706,8 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err = -ENOMEM; goto out; } + + ceph_caps_metric(ci, CEPH_STAT_CAP_INLINE_DATA);Should a check for inline data really count here?
Currently all the
INLINE_DATAis in 'force' mode, so we can ignore it.Quoted 10 lines
Quoted 10 lines
fs/ceph/addr.c +0
-0err = __ceph_do_getattr(inode, page,
CEPH_STAT_CAP_INLINE_DA
TA, true);
if (err < 0) {fs/ceph/caps.c +0
-0@@ -783,6 +783,75 @@ static int __cap_is_valid(struct ceph_cap *cap)Quoted 8 lines
Quoted 5 lines
fs/ceph/caps.c +0
-0return 1; }+/*
+ * Counts the cap metric.
+ */This needs some comments. Specifically, what should this be counting and
how?Will add it.
The
__ceph_caps_metric()will traverse the inode'si_capstry to get the
'issued' excepting the 'invoking' caps until it get enough caps in 'mask'. Thei_capstraverse logic is following__ceph_caps_issued_mask().Quoted 80 lines
Quoted 60 lines
fs/ceph/caps.c +20
-0+void __ceph_caps_metric (struct ceph_inode_info *ci,int mask) +{ + int have = ci->i_snap_caps; + struct ceph_mds_session *s; + struct ceph_cap *cap; + struct rb_node *p; + bool skip_auth = false; + + lockdep_assert_held(&ci->i_ceph_lock); + + if (mask <= 0) + return; + + /* Counts the snap caps metric in the auth cap */ + if (ci->i_auth_cap) { + cap = ci->i_auth_cap; + if (have) { + have |= cap->issued; + + dout("%s %p cap %p issued %s, mask %s\n",
__func__,
+&ci->vfs_inode, cap, ceph_cap_string(cap-Quoted 1 line
issued),
+
ceph_cap_string(mask));
+
+ s =ceph_get_mds_session(cap->session); + if (s) {
+ if (mask & have)
+ percpu_counter_inc(&s-Quoted 1 line
i_caps_hit);+ else
+ percpu_counter_inc(&s-Quoted 1 line
i_caps_mis);+
ceph_put_mds_session(s);
+ }
+skip_auth= true;
+ }
+ }fs/ceph/caps.c +22
-0+ + if ((mask & have) == mask) + return; + + /* Checks others */ + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (!__cap_is_valid(cap)) + continue; + + if (skip_auth && cap == ci->i_auth_cap) + continue; + + dout("%s %p cap %p issued %s, mask %s\n", __func__, + &ci->vfs_inode, cap, ceph_cap_string(cap->issued), + ceph_cap_string(mask)); + + s = ceph_get_mds_session(cap->session); + if (s) { + if (mask & cap->issued) +percpu_counter_inc (&s->i_caps_hit); + elsefs/ceph/caps.c static int __cap_is_valid(struct ceph_cap *cap) · line 854
Quoted 10 lines
fs/ceph/caps.c +10
-0+percpu_counter_inc (&s->i_caps_mis); + ceph_put_mds_session(s); + } + + have |= cap->issued; + if ((mask & have) == mask) + return; + } +} +I'm trying to understand what happens with the above when more than one
ceph_caphas the same bit set in "issued". For instance:Suppose we're doing the check for a statx call, and we're trying to get
caps for pAsFsLs. We have two MDS's and they've each granted us caps for
the inode, say:Quoted 8 lines
MDS 0: pAs
MDS 1: pAsLsFsWe check the cap0 first, and consider it a hit, and then we check cap1
and consider it a hit as well. So that seems like it's being double-
counted.Yeah, it will.
In case2:
MDS 0: pAsFs
MDS 1: pAsLs
For this case and yours both the
i_cap0andi_cap1are 'hit'.In case3 :
MDS0: pAsFsLs
MDS1: pAs
Only the
i_cap0is 'hit'.i_cap1will not be counted any 'hit' or 'mis'.In case4:
MDS0: p
MDS1: pAsLsFs
i_cap0will 'mis' andi_cap1will 'hit'.All the logic are the same with
__ceph_caps_issued_mask()does.The 'hit' means to get all the caps in 'mask we have checked the i_cap[0~N] and if they have a subset in 'mask', and the 'mis' means we
have checked the i_cap[0~N] but they do not.For the i_cap[N+1 ~ M], if we won't touch them because we have already
gotten enough caps needed in 'mask', they won't count any 'hit' or 'mis'Quoted 7 lines
All in all, the current logic is that the 'hit' means 'touched' and it
have some of what we needed, and the 'mis' means 'touched' and missed
any of what we needed.That seems sort of arbitrary, given that you're going to get different results depending on the index of the MDS with the caps. For instance:
MDS0: pAsLsFs
MDS1: pAs...vs...
MDS0: pAs
MDS1: pAsLsFsQuoted 7 lines
If we assume we're looking for pAsLsFs, then the first scenario will just end up with 1 hit and the second will give you 2. AFAIU, the two MDSs are peers, so it really seems like the index should not matter here.
I'm really struggling to understand how these numbers will be useful. What, specifically, are we trying to count here and why?
Maybe we need count the hit/mis only once, the fake code like:
// Case1: check the auth caps first
if (
auth_cap& mask == mask) {s->hit++;
return;
}
// Case2: check all the other one by one
for (caps :
i_caps) {if (caps & mask == mask) {
s->hit++;
return;
}
c |= caps;
}
// Case3:
if (c & mask == mask)
s->hit++;
else
s->mis++;
return;
Quoted 8 lines
....
And for the session 's->' here, if one
i_capcan hold all the 'mask' requested, like the Case1 and Case2 it will be i_cap's corresponding session. Or for Case3 we could choose any session.But the above is still not very graceful of counting the cap metrics too.
IMO, for the cap hit/miss counter should be a global one just like the
dentry_leasedoes in [PATCH 01/11], will this make sense ?
Currently in fuse client, for each inode it is its auth_cap->session's responsibility to do all the cap hit/mis counting if it has a auth_cap, or choose any one exist.
Maybe this is one acceptable approach.
Quoted 361 lines
Thanks,
Quoted 357 lines
Quoted 357 lines
Quoted 356 lines
ISTM, that what you really want to do here is logically or all of the
cap->issuedfields together, and then check that vs. the mask value, and
count only one hit or miss per inode.That said, it's not 100% clear what you're counting as a hit or miss
here, so please let me know if I have that wrong.Quoted 347 lines
/*
* Return set of valid cap bits issued to us. Note that caps time
* out, and may be invalidated in bulk if the client session times outfs/ceph/caps.c +1
-0@@ -2746,6 +2815,7 @@ static void check_max_size(struct inode *inode, loff_t endoff) int ceph_try_get_caps(struct inode *inode, int need, int want, bool nonblock, int *got) { + struct ceph_inode_info *ci = ceph_inode(inode); int ret; BUG_ON(need & ~CEPH_CAP_FILE_RD);@@ -2758,6 +2828,7 @@ int ceph_try_get_caps(struct inode *inode, int need, int want,BUG_ON(want & ~(CEPH_CAP_FILE_CACHE |
CEPH_CAP_FILE_LAZYIO|
CEPH_CAP_FILE_SHARED|CEPH_CAP_FILE_EXCL|
CEPH_CAP_ANY_DIR_OPS));
+ceph_caps_metric(ci, need | want);
ret =try_get_cap_refs(inode, need, want, 0, nonblock, got);
return ret ==-EAGAIN? 0 : ret;
}fs/ceph/caps.c +1
-0@@ -2784,6 +2855,8 @@ int ceph_get_caps(struct file *filp, int need, int want, fi->filp_gen != READ_ONCE(fsc->filp_gen)) return -EBADF; + ceph_caps_metric(ci, need | want); + while (true) { if (endoff > 0)
check_max_size(inode, endoff);fs/ceph/caps.c +0
-0@@ -2871,6 +2944,7 @@ int ceph_get_caps(struct file *filp, int need, int want, * getattr request will bring inlinedata into
* page cache
*/
+ceph_caps_metric(ci, CEPH_STAT_CAP_INLINE_DATA);
ret = __ceph_do_getattr(inode, NULL,CEPH_STAT_CAP_INLINE_DATA,
true);fs/ceph/debugfs.c +1
-0@@ -128,6 +128,7 @@ static int metric_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_mds_client *mdsc = fsc->mdsc; + int i; seq_printf(s,"item total miss hit\n" );
seq_printf(s, "-------------------------------------- -----------\n");fs/ceph/debugfs.c +0
-0@@ -137,6 +138,25 @@ static int metric_show(struct seq_file *s, void *p)percpu_counter_sum(&mdsc-
Quoted 1 line
metric.d_lease_mis),
percpu_counter_sum(&mdsc-
Quoted 1 line
metric.d_lease_hit));
+
seq_printf(s, "\n");
+ seq_printf(s,
"session caps miss hit\n"); +seq_printf(s, "---------------------------------------------- ---\n");fs/ceph/debugfs.c +16
-0+ + mutex_lock(&mdsc->mutex); + for (i = 0; i < mdsc->max_sessions; i++) { + struct ceph_mds_session *session; + + session = __ceph_lookup_mds_session(mdsc, i); + if (!session) + continue; + seq_printf(s, "%-14d%-16d%-16lld%lld\n", i, + session->s_nr_caps, +percpu_counter_sum (&session->i_caps_mis), +percpu_counter_sum (&session->i_caps_hit)); + ceph_put_mds_session(session); + } + mutex_unlock(&mdsc->mutex); +return 0;
}
diff --gita/fs/ceph/dir.cb/fs/ceph/dir.c
index658c55b323cc..33eb239e09e2100644fs/ceph/dir.c +0
-0@@ -313,7 +313,7 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx)struct
ceph_fs_client*fsc =ceph_inode_to_client(inode);
structceph_mds_client*mdsc =fsc->mdsc;
int i;
- int err;
+ int err, ret = -1;
unsigned frag = -1;
structceph_mds_reply_info_parsed*rinfo;
@@ -346,13 +346,16 @@ static intceph_readdir(struct file *file, struct dir_context *ctx)
!ceph_test_mount_opt(fsc, NOASYNCREADDIR) &&ceph_snap(inode)!=CEPH_SNAPDIR&&__ceph_dir_is_complete_ordered(ci)&&
-__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1)) { + (ret =__ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1))) {
intshared_gen= atomic_read(&ci-Quoted 1 line
i_shared_gen);+
__ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED);spin_unlock(&ci->i_ceph_lock);
err =__dcache_readdir(file, ctx, shared_gen); if (err !=-EAGAIN)
return err;
} else {
+ if (ret != -1)
+__ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED);spin_unlock(&ci->i_ceph_lock);
}
@@ -757,6 +760,8 @@ static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry,
structceph_dentry_info*di =ceph_dentry(dentry);
spin_lock(&ci->i_ceph_lock);
+__ceph_caps_metric(ci, CEPH_CAP_FILE_SHARED); +
dout(" dir %p flags are %d\n", dir, ci-Quoted 1 line
i_ceph_flags);if (strncmp(dentry->d_name.name,
fsc->mount_options->snapdir_name,fs/ceph/file.c +2
-0@@ -384,6 +384,8 @@ int ceph_open(struct inode *inode, structfile *file) * asynchronously. */ spin_lock(&ci->i_ceph_lock); + __ceph_caps_metric(ci, wanted); +if (
__ceph_is_any_real_caps(ci)&& (((fmode &CEPH_FILE_MODE_WR) == 0) || ci-Quoted 1 line
i_auth_cap)) {int
mds_wanted=__ceph_caps_mds_wanted(ci, true);fs/ceph/file.c +0
-0@@ -1340,6 +1342,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to) return -ENOMEM; } + ceph_caps_metric(ci, CEPH_STAT_CAP_INLINE_DATA); statret = __ceph_do_getattr(inode, page,
CEPH_STAT_CAP_INLINE_DATA, !!page);
if (statret < 0) {fs/ceph/mds_client.c +0
-0@@ -558,6 +558,8 @@ void ceph_put_mds_session(struct ceph_mds_session *s) if (refcount_dec_and_test(&s->s_ref)) { if (s->s_auth.authorizer)ceph_auth_destroy_authorizer(s-
Quoted 1 line
s_auth.authorizer);
+
percpu_counter_destroy(&s->i_caps_hit);
+percpu_counter_destroy(&s->i_caps_mis);
kfree(s);
}
}fs/ceph/mds_client.c +4
-1@@ -598,6 +600,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, int mds) { struct ceph_mds_session *s; + int err; if (mds >= mdsc->mdsmap->possible_max_rank) return ERR_PTR(-EINVAL);@@ -612,8 +615,10 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, dout("%s: realloc to %d\n", __func__, newmax); sa = kcalloc(newmax, sizeof(void *), GFP_NOFS); - if (!sa) + if (!sa) { + err = -ENOMEM; goto fail_realloc; + } if (mdsc->sessions) { memcpy(sa, mdsc->sessions,
mdsc->max_sessions*sizeof(void *));fs/ceph/mds_client.c +6
-0@@ -653,6 +658,13 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, INIT_LIST_HEAD(&s->s_cap_flushing); + err = percpu_counter_init(&s->i_caps_hit, 0, GFP_NOFS); + if (err) + goto fail_realloc; + err = percpu_counter_init(&s->i_caps_mis, 0, GFP_NOFS); + if (err) + goto fail_init; + mdsc->sessions[mds] = s; atomic_inc(&mdsc->num_sessions);refcount_inc(&s->s_ref); /* one ref to sessions[], one to caller */
fs/ceph/mds_client.c +0
-0@@ -662,6 +674,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, return s;+fail_init:
+percpu_counter_destroy(&s->i_caps_hit);
fail_realloc:
kfree(s);
returnERR_PTR(-ENOMEM);fs/ceph/mds_client.h +0
-0@@ -201,6 +201,9 @@ struct ceph_mds_session {struct list_head s_waiting; /* waiting requests */ struct list_head s_unsafe; /* unsafe requests */ +
+ structpercpu_counteri_caps_hit;
+ structpercpu_counteri_caps_mis;
};
/*fs/ceph/quota.c +7
-2@@ -147,9 +147,14 @@ static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc, return NULL; } if (qri->inode) { + struct ceph_inode_info *ci = ceph_inode(qri->inode); + int ret; + + ceph_caps_metric(ci, CEPH_STAT_CAP_INODE); + /* get caps */ - int ret = __ceph_do_getattr(qri->inode, NULL, - CEPH_STAT_CAP_INODE, true); + ret = __ceph_do_getattr(qri->inode, NULL, + CEPH_STAT_CAP_INODE, true); if (ret >= 0) in = qri->inode;else
fs/ceph/super.h +0
-0@@ -641,6 +641,14 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci) return !RB_EMPTY_ROOT(&ci->i_caps); }+extern void
__ceph_caps_metric(struct ceph_inode_info *ci, int mask);
+static inline voidceph_caps_metric(struct ceph_inode_info *ci, int mask)fs/ceph/super.h +6
-0+{ + spin_lock(&ci->i_ceph_lock); + __ceph_caps_metric(ci, mask); + spin_unlock(&ci->i_ceph_lock); +} +extern int
__ceph_caps_issued(struct ceph_inode_info *ci, int *implemented);
extern int__ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int t);
extern int __ceph_caps_issued_other(structceph_inode_info*ci,fs/ceph/super.h +0
-0@@ -927,6 +935,9 @@ extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, int mask, bool force);static inline int
ceph_do_getattr(struct inode *inode, int mask, bool force)
{
+ structceph_inode_info*ci =ceph_inode(inode); +
+ceph_caps_metric(ci, mask);
return__ceph_do_getattr(inode, NULL, mask, force); }
extern intceph_permission(struct inode *inode, int mask);fs/ceph/xattr.c +1
-0@@ -829,6 +829,7 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, struct ceph_vxattr *vxattr = NULL; int req_mask; ssize_t err; + int ret = -1; /* let's see if a virtual xattr was requested */ vxattr = ceph_match_vxattr(inode, name);@@ -856,7 +857,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value,if (
ci->i_xattrs.version== 0 ||
!((req_mask &CEPH_CAP_XATTR_SHARED) ||
-__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1))) { + (ret =__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1)))) {
+ if (ret != -1)
+__ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED);spin_unlock(&ci->i_ceph_lock);
/* security module gets xattr while filling trace */fs/ceph/xattr.c +4
-1@@ -871,6 +874,9 @@ ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, if (err) return err; spin_lock(&ci->i_ceph_lock); + } else { + if (ret != -1) + __ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED); } err = __build_xattrs(inode);@@ -907,19 +913,24 @@ ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size) struct ceph_inode_info *ci = ceph_inode(inode); bool len_only = (size == 0); u32 namelen; - int err; + int err, ret = -1; spin_lock(&ci->i_ceph_lock); dout("listxattr %p ver=%lld index_ver=%lld\n", inode, ci->i_xattrs.version, ci->i_xattrs.index_version);if (
ci->i_xattrs.version== 0 ||
- !__ceph_caps_issued_mask(ci,CEPH_CAP_XATTR_SHARED, 1)) { + !(ret =__ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 1))) {
+ if (ret != -1)
+__ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED);spin_unlock(&ci->i_ceph_lock);
err =ceph_do_getattr(inode, CEPH_STAT_CAP_XATTR, true);
if (err)
return err;
spin_lock(&ci->i_ceph_lock);
+ } else {
+ if (ret != -1)
+__ceph_caps_metric(ci, CEPH_CAP_XATTR_SHARED); }
err =__build_xattrs(inode);
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 70 lines · Xiubo Li
Quoted 69 lines · Xiubo Li
Quoted 12 lines · Jeffrey Layton
That seems sort of arbitrary, given that you're going to get different results depending on the index of the MDS with the caps. For instance:
MDS0: pAsLsFs
MDS1: pAs...vs...
MDS0: pAs
MDS1: pAsLsFsQuoted 7 lines
If we assume we're looking for pAsLsFs, then the first scenario will just end up with 1 hit and the second will give you 2. AFAIU, the two MDSs are peers, so it really seems like the index should not matter here.
I'm really struggling to understand how these numbers will be useful. What, specifically, are we trying to count here and why?
Maybe we need count the hit/mis only once, the fake code like:
// Case1: check the auth caps first
if (
auth_cap& mask == mask) {s->hit++; return;}
// Case2: check all the other one by one
for (caps :
i_caps) {if (caps & mask == mask) { s->hit++; return; } c |= caps;}
// Case3:
if (c & mask == mask)
s->hit++;else
s->mis++;return;
....
And for the session 's->' here, if one
i_capcan hold all the 'mask' requested, like the Case1 and Case2 it will be i_cap's corresponding session. Or for Case3 we could choose any session.But the above is still not very graceful of counting the cap metrics too.
Quoted 8 lines
Quoted 3 lines
IMO, for the cap hit/miss counter should be a global one just like the
dentry_leasedoes in [PATCH 01/11], will this make sense ?Currently in fuse client, for each inode it is its auth_cap->session's responsibility to do all the cap hit/mis counting if it has a
auth_cap, or choose any one exist.Maybe this is one acceptable approach.
Again, it's not clear to me what you're trying to measure.
Typically, when you're counting hits and misses on a cache, what you care about is whether you had to wait to fill the cache in order to proceed. That means a lookup in the case of the dcache, but for this it's a cap request. If we have a miss, then we're going to ask a single MDS to resolve it.
To me, it doesn't really make a lot of sense to track this at the session level since the client deals with cap hits and misses as a union of the caps for each session. Keeping per-superblock stats makes a lot more sense in my opinion.
That makes this easy to determine too. You just logically OR all of the "issued" masks together (and maybe the implemented masks in requests that allow that), and check whether that covers the mask you need. If it does, then you have a hit, if not, a miss.
So, to be clear, what we'd be measuring in that case is cap cache checks per superblock. Is that what you're looking to measure with this?
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
Quoted 86 lines · Jeff Layton
Quoted 69 lines · Xiubo Li
Quoted 68 lines · Xiubo Li
Quoted 12 lines · Jeffrey Layton
That seems sort of arbitrary, given that you're going to get different results depending on the index of the MDS with the caps. For instance:
MDS0: pAsLsFs
MDS1: pAs...vs...
MDS0: pAs
MDS1: pAsLsFsQuoted 7 lines
If we assume we're looking for pAsLsFs, then the first scenario will just end up with 1 hit and the second will give you 2. AFAIU, the two MDSs are peers, so it really seems like the index should not matter here.
I'm really struggling to understand how these numbers will be useful. What, specifically, are we trying to count here and why?
Maybe we need count the hit/mis only once, the fake code like:
// Case1: check the auth caps first
if (
auth_cap& mask == mask) {s->hit++; return;}
// Case2: check all the other one by one
for (caps :
i_caps) {if (caps & mask == mask) { s->hit++; return; } c |= caps;}
// Case3:
if (c & mask == mask)
s->hit++;else
s->mis++;return;
....
And for the session 's->' here, if one
i_capcan hold all the 'mask' requested, like the Case1 and Case2 it will be i_cap's corresponding session. Or for Case3 we could choose any session.But the above is still not very graceful of counting the cap metrics too.
Quoted 8 lines
Quoted 3 lines
IMO, for the cap hit/miss counter should be a global one just like the
dentry_leasedoes in [PATCH 01/11], will this make sense ?Currently in fuse client, for each inode it is its auth_cap->session's responsibility to do all the cap hit/mis counting if it has a
auth_cap, or choose any one exist.Maybe this is one acceptable approach.
Again, it's not clear to me what you're trying to measure.
Typically, when you're counting hits and misses on a cache, what you care about is whether you had to wait to fill the cache in order to proceed. That means a lookup in the case of the dcache, but for this it's a cap request. If we have a miss, then we're going to ask a single MDS to resolve it.
Quoted 4 lines
To me, it doesn't really make a lot of sense to track this at the session level since the client deals with cap hits and misses as a union of the caps for each session. Keeping per-superblock stats makes a lot more sense in my opinion.
This approach will be the same with the others, which are also per-superblock or global.
Quoted 4 lines
That makes this easy to determine too. You just logically OR all of the "issued" masks together (and maybe the implemented masks in requests that allow that), and check whether that covers the mask you need. If it does, then you have a hit, if not, a miss.
Yeah, if so it will much easier to measure the hit/miss.
Quoted 3 lines
So, to be clear, what we'd be measuring in that case is cap cache checks per superblock. Is that what you're looking to measure with this?
Basing per-superblock looks good for me, then there will need some change in the ceph side, which is receiving and showing the cap hit/miss per-session.
Thanks.
Re: [PATCH resend v5 02/11] ceph: add caps perf metric for each session
In reply to Jeff LaytonQuoted 4 lines · Jeff Layton
Quoted 3 lines · Xiubo Li
Quoted 2 lines · Xiubo Li
On 2020/1/31 3:00, Jeffrey Layton wrote:
[...]
Quoted 18 lines
Quoted 5 lines
Currently in fuse client, for each inode it is its auth_cap->session's responsibility to do all the cap hit/mis counting if it has a
auth_cap, or choose any one exist.Maybe this is one acceptable approach.
Again, it's not clear to me what you're trying to measure.
Typically, when you're counting hits and misses on a cache, what you care about is whether you had to wait to fill the cache in order to proceed. That means a lookup in the case of the dcache, but for this it's a cap request. If we have a miss, then we're going to ask a single MDS to resolve it.
To me, it doesn't really make a lot of sense to track this at the session level since the client deals with cap hits and misses as a union of the caps for each session. Keeping per-superblock stats makes a lot more sense in my opinion.
Quoted 8 lines
That makes this easy to determine too. You just logically OR all of the "issued" masks together (and maybe the implemented masks in requests that allow that), and check whether that covers the mask you need. If it does, then you have a hit, if not, a miss.
So, to be clear, what we'd be measuring in that case is cap cache checks per superblock. Is that what you're looking to measure with this?
The following is the new approach:
65 +/*
66 + * Counts the cap metric.
67 + *
68 + * This will try to traverse all the ci->i_caps, if we can 69 + * get all the cap 'mask' it will count the hit, or the mis. 70 + */
71 +void __ceph_caps_metric(struct ceph_inode_info *ci, int mask) 72 +{
73 + struct ceph_mds_client *mdsc =
74 + ceph_sb_to_client(ci->vfs_inode.i_sb)->mdsc;
75 + struct ceph_client_metric *metric = &mdsc->metric; 76 + int issued;
77 +
78 + lockdep_assert_held(&ci->i_ceph_lock);
79 +
80 + if (mask <= 0)
81 + return;
82 +
83 + issued = __ceph_caps_issued(ci, NULL);
84 +
85 + if ((mask & issued) == mask)
86 + percpu_counter_inc(&metric->i_caps_hit);
87 + else
88 + percpu_counter_inc(&metric->i_caps_mis);
89 +}
90 +
The cap hit/mis metric are per-superblock, just like the others.
Thanks.
PATCH RESEND v5 3/11 ceph: move ceph_osdc_{read,write}pages to ceph.ko
In reply to [email protected]From: Xiubo Li <[email protected]>
Since this helpers are only used by cpeh.ko, let's move it to ceph.ko and rename to _sync_.
--- fs/ceph/addr.c | 86 ++++++++++++++++++++++++++++++++- include/linux/ceph/osd_client.h | 17 ------- net/ceph/osd_client.c | 79 ------------------------------ 3 files changed, 84 insertions(+), 98 deletions(-)
fs/ceph/addr.c +84 -2
@@ -182,6 +182,47 @@ static int ceph_releasepage(struct page *page, gfp_t g)return !PagePrivate (page); } +/* +* Read some contiguous pages. If we cross a stripe boundary, shorten +* *plen. Return number of bytes read, or error. +*/ +static int ceph_sync_readpages (struct ceph_fs_client *fsc, +struct ceph_vino vino, +struct ceph_file_layout *layout, +u64 off,u64 *plen, +u32 truncate_seq,u64 truncate_size, +struct page **pages,int num_pages, +int page_align) +{ +struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_osd_request *req; +int rc =0 ; + +dout ("readpages on ino %llx.%llx on %llu~%llu\n" , vino.ino, + vino.snap, off, *plen); + req =ceph_osdc_new_request (osdc, layout, vino, off, plen,0 ,1 , +CEPH_OSD_OP_READ ,CEPH_OSD_FLAG_READ , +NULL , truncate_seq, truncate_size, + false); +if (IS_ERR (req)) +return PTR_ERR (req); + +/* it may be a short read due to an object boundary */ +osd_req_op_extent_osd_data_pages (req,0 , + pages, *plen, page_align, false, false); + +dout ("readpages final extent is %llu~%llu (%llu bytes align %d)\n" , + off, *plen, *plen, page_align); + + rc =ceph_osdc_start_request (osdc, req, false); +if (!rc) + rc =ceph_osdc_wait_request (osdc, req); + +ceph_osdc_put_request (req); +dout ("readpages result %d\n" , rc); +return rc; +} +/* * read a single page, without unlocking it. */
@@ -218,7 +259,7 @@ static int ceph_do_readpage(struct file *filp, struct page *page)dout ("readpage inode %p file %p page %p index %lu\n" , inode, filp, page, page->index); - err =ceph_osdc_readpages (&fsc->client->osdc,ceph_vino (inode), + err =ceph_sync_readpages (fsc,ceph_vino (inode), &ci->i_layout, off, &len, ci->i_truncate_seq, ci->i_truncate_size, &page,1 ,0 );
@@ -570,6 +611,47 @@ static u64 get_writepages_data_length(struct inode *inode,return end > start ? end - start :0 ; } +/* +* do a synchronous write on N pages +*/ +static int ceph_sync_writepages (struct ceph_fs_client *fsc, +struct ceph_vino vino, +struct ceph_file_layout *layout, +struct ceph_snap_context *snapc, +u64 off,u64 len, +u32 truncate_seq,u64 truncate_size, +struct timespec64 *mtime, +struct page **pages,int num_pages) +{ +struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_osd_request *req; +int rc =0 ; +int page_align = off & ~PAGE_MASK ; + + req =ceph_osdc_new_request (osdc, layout, vino, off, &len,0 ,1 , +CEPH_OSD_OP_WRITE ,CEPH_OSD_FLAG_WRITE , + snapc, truncate_seq, truncate_size, + true); +if (IS_ERR (req)) +return PTR_ERR (req); + +/* it may be a short write due to an object boundary */ +osd_req_op_extent_osd_data_pages (req,0 , pages, len, page_align, + false, false); +dout ("writepages %llu~%llu (%llu bytes)\n" , off, len, len); + + req->r_mtime = *mtime; + rc =ceph_osdc_start_request (osdc, req, true); +if (!rc) + rc =ceph_osdc_wait_request (osdc, req); + +ceph_osdc_put_request (req); +if (rc ==0 ) + rc = len; +dout ("writepages result %d\n" , rc); +return rc; +} +/* * Write a single page, but leave the page locked. *
@@ -628,7 +710,7 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc)set_bdi_congested (inode_to_bdi (inode),BLK_RW_ASYNC );set_page_writeback (page); - err = ceph_osdc_writepages(&fsc->client->osdc, ceph_vino(inode), + err = ceph_sync_writepages(fsc, ceph_vino(inode), &ci->i_layout, snapc, page_off, len, ceph_wbc.truncate_seq, ceph_wbc.truncate_size,
include/linux/ceph/osd_client.h +0 -17
@@ -509,23 +509,6 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, struct page *req_page, size_t req_len, struct page **resp_pages, size_t *resp_len); -extern int ceph_osdc_readpages (struct ceph_osd_client *osdc, -struct ceph_vino vino, -struct ceph_file_layout *layout, -u64 off,u64 *plen, -u32 truncate_seq,u64 truncate_size, -struct page **pages,int nr_pages, -int page_align); - -extern int ceph_osdc_writepages (struct ceph_osd_client *osdc, -struct ceph_vino vino, -struct ceph_file_layout *layout, -struct ceph_snap_context *sc, -u64 off,u64 len, -u32 truncate_seq,u64 truncate_size, -struct timespec64 *mtime, -struct page **pages,int nr_pages); - int ceph_osdc_copy_from(struct ceph_osd_client *osdc, u64 src_snapid, u64 src_version, struct ceph_object_id *src_oid,
net/ceph/osd_client.c +0 -79
85 lines not on this page · raw message
Signature · 1 line
2.21.0
Re: [PATCH resend v5 03/11] ceph: move ceph_osdc_{read,write}pages to ceph.ko
Quoted 238 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
Since this helpers are only used by cpeh.ko, let's move it to ceph.ko and rename to _sync_.
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/addr.c | 86 ++++++++++++++++++++++++++++++++- include/linux/ceph/osd_client.h | 17 ------- net/ceph/osd_client.c | 79 ------------------------------ 3 files changed, 84 insertions(+), 98 deletions(-)fs/ceph/addr.c +84
-2@@ -182,6 +182,47 @@ static int ceph_releasepage(struct page *page, gfp_t g)return !PagePrivate (page); } +/* +* Read some contiguous pages. If we cross a stripe boundary, shorten +* *plen. Return number of bytes read, or error. +*/ +static int ceph_sync_readpages (struct ceph_fs_client *fsc, +struct ceph_vino vino, +struct ceph_file_layout *layout, +u64 off,u64 *plen, +u32 truncate_seq,u64 truncate_size, +struct page **pages,int num_pages, +int page_align) +{ +struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_osd_request *req; +int rc =0 ; + +dout ("readpages on ino %llx.%llx on %llu~%llu\n" , vino.ino, + vino.snap, off, *plen); + req =ceph_osdc_new_request (osdc, layout, vino, off, plen,0 ,1 , +CEPH_OSD_OP_READ ,CEPH_OSD_FLAG_READ , +NULL , truncate_seq, truncate_size, + false); +if (IS_ERR (req)) +return PTR_ERR (req); + +/* it may be a short read due to an object boundary */ +osd_req_op_extent_osd_data_pages (req,0 , + pages, *plen, page_align, false, false); + +dout ("readpages final extent is %llu~%llu (%llu bytes align %d)\n" , + off, *plen, *plen, page_align); + + rc =ceph_osdc_start_request (osdc, req, false); +if (!rc) + rc =ceph_osdc_wait_request (osdc, req); + +ceph_osdc_put_request (req); +dout ("readpages result %d\n" , rc); +return rc; +} +/* * read a single page, without unlocking it. */ @@ -218,7 +259,7 @@ static int ceph_do_readpage(struct file *filp, struct page *page)dout ("readpage inode %p file %p page %p index %lu\n" , inode, filp, page, page->index); - err =ceph_osdc_readpages (&fsc->client->osdc,ceph_vino (inode), + err =ceph_sync_readpages (fsc,ceph_vino (inode), &ci->i_layout, off, &len, ci->i_truncate_seq, ci->i_truncate_size, &page,1 ,0 );@@ -570,6 +611,47 @@ static u64 get_writepages_data_length(struct inode *inode,return end > start ? end - start :0 ; } +/* +* do a synchronous write on N pages +*/ +static int ceph_sync_writepages (struct ceph_fs_client *fsc, +struct ceph_vino vino, +struct ceph_file_layout *layout, +struct ceph_snap_context *snapc, +u64 off,u64 len, +u32 truncate_seq,u64 truncate_size, +struct timespec64 *mtime, +struct page **pages,int num_pages) +{ +struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_osd_request *req; +int rc =0 ; +int page_align = off & ~PAGE_MASK ; + + req =ceph_osdc_new_request (osdc, layout, vino, off, &len,0 ,1 , +CEPH_OSD_OP_WRITE ,CEPH_OSD_FLAG_WRITE , + snapc, truncate_seq, truncate_size, + true); +if (IS_ERR (req)) +return PTR_ERR (req); + +/* it may be a short write due to an object boundary */ +osd_req_op_extent_osd_data_pages (req,0 , pages, len, page_align, + false, false); +dout ("writepages %llu~%llu (%llu bytes)\n" , off, len, len); + + req->r_mtime = *mtime; + rc =ceph_osdc_start_request (osdc, req, true); +if (!rc) + rc =ceph_osdc_wait_request (osdc, req); + +ceph_osdc_put_request (req); +if (rc ==0 ) + rc = len; +dout ("writepages result %d\n" , rc); +return rc; +} +/* * Write a single page, but leave the page locked. * @@ -628,7 +710,7 @@ static int writepage_nounlock(struct page *page, struct writeback_control *wbc)set_bdi_congested (inode_to_bdi (inode),BLK_RW_ASYNC );set_page_writeback (page); - err = ceph_osdc_writepages(&fsc->client->osdc, ceph_vino(inode), + err = ceph_sync_writepages(fsc, ceph_vino(inode), &ci->i_layout, snapc, page_off, len, ceph_wbc.truncate_seq, ceph_wbc.truncate_size,include/linux/ceph/osd_client.h +0
-17@@ -509,23 +509,6 @@ int ceph_osdc_call(struct ceph_osd_client *osdc, struct page *req_page, size_t req_len, struct page **resp_pages, size_t *resp_len); -extern int ceph_osdc_readpages (struct ceph_osd_client *osdc, -struct ceph_vino vino, -struct ceph_file_layout *layout, -u64 off,u64 *plen, -u32 truncate_seq,u64 truncate_size, -struct page **pages,int nr_pages, -int page_align); - -extern int ceph_osdc_writepages (struct ceph_osd_client *osdc, -struct ceph_vino vino, -struct ceph_file_layout *layout, -struct ceph_snap_context *sc, -u64 off,u64 len, -u32 truncate_seq,u64 truncate_size, -struct timespec64 *mtime, -struct page **pages,int nr_pages); - int ceph_osdc_copy_from(struct ceph_osd_client *osdc, u64 src_snapid, u64 src_version, struct ceph_object_id *src_oid,net/ceph/osd_client.c +0
-72@@ -5230,85 +5230,6 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc)ceph_msgpool_destroy (&osdc->msgpool_op_reply); } -/* -* Read some contiguous pages. If we cross a stripe boundary, shorten -* *plen. Return number of bytes read, or error. -*/ -int ceph_osdc_readpages (struct ceph_osd_client *osdc, -struct ceph_vino vino,struct ceph_file_layout *layout, -u64 off,u64 *plen, -u32 truncate_seq,u64 truncate_size, -struct page **pages,int num_pages,int page_align) -{ -struct ceph_osd_request *req; -int rc =0 ; - -dout ("readpages on ino %llx.%llx on %llu~%llu\n" , vino.ino, - vino.snap, off, *plen); - req =ceph_osdc_new_request (osdc, layout, vino, off, plen,0 ,1 , -CEPH_OSD_OP_READ ,CEPH_OSD_FLAG_READ , -NULL , truncate_seq, truncate_size, - false); -if (IS_ERR (req)) -return PTR_ERR (req); - -/* it may be a short read due to an object boundary */ -osd_req_op_extent_osd_data_pages (req,0 , - pages, *plen, page_align, false, false); - -dout ("readpages final extent is %llu~%llu (%llu bytes align %d)\n" , - off, *plen, *plen, page_align); - - rc =ceph_osdc_start_request (osdc, req, false); -if (!rc) - rc =ceph_osdc_wait_request (osdc, req); - -ceph_osdc_put_request (req); -dout ("readpages result %d\n" , rc); -return rc; -} -EXPORT_SYMBOL (ceph_osdc_readpages); - -/* -* do a synchronous write on N pages -*/ -int ceph_osdc_writepages (struct ceph_osd_client *osdc,struct ceph_vino vino, -struct ceph_file_layout *layout, -struct ceph_snap_context *snapc, -u64 off,u64 len, -u32 truncate_seq,u64 truncate_size, -struct timespec64 *mtime, -struct page **pages,int num_pages) -{ -struct ceph_osd_request *req; -int rc =0 ; -int page_align = off & ~PAGE_MASK ; - - req =ceph_osdc_new_request (osdc, layout, vino, off, &len,0 ,1 , -CEPH_OSD_OP_WRITE ,CEPH_OSD_FLAG_WRITE , - snapc, truncate_seq, truncate_size, - true); -if (IS_ERR (req)) -return PTR_ERR (req); - -/* it may be a short write due to an object boundary */ -osd_req_op_extent_osd_data_pages (req,0 , pages, len, page_align, - false, false); -dout ("writepages %llu~%llu (%llu bytes)\n" , off, len, len); - - req->r_mtime = *mtime; - rc =ceph_osdc_start_request (osdc, req, true); -if (!rc) - rc =ceph_osdc_wait_request (osdc, req); - -ceph_osdc_put_request (req);
net/ceph/osd_client.c void ceph_osdc_stop(struct ceph_osd_client *osdc) · line 5235
Quoted 10 lines
net/ceph/osd_client.c +0
-7-if (rc ==0 ) - rc = len; -dout ("writepages result %d\n" , rc); -return rc; -} -EXPORT_SYMBOL (ceph_osdc_writepages); - static int osd_req_op_copy_from_init(struct ceph_osd_request *req, u64 src_snapid, u64 src_version, struct ceph_object_id *src_oid,
This looks like a nice cleanup. I'll plan to merge this one after a bit of testing.
Signature · 1 line
Jeff Layton <[email protected]>
PATCH RESEND v5 4/11 ceph: add r_end_stamp for the osdc request
In reply to [email protected]From: Xiubo Li <[email protected]>
Grab the osdc requests' end time stamp.
--- include/linux/ceph/osd_client.h | 1 + net/ceph/osd_client.c | 2 ++ 2 files changed, 3 insertions(+)
include/linux/ceph/osd_client.h +1 -0
@@ -213,6 +213,7 @@ struct ceph_osd_request {/* internal */ unsigned long r_stamp;/* jiffies, send or check time */ unsigned long r_start_stamp;/* jiffies */ +unsigned long r_end_stamp;/* jiffies */ int r_attempts;u32 r_map_dne_bound;
net/ceph/osd_client.c +2 -0
@@ -2389,6 +2389,8 @@ static void finish_request(struct ceph_osd_request *req)WARN_ON (lookup_request_mc (&osdc->map_checks, req->r_tid));dout ("%s req %p tid %llu\n" , __func__, req, req->r_tid); + req->r_end_stamp = jiffies; +if (req->r_osd)unlink_request (req->r_osd, req);atomic_dec (&osdc->num_requests);
Signature · 1 line
2.21.0
Re: [PATCH resend v5 04/11] ceph: add r_end_stamp for the osdc request
Quoted 27 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
Grab the osdc requests' end time stamp.
Signed-off-by Xiubo Li <[email protected]>--- include/linux/ceph/osd_client.h | 1 + net/ceph/osd_client.c | 2 ++ 2 files changed, 3 insertions(+)include/linux/ceph/osd_client.h +1
-0@@ -213,6 +213,7 @@ struct ceph_osd_request {/* internal */ unsigned long r_stamp;/* jiffies, send or check time */ unsigned long r_start_stamp;/* jiffies */ +unsigned long r_end_stamp;/* jiffies */ int r_attempts;u32 r_map_dne_bound;net/ceph/osd_client.c +0
-0
net/ceph/osd_client.c static void finish_request(struct ceph_osd_request *req) · line 2396
Quoted 8 lines
net/ceph/osd_client.c +2
-0@@ -2389,6 +2389,8 @@ static void finish_request(struct ceph_osd_request *req)WARN_ON (lookup_request_mc (&osdc->map_checks, req->r_tid));dout ("%s req %p tid %llu\n" , __func__, req, req->r_tid); + req->r_end_stamp = jiffies; +if (req->r_osd)unlink_request (req->r_osd, req);atomic_dec (&osdc->num_requests);
Maybe fold this patch into #6 in this series? I'd prefer to add the new field along with its first user.
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 04/11] ceph: add r_end_stamp for the osdc request
Quoted 31 lines · Jeff Layton
Quoted 30 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
Grab the osdc requests' end time stamp.
Signed-off-by Xiubo Li <[email protected]>--- include/linux/ceph/osd_client.h | 1 + net/ceph/osd_client.c | 2 ++ 2 files changed, 3 insertions(+)include/linux/ceph/osd_client.h +1
-0@@ -213,6 +213,7 @@ struct ceph_osd_request {/* internal */ unsigned long r_stamp;/* jiffies, send or check time */ unsigned long r_start_stamp;/* jiffies */ +unsigned long r_end_stamp;/* jiffies */ int r_attempts;u32 r_map_dne_bound;net/ceph/osd_client.c +0
-0@@ -2389,6 +2389,8 @@ static void finish_request(struct ceph_osd_request *req)WARN_ON (lookup_request_mc (&osdc->map_checks, req->r_tid));dout ("%s req %p tid %llu\n" , __func__, req, req->r_tid);
Quoted 7 lines
net/ceph/osd_client.c static void finish_request(struct ceph_osd_request *req) · line 2396
Quoted 5 lines
net/ceph/osd_client.c +2
-0+ req->r_end_stamp = jiffies; +if (req->r_osd)unlink_request (req->r_osd, req);atomic_dec (&osdc->num_requests);Maybe fold this patch into #6 in this series? I'd prefer to add the new field along with its first user.
Sure, will merge it.
Thanks.
PATCH RESEND v5 5/11 ceph: add global read latency metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
read 73 3590000 49178082
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/addr.c | 8 ++++++++ fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/file.c | 15 +++++++++++++++ fs/ceph/mds_client.c | 29 +++++++++++++++++++++++------ fs/ceph/mds_client.h | 9 ++------- fs/ceph/metric.h | 30 ++++++++++++++++++++++++++++++ 6 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 fs/ceph/metric.h
fs/ceph/addr.c +8 -0
@@ -195,6 +195,7 @@ static int ceph_sync_readpages(struct ceph_fs_client *fsc, int page_align) {struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_request *req;int rc =0 ;
@@ -218,6 +219,8 @@ static int ceph_sync_readpages(struct ceph_fs_client *fsc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +ceph_update_read_latency (metric, req, rc); +ceph_osdc_put_request (req);dout ("readpages result %d\n" , rc);return rc;
@@ -301,6 +304,8 @@ static int ceph_readpage(struct file *filp, struct page *page)static void finish_read (struct ceph_osd_request *req) {struct inode *inode = req->r_inode; +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_data *osd_data;int rc = req->r_result <=0 ? req->r_result :0 ;int bytes = req->r_result >=0 ? req->r_result :0 ;
@@ -338,6 +343,9 @@ static void finish_read(struct ceph_osd_request *req)put_page (page); bytes -=PAGE_SIZE ; } + +ceph_update_read_latency (metric, req, rc); +kfree (osd_data->pages); }
fs/ceph/debugfs.c +11 -0
@@ -128,8 +128,19 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +s64 total, sum, avg =0 ;int i; +seq_printf (s,"item total sum_lat(us) avg_lat(us)\n" ); +seq_printf (s,"-----------------------------------------------------\n" ); + + total =percpu_counter_sum (&mdsc->metric.total_reads); + sum =percpu_counter_sum (&mdsc->metric.read_latency_sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/file.c +15 -0
@@ -588,6 +588,7 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_client *osdc = &fsc->client->osdc;ssize_t ret;u64 off = iocb->ki_pos;
@@ -660,6 +661,9 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, ret =ceph_osdc_start_request (osdc, req, false);if (!ret) ret =ceph_osdc_wait_request (osdc, req); + +ceph_update_read_latency (metric, req, ret); +ceph_osdc_put_request (req); i_size =i_size_read (inode);
@@ -798,13 +802,20 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs); +BUG_ON (!aio_req);dout ("ceph_aio_complete_req %p rc %d bytes %u\n" , inode, rc, osd_data->bvec_pos.iter.bi_size); +/* r_start_stamp == 0 means the request was not submitted */ +if (req->r_start_stamp && !aio_req->write) +ceph_update_read_latency (metric, req, rc); +if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;BUG_ON (!aio_req->write);
@@ -933,6 +944,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct bio_vec *bvecs;
@@ -1049,6 +1061,9 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if (!write) +ceph_update_read_latency (metric, req, ret); + size =i_size_read (inode);if (!write) {if (ret == -ENOENT )
fs/ceph/mds_client.c +23 -6
@@ -4182,14 +4182,29 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)atomic64_set (&metric->total_dentries,0 ); ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret) -return ret; +return ret;; ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL ); -if (ret) { -percpu_counter_destroy (&metric->d_lease_hit); -return ret; - } +if (ret) + goto err_dlease_mis; -return 0 ; + ret =percpu_counter_init (&metric->total_reads,0 ,GFP_KERNEL ); +if (ret) + goto err_total_reads; + + ret =percpu_counter_init (&metric->read_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_read_latency_sum; + +return ret; + +err_read_latency_sum: +percpu_counter_destroy (&metric->total_reads); +err_total_reads: +percpu_counter_destroy (&metric->d_lease_mis); +err_dlease_mis: +percpu_counter_destroy (&metric->d_lease_hit); + +return ret; }int ceph_mdsc_init (struct ceph_fs_client *fsc)
@@ -4529,6 +4544,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.read_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_reads);percpu_counter_destroy (&mdsc->metric.d_lease_mis);percpu_counter_destroy (&mdsc->metric.d_lease_hit);
fs/ceph/mds_client.h +2 -7
@@ -16,6 +16,8 @@#include <linux/ceph/mdsmap.h> #include <linux/ceph/auth.h> +#include "metric.h" +/* The first 8 bits are reserved for old ceph releases */ enum ceph_feature_type {CEPHFS_FEATURE_MIMIC =8 ,
@@ -361,13 +363,6 @@ struct cap_wait {int want; }; -/* This is the global metrics */ -struct ceph_client_metric { -atomic64_t total_dentries; -struct percpu_counter d_lease_hit; -struct percpu_counter d_lease_mis; -}; -/* * mds client state */
fs/ceph/metric.h new file +30 -0
@@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_MDS_METRIC_H +#define _FS_CEPH_MDS_METRIC_H + +#include <linux/ceph/osd_client.h> + +/* This is the global metrics */ +struct ceph_client_metric { +atomic64_t total_dentries; +struct percpu_counter d_lease_hit; +struct percpu_counter d_lease_mis; + +struct percpu_counter total_reads; +struct percpu_counter read_latency_sum; +}; + +static inline void ceph_update_read_latency (struct ceph_client_metric *m, +struct ceph_osd_request *req, +int rc) +{ +if (!m || !req) +return ; + +if (rc >=0 || rc == -ENOENT || rc == -ETIMEDOUT ) { +s64 latency = req->r_end_stamp - req->r_start_stamp; +percpu_counter_inc (&m->total_reads); +percpu_counter_add (&m->read_latency_sum, latency); + } +} +#endif
Signature · 1 line
2.21.0
Re: [PATCH resend v5 05/11] ceph: add global read latency metric support
Quoted 23 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
item total
sum_lat(us)avg_lat(us)
-----------------------------------------------------
read 73 3590000 49178082URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/addr.c | 8 ++++++++ fs/ceph/debugfs.c | 11 +++++++++++ fs/ceph/file.c | 15 +++++++++++++++ fs/ceph/mds_client.c | 29 +++++++++++++++++++++++------ fs/ceph/mds_client.h | 9 ++------- fs/ceph/metric.h | 30 ++++++++++++++++++++++++++++++ 6 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 fs/ceph/metric.hfs/ceph/addr.c +0
-0
fs/ceph/addr.c static int ceph_sync_readpages(struct ceph_fs_client *fsc, · line 198
Quoted 4 lines
fs/ceph/addr.c +1
-0@@ -195,6 +195,7 @@ static int ceph_sync_readpages(struct ceph_fs_client *fsc, int page_align) {struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_client_metric *metric = &fsc->mdsc->metric;
nit: I think you can drop this variable and just dereference the metric field directly below where it's used. Ditto in other places where "metric" is only used once in the function.
Quoted 121 lines
fs/ceph/addr.c +7
-0struct ceph_osd_request *req;int rc =0 ;@@ -218,6 +219,8 @@ static int ceph_sync_readpages(struct ceph_fs_client *fsc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +ceph_update_read_latency (metric, req, rc); +ceph_osdc_put_request (req);dout ("readpages result %d\n" , rc);return rc;@@ -301,6 +304,8 @@ static int ceph_readpage(struct file *filp, struct page *page)static void finish_read (struct ceph_osd_request *req) {struct inode *inode = req->r_inode; +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_data *osd_data;int rc = req->r_result <=0 ? req->r_result :0 ;int bytes = req->r_result >=0 ? req->r_result :0 ;@@ -338,6 +343,9 @@ static void finish_read(struct ceph_osd_request *req)put_page (page); bytes -=PAGE_SIZE ; } + +ceph_update_read_latency (metric, req, rc); +kfree (osd_data->pages); }fs/ceph/debugfs.c +11
-0@@ -128,8 +128,19 @@ static int metric_show(struct seq_file *s, void *p) {struct ceph_fs_client *fsc = s->private;struct ceph_mds_client *mdsc = fsc->mdsc; +s64 total, sum, avg =0 ;int i; +seq_printf (s,"item total sum_lat(us) avg_lat(us)\n" ); +seq_printf (s,"-----------------------------------------------------\n" ); + + total =percpu_counter_sum (&mdsc->metric.total_reads); + sum =percpu_counter_sum (&mdsc->metric.read_latency_sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );fs/ceph/file.c +15
-0@@ -588,6 +588,7 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_client *osdc = &fsc->client->osdc;ssize_t ret;u64 off = iocb->ki_pos;@@ -660,6 +661,9 @@ static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, ret =ceph_osdc_start_request (osdc, req, false);if (!ret) ret =ceph_osdc_wait_request (osdc, req); + +ceph_update_read_latency (metric, req, ret); +ceph_osdc_put_request (req); i_size =i_size_read (inode);@@ -798,13 +802,20 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req)struct inode *inode = req->r_inode;struct ceph_aio_request *aio_req = req->r_priv;struct ceph_osd_data *osd_data =osd_req_op_extent_osd_data (req,0 ); +struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;BUG_ON (osd_data->type !=CEPH_OSD_DATA_TYPE_BVECS );BUG_ON (!osd_data->num_bvecs); +BUG_ON (!aio_req);dout ("ceph_aio_complete_req %p rc %d bytes %u\n" , inode, rc, osd_data->bvec_pos.iter.bi_size); +/* r_start_stamp == 0 means the request was not submitted */ +if (req->r_start_stamp && !aio_req->write) +ceph_update_read_latency (metric, req, rc); +if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;BUG_ON (!aio_req->write);@@ -933,6 +944,7 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct bio_vec *bvecs;@@ -1049,6 +1061,9 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +if (!write) +ceph_update_read_latency (metric, req, ret); + size =i_size_read (inode);if (!write) {if (ret == -ENOENT )fs/ceph/mds_client.c +0
-0
fs/ceph/mds_client.c static int ceph_mdsc_metric_init(struct ceph_client_metric *metric) · line 4185
Quoted 5 lines
fs/ceph/mds_client.c +1
-1@@ -4182,14 +4182,29 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)atomic64_set (&metric->total_dentries,0 ); ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret) -return ret; +return ret;;
drop this, please ^^^
Quoted 102 lines
fs/ceph/mds_client.c +22
-5ret =percpu_counter_init (&metric->d_lease_mis,0 ,GFP_KERNEL ); -if (ret) { -percpu_counter_destroy (&metric->d_lease_hit); -return ret; - } +if (ret) + goto err_dlease_mis; -return 0 ; + ret =percpu_counter_init (&metric->total_reads,0 ,GFP_KERNEL ); +if (ret) + goto err_total_reads; + + ret =percpu_counter_init (&metric->read_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_read_latency_sum; + +return ret; + +err_read_latency_sum: +percpu_counter_destroy (&metric->total_reads); +err_total_reads: +percpu_counter_destroy (&metric->d_lease_mis); +err_dlease_mis: +percpu_counter_destroy (&metric->d_lease_hit); + +return ret; }int ceph_mdsc_init (struct ceph_fs_client *fsc)@@ -4529,6 +4544,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.read_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_reads);percpu_counter_destroy (&mdsc->metric.d_lease_mis);percpu_counter_destroy (&mdsc->metric.d_lease_hit);fs/ceph/mds_client.h +2
-7@@ -16,6 +16,8 @@#include <linux/ceph/mdsmap.h> #include <linux/ceph/auth.h> +#include "metric.h" +/* The first 8 bits are reserved for old ceph releases */ enum ceph_feature_type {CEPHFS_FEATURE_MIMIC =8 ,@@ -361,13 +363,6 @@ struct cap_wait {int want; }; -/* This is the global metrics */ -struct ceph_client_metric { -atomic64_t total_dentries; -struct percpu_counter d_lease_hit; -struct percpu_counter d_lease_mis; -}; -/* * mds client state */ fs/ceph/metric.h new file +30
-0@@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_MDS_METRIC_H +#define _FS_CEPH_MDS_METRIC_H + +#include <linux/ceph/osd_client.h> + +/* This is the global metrics */ +struct ceph_client_metric { +atomic64_t total_dentries; +struct percpu_counter d_lease_hit; +struct percpu_counter d_lease_mis; + +struct percpu_counter total_reads; +struct percpu_counter read_latency_sum; +}; + +static inline void ceph_update_read_latency (struct ceph_client_metric *m, +struct ceph_osd_request *req, +int rc) +{ +if (!m || !req) +return ; + +if (rc >=0 || rc == -ENOENT || rc == -ETIMEDOUT ) { +s64 latency = req->r_end_stamp - req->r_start_stamp; +percpu_counter_inc (&m->total_reads); +percpu_counter_add (&m->read_latency_sum, latency); + } +} +#endif
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 05/11] ceph: add global read latency metric support
Quoted 1 line · Jeff Layton
On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
[...]
Quoted 4 lines
Quoted 4 lines
fs/ceph/addr.c +0
-0
Quoted 8 lines
fs/ceph/addr.c static int ceph_sync_readpages(struct ceph_fs_client *fsc, · line 198
Quoted 5 lines
fs/ceph/addr.c +1
-0@@ -195,6 +195,7 @@ static int ceph_sync_readpages(struct ceph_fs_client *fsc, int page_align) {struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_client_metric *metric = &fsc->mdsc->metric;nit: I think you can drop this variable and just dereference the metric field directly below where it's used. Ditto in other places where "metric" is only used once in the function.
Will fix them all.
Quoted 11 lines
fs/ceph/mds_client.c static int ceph_mdsc_metric_init(struct ceph_client_metric *metric) · line 4185
Quoted 10 lines
fs/ceph/mds_client.c +1
-1@@ -4182,14 +4182,29 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)atomic64_set (&metric->total_dentries,0 ); ret =percpu_counter_init (&metric->d_lease_hit,0 ,GFP_KERNEL );if (ret) -return ret; +return ret;;drop this, please ^^^
Will fix it.
Thanks.
PATCH RESEND v5 6/11 ceph: add global write latency metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
write 222 5287750000 23818693
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/addr.c | 10 ++++++++++ fs/ceph/debugfs.c | 6 ++++++ fs/ceph/file.c | 14 +++++++++++--- fs/ceph/mds_client.c | 15 ++++++++++++++- fs/ceph/metric.h | 17 +++++++++++++++++ 5 files changed, 58 insertions(+), 4 deletions(-)
fs/ceph/addr.c +10 -0
@@ -632,6 +632,7 @@ static int ceph_sync_writepages(struct ceph_fs_client *fsc, struct page **pages, int num_pages) {struct ceph_osd_client *osdc = &fsc->client->osdc; +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_request *req;int rc =0 ;int page_align = off & ~PAGE_MASK ;
@@ -653,6 +654,8 @@ static int ceph_sync_writepages(struct ceph_fs_client *fsc,if (!rc) rc =ceph_osdc_wait_request (osdc, req); +ceph_update_write_latency (metric, req, rc); +ceph_osdc_put_request (req);if (rc ==0 ) rc = len;
@@ -792,6 +795,7 @@ static void writepages_finish(struct ceph_osd_request *req)struct ceph_snap_context *snapc = req->r_snapc;struct address_space *mapping = inode->i_mapping;struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;bool remove_page;dout ("writepages_finish %p rc %d\n" , inode, rc);
@@ -804,6 +808,8 @@ static void writepages_finish(struct ceph_osd_request *req)ceph_clear_error_write (ci); } +ceph_update_write_latency (metric, req, rc); +/* * We lost the cache cap, need to truncate the page before * it is unlocked, otherwise we'd truncate it later in the
@@ -1752,6 +1758,7 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page)struct inode *inode =file_inode (filp);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_osd_request *req;struct page *page =NULL ;u64 len, inline_version;
@@ -1864,6 +1871,9 @@ int ceph_uninline_data(struct file *filp, struct page *locked_page) err =ceph_osdc_start_request (&fsc->client->osdc, req, false);if (!err) err =ceph_osdc_wait_request (&fsc->client->osdc, req); + +ceph_update_write_latency (metric, req, err); + out_put:ceph_osdc_put_request (req);if (err == -ECANCELED )
fs/ceph/debugfs.c +6 -0
@@ -140,6 +140,12 @@ static int metric_show(struct seq_file *s, void *p)seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"read" , total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + total =percpu_counter_sum (&mdsc->metric.total_writes); + sum =percpu_counter_sum (&mdsc->metric.write_latency_sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"write" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/file.c +11 -3
@@ -813,8 +813,12 @@ static void ceph_aio_complete_req(struct ceph_osd_request *req) inode, rc, osd_data->bvec_pos.iter.bi_size);/* r_start_stamp == 0 means the request was not submitted */ -if (req->r_start_stamp && !aio_req->write) -ceph_update_read_latency (metric, req, rc); +if (req->r_start_stamp) { +if (aio_req->write) +ceph_update_write_latency (metric, req, rc); +else +ceph_update_read_latency (metric, req, rc); + }if (rc == -EOLDSNAPC ) {struct ceph_aio_work *aio_work;
@@ -1061,7 +1065,9 @@ ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); -if (!write) +if (write) +ceph_update_write_latency (metric, req, ret); +else ceph_update_read_latency (metric, req, ret); size =i_size_read (inode);
@@ -1150,6 +1156,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,struct inode *inode =file_inode (file);struct ceph_inode_info *ci =ceph_inode (inode);struct ceph_fs_client *fsc =ceph_inode_to_client (inode); +struct ceph_client_metric *metric = &fsc->mdsc->metric;struct ceph_vino vino;struct ceph_osd_request *req;struct page **pages;
@@ -1235,6 +1242,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,if (!ret) ret =ceph_osdc_wait_request (&fsc->client->osdc, req); +ceph_update_write_latency (metric, req, ret); out:ceph_osdc_put_request (req);if (ret !=0 ) {
fs/ceph/mds_client.c +14 -1
@@ -4195,8 +4195,19 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)if (ret) goto err_read_latency_sum; -return ret; + ret =percpu_counter_init (&metric->total_writes,0 ,GFP_KERNEL ); +if (ret) + goto err_total_writes; + ret =percpu_counter_init (&metric->write_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_write_latency_sum; + +return ret; +err_write_latency_sum: +percpu_counter_destroy (&metric->total_writes); +err_total_writes: +percpu_counter_destroy (&metric->read_latency_sum); err_read_latency_sum:percpu_counter_destroy (&metric->total_reads); err_total_reads:
@@ -4544,6 +4555,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.write_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_writes);percpu_counter_destroy (&mdsc->metric.read_latency_sum);percpu_counter_destroy (&mdsc->metric.total_reads);percpu_counter_destroy (&mdsc->metric.d_lease_mis);
fs/ceph/metric.h +17 -0
@@ -12,6 +12,9 @@ struct ceph_client_metric {struct percpu_counter total_reads;struct percpu_counter read_latency_sum; + +struct percpu_counter total_writes; +struct percpu_counter write_latency_sum; }; static inline void ceph_update_read_latency(struct ceph_client_metric *m,
@@ -27,4 +30,18 @@ static inline void ceph_update_read_latency(struct ceph_client_metric *m, percpu_counter_add(&m->read_latency_sum, latency); } } + +static inline void ceph_update_write_latency (struct ceph_client_metric *m, +struct ceph_osd_request *req, +int rc) +{ +if (!m || !req) +return ; + +if (!rc || rc == -ETIMEDOUT ) { +s64 latency = req->r_end_stamp - req->r_start_stamp; +percpu_counter_inc (&m->total_writes); +percpu_counter_add (&m->write_latency_sum, latency); + } +}#endif
Signature · 1 line
2.21.0
PATCH RESEND v5 7/11 ceph: add global metadata perf metric support
In reply to [email protected]From: Xiubo Li <[email protected]>
item total sum_lat(us) avg_lat(us)
-----------------------------------------------------
metadata 1288 24506000 19026
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 6 ++++++ fs/ceph/mds_client.c | 25 +++++++++++++++++++++++++ fs/ceph/metric.h | 13 +++++++++++++ 3 files changed, 44 insertions(+)
fs/ceph/debugfs.c +6 -0
@@ -146,6 +146,12 @@ static int metric_show(struct seq_file *s, void *p)seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"write" , total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); + total =percpu_counter_sum (&mdsc->metric.total_metadatas); + sum =percpu_counter_sum (&mdsc->metric.metadata_latency_sum); + avg = total ? sum / total :0 ; +seq_printf (s,"%-14s%-12lld%-16lld%lld\n" ,"metadata" , + total, sum /NSEC_PER_USEC , avg /NSEC_PER_USEC ); +seq_printf (s,"\n" );seq_printf (s,"item total miss hit\n" );seq_printf (s,"-------------------------------------------------\n" );
fs/ceph/mds_client.c +25 -0
@@ -2772,6 +2772,12 @@ static int ceph_mdsc_wait_request(struct ceph_mds_client *mdsc,else err = timeleft;/* killed */ } + +if (!err || err == -EIO ) { +s64 latency = jiffies - req->r_started; +ceph_update_metadata_latency (&mdsc->metric, latency); + } +dout ("do_request waited, got %d\n" , err);mutex_lock (&mdsc->mutex);
@@ -3033,6 +3039,11 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg)/* kick calling process */ complete_request (mdsc, req); + +if (!result || result == -ENOENT ) { +s64 latency = jiffies - req->r_started; +ceph_update_metadata_latency (&mdsc->metric, latency); + } out:ceph_mdsc_put_request (req);return ;
@@ -4203,7 +4214,19 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)if (ret) goto err_write_latency_sum; + ret =percpu_counter_init (&metric->total_metadatas,0 ,GFP_KERNEL ); +if (ret) + goto err_total_metadatas; + + ret =percpu_counter_init (&metric->metadata_latency_sum,0 ,GFP_KERNEL ); +if (ret) + goto err_metadata_latency_sum; +return ret; +err_metadata_latency_sum: +percpu_counter_destroy (&metric->total_metadatas); +err_total_metadatas: +percpu_counter_destroy (&metric->write_latency_sum); err_write_latency_sum:percpu_counter_destroy (&metric->total_writes); err_total_writes:
@@ -4555,6 +4578,8 @@ void ceph_mdsc_destroy(struct ceph_fs_client *fsc)ceph_mdsc_stop (mdsc); +percpu_counter_destroy (&mdsc->metric.metadata_latency_sum); +percpu_counter_destroy (&mdsc->metric.total_metadatas);percpu_counter_destroy (&mdsc->metric.write_latency_sum);percpu_counter_destroy (&mdsc->metric.total_writes);percpu_counter_destroy (&mdsc->metric.read_latency_sum);
fs/ceph/metric.h +13 -0
@@ -15,6 +15,9 @@ struct ceph_client_metric {struct percpu_counter total_writes;struct percpu_counter write_latency_sum; + +struct percpu_counter total_metadatas; +struct percpu_counter metadata_latency_sum; }; static inline void ceph_update_read_latency(struct ceph_client_metric *m,
@@ -44,4 +47,14 @@ static inline void ceph_update_write_latency(struct ceph_client_metric *m, percpu_counter_add(&m->write_latency_sum, latency); } } + +static inline void ceph_update_metadata_latency (struct ceph_client_metric *m, +s64 latency) +{ +if (!m) +return ; + +percpu_counter_inc (&m->total_metadatas); +percpu_counter_add (&m->metadata_latency_sum, latency); +}#endif
Signature · 1 line
2.21.0
PATCH RESEND v5 8/11 ceph: periodically send perf metrics to MDS
In reply to [email protected]From: Xiubo Li <[email protected]>
Add enable/disable sending metrics to MDS debugfs and disabled as default, if it's enabled the kclient will send metrics every second.
This will send global dentry lease hit/miss and read/write/metadata latency metrics and each session's caps hit/miss metric to MDS.
Every time only sends the global metrics once via any availible session.
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 44 +++++++- fs/ceph/mds_client.c | 201 ++++++++++++++++++++++++++++++++--- fs/ceph/mds_client.h | 3 + fs/ceph/metric.h | 76 +++++++++++++ fs/ceph/super.h | 1 + include/linux/ceph/ceph_fs.h | 1 + 6 files changed, 307 insertions(+), 19 deletions(-)
fs/ceph/debugfs.c +42 -2
@@ -124,6 +124,40 @@ static int mdsc_show(struct seq_file *s, void *p)return 0 ; } +/* +* metrics debugfs +*/ +static int sending_metrics_set (void *data,u64 val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +if (val >1 ) { +pr_err ("Invalid sending metrics set value %llu\n" , val); +return -EINVAL ; + } + +mutex_lock (&mdsc->mutex); + mdsc->sending_metrics = (unsigned int )val; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +static int metric_show (struct seq_file *s,void *p) {struct ceph_fs_client *fsc = s->private;
@@ -302,11 +336,9 @@ static int congestion_kb_get(void *data, u64 *val) *val = (u64 )fsc->mount_options->congestion_kb;return 0 ; } -DEFINE_SIMPLE_ATTRIBUTE (congestion_kb_fops, congestion_kb_get, congestion_kb_set,"%llu\n" ); -void ceph_fs_debugfs_cleanup (struct ceph_fs_client *fsc) {dout ("ceph_fs_debugfs_cleanup\n" );
@@ -316,6 +348,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_mds_sessions);debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric); +debugfs_remove (fsc->debugfs_sending_metrics);debugfs_remove (fsc->debugfs_mdsc); }
@@ -356,6 +389,13 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &mdsc_show_fops); + fsc->debugfs_sending_metrics = +debugfs_create_file ("sending_metrics" , +0600 , + fsc->client->debugfs_dir, + fsc, + &sending_metrics_fops); + fsc->debugfs_metric = debugfs_create_file("metrics", 0400, fsc->client->debugfs_dir,
fs/ceph/mds_client.c +184 -17
259 lines not on this page · raw message
fs/ceph/mds_client.h +3 -0
@@ -451,6 +451,9 @@ struct ceph_mds_client {struct list_head dentry_leases;/* fifo list */ struct list_head dentry_dir_leases;/* lru list */ +/* metrics */ +unsigned int sending_metrics; +unsigned int ticks;struct ceph_client_metric metric;spinlock_t snapid_map_lock;
fs/ceph/metric.h +76 -0
82 lines not on this page · raw message
fs/ceph/super.h +1 -0
@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_sending_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
include/linux/ceph/ceph_fs.h +1 -0
@@ -130,6 +130,7 @@ struct ceph_dir_layout {#define CEPH_MSG_CLIENT_REQUEST 24#define CEPH_MSG_CLIENT_REQUEST_FORWARD 25#define CEPH_MSG_CLIENT_REPLY 26 +#define CEPH_MSG_CLIENT_METRICS 29#define CEPH_MSG_CLIENT_CAPS 0x310#define CEPH_MSG_CLIENT_LEASE 0x311#define CEPH_MSG_CLIENT_SNAP 0x312
Signature · 1 line
2.21.0
Re: [PATCH resend v5 08/11] ceph: periodically send perf metrics to MDS
Quoted 36 lines · On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
From: Xiubo Li <[email protected]>
Add enable/disable sending metrics to MDS debugfs and disabled as default, if it's enabled the kclient will send metrics every second.
This will send global dentry lease hit/miss and read/write/metadata latency metrics and each session's caps hit/miss metric to MDS.
Every time only sends the global metrics once via any availible session.
URL: https://tracker.ceph.com/issues/43215
Signed-off-by Xiubo Li <[email protected]>--- fs/ceph/debugfs.c | 44 +++++++- fs/ceph/mds_client.c | 201 ++++++++++++++++++++++++++++++++--- fs/ceph/mds_client.h | 3 + fs/ceph/metric.h | 76 +++++++++++++ fs/ceph/super.h | 1 + include/linux/ceph/ceph_fs.h | 1 + 6 files changed, 307 insertions(+), 19 deletions(-)fs/ceph/debugfs.c +5
-0@@ -124,6 +124,40 @@ static int mdsc_show(struct seq_file *s, void *p)return 0 ; } +/* +* metrics debugfs +*/ +static int sending_metrics_set (void *data,u64 val) +{
fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 141
Quoted 10 lines
fs/ceph/debugfs.c +10
-0+struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +if (val >1 ) { +pr_err ("Invalid sending metrics set value %llu\n" , val); +return -EINVAL ; + } + +mutex_lock (&mdsc->mutex); + mdsc->sending_metrics = (unsigned int )val;
Shouldn't that be a bool cast? Do we even need a cast there?
Quoted 9 lines
fs/ceph/debugfs.c +9
-0+mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc;
fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 160
Quoted 10 lines
fs/ceph/debugfs.c +10
-0+ +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +
I'd like to hear more about how we expect users to use this facility. This debugfs file doesn't seem consistent with the rest of the UI, and I imagine if the box reboots you'd have to (manually) re-enable it after mount, right? Maybe this should be a mount option instead?
Quoted 186 lines
fs/ceph/debugfs.c +8
-2static int metric_show (struct seq_file *s,void *p) {struct ceph_fs_client *fsc = s->private;@@ -302,11 +336,9 @@ static int congestion_kb_get(void *data, u64 *val) *val = (u64 )fsc->mount_options->congestion_kb;return 0 ; } -DEFINE_SIMPLE_ATTRIBUTE (congestion_kb_fops, congestion_kb_get, congestion_kb_set,"%llu\n" ); -void ceph_fs_debugfs_cleanup (struct ceph_fs_client *fsc) {dout ("ceph_fs_debugfs_cleanup\n" );@@ -316,6 +348,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_mds_sessions);debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric); +debugfs_remove (fsc->debugfs_sending_metrics);debugfs_remove (fsc->debugfs_mdsc); }@@ -356,6 +389,13 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) fsc, &mdsc_show_fops); + fsc->debugfs_sending_metrics = +debugfs_create_file ("sending_metrics" , +0600 , + fsc->client->debugfs_dir, + fsc, + &sending_metrics_fops); + fsc->debugfs_metric = debugfs_create_file("metrics", 0400, fsc->client->debugfs_dir,fs/ceph/mds_client.c +136
-0@@ -4104,13 +4104,156 @@ static void maybe_recover_session(struct ceph_mds_client *mdsc)ceph_force_reconnect (fsc->sb); } +/* +* called under s_mutex +*/ +static bool ceph_mdsc_send_metrics (struct ceph_mds_client *mdsc, +struct ceph_mds_session *s, +bool skip_global) +{ +struct ceph_metric_head *head; +struct ceph_metric_cap *cap; +struct ceph_metric_dentry_lease *lease; +struct ceph_metric_read_latency *read; +struct ceph_metric_write_latency *write; +struct ceph_metric_metadata_latency *meta; +struct ceph_msg *msg; +struct timespec64 ts; +s32 len =sizeof (*head) +sizeof (*cap); +s64 sum, total, avg; +s32 items =0 ; + +if (!mdsc || !s) +return false; + +if (!skip_global) { + len +=sizeof (*lease); + len +=sizeof (*read); + len +=sizeof (*write); + len +=sizeof (*meta); + } + + msg =ceph_msg_new (CEPH_MSG_CLIENT_METRICS , len,GFP_NOFS , true); +if (!msg) { +pr_err ("send metrics to mds%d, failed to allocate message\n" , + s->s_mds); +return false; + } + + head = msg->front.iov_base; + +/* encode the cap metric */ + cap = (struct ceph_metric_cap *)(head +1 ); + cap->type =cpu_to_le32 (CLIENT_METRIC_TYPE_CAP_INFO ); + cap->ver =1 ; + cap->campat =1 ; + cap->data_len =cpu_to_le32 (sizeof (*cap) -10 ); + cap->hit =cpu_to_le64 (percpu_counter_sum (&s->i_caps_hit)); + cap->mis =cpu_to_le64 (percpu_counter_sum (&s->i_caps_mis)); + cap->total =cpu_to_le64 (s->s_nr_caps); + items++; + +dout ("cap metric hit %lld, mis %lld, total caps %lld" , +le64_to_cpu (cap->hit),le64_to_cpu (cap->mis), +le64_to_cpu (cap->total)); + +/* only send the global once */ +if (skip_global) + goto skip_global; + +/* encode the dentry lease metric */ + lease = (struct ceph_metric_dentry_lease *)(cap +1 ); + lease->type =cpu_to_le32 (CLIENT_METRIC_TYPE_DENTRY_LEASE ); + lease->ver =1 ; + lease->campat =1 ; + lease->data_len =cpu_to_le32 (sizeof (*lease) -10 ); + lease->hit =cpu_to_le64 (percpu_counter_sum (&mdsc->metric.d_lease_hit)); + lease->mis =cpu_to_le64 (percpu_counter_sum (&mdsc->metric.d_lease_mis)); + lease->total =cpu_to_le64 (atomic64_read (&mdsc->metric.total_dentries)); + items++; + +dout ("dentry lease metric hit %lld, mis %lld, total dentries %lld" , +le64_to_cpu (lease->hit),le64_to_cpu (lease->mis), +le64_to_cpu (lease->total)); + +/* encode the read latency metric */ + read = (struct ceph_metric_read_latency *)(lease +1 ); + read->type =cpu_to_le32 (CLIENT_METRIC_TYPE_READ_LATENCY ); + read->ver =1 ; + read->campat =1 ; + read->data_len =cpu_to_le32 (sizeof (*read) -10 ); + total =percpu_counter_sum (&mdsc->metric.total_reads), + sum =percpu_counter_sum (&mdsc->metric.read_latency_sum); + avg = total ? sum / total :0 ; + ts =ns_to_timespec64 (avg); + read->sec =cpu_to_le32 (ts.tv_sec); + read->nsec =cpu_to_le32 (ts.tv_nsec); + items++; + +dout ("read latency metric total %lld, sum lat %lld, avg lat %lld" , + total, sum, avg); + +/* encode the write latency metric */ + write = (struct ceph_metric_write_latency *)(read +1 ); + write->type =cpu_to_le32 (CLIENT_METRIC_TYPE_WRITE_LATENCY ); + write->ver =1 ; + write->campat =1 ; + write->data_len =cpu_to_le32 (sizeof (*write) -10 ); + total =percpu_counter_sum (&mdsc->metric.total_writes), + sum =percpu_counter_sum (&mdsc->metric.write_latency_sum); + avg = total ? sum / total :0 ; + ts =ns_to_timespec64 (avg); + write->sec =cpu_to_le32 (ts.tv_sec); + write->nsec =cpu_to_le32 (ts.tv_nsec); + items++; + +dout ("write latency metric total %lld, sum lat %lld, avg lat %lld" , + total, sum, avg); + +/* encode the metadata latency metric */ + meta = (struct ceph_metric_metadata_latency *)(write +1 ); + meta->type =cpu_to_le32 (CLIENT_METRIC_TYPE_METADATA_LATENCY ); + meta->ver =1 ; + meta->campat =1 ; + meta->data_len =cpu_to_le32 (sizeof (*meta) -10 ); + total =percpu_counter_sum (&mdsc->metric.total_metadatas), + sum =percpu_counter_sum (&mdsc->metric.metadata_latency_sum); + avg = total ? sum / total :0 ; + ts =ns_to_timespec64 (avg); + meta->sec =cpu_to_le32 (ts.tv_sec); + meta->nsec =cpu_to_le32 (ts.tv_nsec); + items++; + +dout ("metadata latency metric total %lld, sum lat %lld, avg lat %lld" , + total, sum, avg); + +skip_global: +put_unaligned_le32 (items, &head->num); + msg->front.iov_len =cpu_to_le32 (len); + msg->hdr.version =cpu_to_le16 (1 ); + msg->hdr.compat_version =cpu_to_le16 (1 ); + msg->hdr.front_len =cpu_to_le32 (msg->front.iov_len); +dout ("send metrics to mds%d %p\n" , s->s_mds, msg); +ceph_con_send (&s->s_con, msg); + +return true; +} +/* * delayed work -- periodically trim expired leases, renew caps with mds */ +#define CEPH_WORK_DELAY_DEF 5static void schedule_delayed (struct ceph_mds_client *mdsc) {
fs/ceph/mds_client.c static void maybe_recover_session(struct ceph_mds_client *mdsc) · line 4255
Quoted 10 lines
fs/ceph/mds_client.c +8
-2-int delay =5 ; -unsigned hz =round_jiffies_relative (HZ * delay); +unsigned int hz; +int delay =CEPH_WORK_DELAY_DEF ; + +mutex_lock (&mdsc->mutex); +if (mdsc->sending_metrics) + delay =1 ; +mutex_unlock (&mdsc->mutex); +
The mdsc->mutex is dropped in the callers a little before this is called, so this is a little too mutex-thrashy. I think you'd be better off changing this function to be called with the mutex still held.
Quoted 139 lines
fs/ceph/mds_client.c +40
-15+ hz =round_jiffies_relative (HZ * delay);schedule_delayed_work (&mdsc->delayed_work, hz); }@@ -4121,18 +4264,28 @@ static void delayed_work(struct work_struct *work) container_of(work, struct ceph_mds_client, delayed_work.work);int renew_interval;int renew_caps; +bool metric_only; +bool sending_metrics; +bool g_skip = false;dout ("mdsc delayed_work\n" );mutex_lock (&mdsc->mutex); - renew_interval = mdsc->mdsmap->m_session_timeout >>2 ; - renew_caps =time_after_eq (jiffies,HZ *renew_interval + - mdsc->last_renew_caps); -if (renew_caps) - mdsc->last_renew_caps = jiffies; + sending_metrics = !!mdsc->sending_metrics; + metric_only = mdsc->sending_metrics && + (mdsc->ticks++ %CEPH_WORK_DELAY_DEF ); + +if (!metric_only) { + renew_interval = mdsc->mdsmap->m_session_timeout >>2 ; + renew_caps =time_after_eq (jiffies,HZ *renew_interval + + mdsc->last_renew_caps); +if (renew_caps) + mdsc->last_renew_caps = jiffies; + }for (i =0 ; i < mdsc->max_sessions; i++) {struct ceph_mds_session *s =__ceph_lookup_mds_session (mdsc, i); +if (!s)continue ;if (s->s_state ==CEPH_MDS_SESSION_CLOSING ) {@@ -4158,13 +4311,20 @@ static void delayed_work(struct work_struct *work)mutex_unlock (&mdsc->mutex);mutex_lock (&s->s_mutex); -if (renew_caps) -send_renew_caps (mdsc, s); -else -ceph_con_keepalive (&s->s_con); -if (s->s_state ==CEPH_MDS_SESSION_OPEN || - s->s_state ==CEPH_MDS_SESSION_HUNG ) -ceph_send_cap_releases (mdsc, s); + +if (sending_metrics) + g_skip =ceph_mdsc_send_metrics (mdsc, s, g_skip); + +if (!metric_only) { +if (renew_caps) +send_renew_caps (mdsc, s); +else +ceph_con_keepalive (&s->s_con); +if (s->s_state ==CEPH_MDS_SESSION_OPEN || + s->s_state ==CEPH_MDS_SESSION_HUNG ) +ceph_send_cap_releases (mdsc, s); + } +mutex_unlock (&s->s_mutex);ceph_put_mds_session (s);@@ -4172,6 +4332,9 @@ static void delayed_work(struct work_struct *work) }mutex_unlock (&mdsc->mutex); +if (metric_only) + goto delay_work; +ceph_check_delayed_caps (mdsc);ceph_queue_cap_reclaim_work (mdsc);@@ -4180,11 +4343,13 @@ static void delayed_work(struct work_struct *work)maybe_recover_session (mdsc); +delay_work:schedule_delayed (mdsc); } -static int ceph_mdsc_metric_init (struct ceph_client_metric *metric) +static int ceph_mdsc_metric_init (struct ceph_mds_client *mdsc) { +struct ceph_client_metric *metric = &mdsc->metric;int ret;if (!metric)@@ -4222,7 +4387,9 @@ static int ceph_mdsc_metric_init(struct ceph_client_metric *metric)if (ret) goto err_metadata_latency_sum; -return ret; + mdsc->sending_metrics =0 ; + mdsc->ticks =0 ; +return 0 ; err_metadata_latency_sum:percpu_counter_destroy (&metric->total_metadatas); err_total_metadatas:@@ -4294,7 +4461,7 @@ int ceph_mdsc_init(struct ceph_fs_client *fsc)init_waitqueue_head (&mdsc->cap_flushing_wq);INIT_WORK (&mdsc->cap_reclaim_work, ceph_cap_reclaim_work);atomic_set (&mdsc->cap_reclaim_pending,0 ); - err =ceph_mdsc_metric_init (&mdsc->metric); + err =ceph_mdsc_metric_init (mdsc);if (err) goto err_mdsmap;fs/ceph/mds_client.h +3
-0@@ -451,6 +451,9 @@ struct ceph_mds_client {struct list_head dentry_leases;/* fifo list */ struct list_head dentry_dir_leases;/* lru list */ +/* metrics */ +unsigned int sending_metrics; +unsigned int ticks;struct ceph_client_metric metric;spinlock_t snapid_map_lock;fs/ceph/metric.h +6
-0@@ -4,6 +4,82 @@#include <linux/ceph/osd_client.h> +enum ceph_metric_type { +CLIENT_METRIC_TYPE_CAP_INFO , +CLIENT_METRIC_TYPE_READ_LATENCY , +CLIENT_METRIC_TYPE_WRITE_LATENCY , +CLIENT_METRIC_TYPE_METADATA_LATENCY , +CLIENT_METRIC_TYPE_DENTRY_LEASE ,
fs/ceph/metric.h line 22
Quoted 10 lines
fs/ceph/metric.h +10
-0+ + CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE, +}; + +/* metric caps header */ +struct ceph_metric_cap { +__le32 type;/* ceph metric type */ + +__u8 ver; +__u8 campat;
I think you meant "compat" here.
Quoted 48 lines
fs/ceph/metric.h +48
-0+ +__le32 data_len;/* length of sizeof(hit + mis + total) */ +__le64 hit; +__le64 mis; +__le64 total; +} __attribute__ ((packed)); + +/* metric dentry lease header */ +struct ceph_metric_dentry_lease { +__le32 type;/* ceph metric type */ + +__u8 ver; +__u8 campat; + +__le32 data_len;/* length of sizeof(hit + mis + total) */ +__le64 hit; +__le64 mis; +__le64 total; +} __attribute__ ((packed)); + +/* metric read latency header */ +struct ceph_metric_read_latency { +__le32 type;/* ceph metric type */ + +__u8 ver; +__u8 campat; + +__le32 data_len;/* length of sizeof(sec + nsec) */ +__le32 sec; +__le32 nsec; +} __attribute__ ((packed)); + +/* metric write latency header */ +struct ceph_metric_write_latency { +__le32 type;/* ceph metric type */ + +__u8 ver; +__u8 campat; + +__le32 data_len;/* length of sizeof(sec + nsec) */ +__le32 sec; +__le32 nsec; +} __attribute__ ((packed)); + +/* metric metadata latency header */ +struct ceph_metric_metadata_latency { +__le32 type;/* ceph metric type */ +
fs/ceph/metric.h line 80
Quoted 10 lines
fs/ceph/metric.h +10
-0+__u8 ver; +__u8 campat; + +__le32 data_len;/* length of sizeof(sec + nsec) */ +__le32 sec; +__le32 nsec; +} __attribute__ ((packed)); + +struct ceph_metric_head { +__le32 num;/* the number of metrics will be sent */
"the number of metrics that will be sent"
Quoted 29 lines
fs/ceph/metric.h +2
-0+} __attribute__ ((packed)); +/* This is the global metrics */ struct ceph_client_metric {atomic64_t total_dentries;fs/ceph/super.h +1
-0@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_sending_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif include/linux/ceph/ceph_fs.h +1
-0@@ -130,6 +130,7 @@ struct ceph_dir_layout {#define CEPH_MSG_CLIENT_REQUEST 24#define CEPH_MSG_CLIENT_REQUEST_FORWARD 25#define CEPH_MSG_CLIENT_REPLY 26 +#define CEPH_MSG_CLIENT_METRICS 29#define CEPH_MSG_CLIENT_CAPS 0x310#define CEPH_MSG_CLIENT_LEASE 0x311#define CEPH_MSG_CLIENT_SNAP 0x312
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 08/11] ceph: periodically send perf metrics to MDS
Quoted 1 line · Jeff Layton
On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
[...]
Quoted 17 lines
Quoted 6 lines
fs/ceph/debugfs.c +5
-0+/* +* metrics debugfs +*/ +static int sending_metrics_set (void *data,u64 val) +{fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 141
Quoted 10 lines
fs/ceph/debugfs.c +10
-0+struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +if (val >1 ) { +pr_err ("Invalid sending metrics set value %llu\n" , val); +return -EINVAL ; + } + +mutex_lock (&mdsc->mutex); + mdsc->sending_metrics = (unsigned int )val;Shouldn't that be a bool cast? Do we even need a cast there?
Will switch sending_metrics to bool type instead.
Quoted 15 lines
Quoted 15 lines
fs/ceph/debugfs.c +15
-0+mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ;
Quoted 8 lines
fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 160
Quoted 4 lines
fs/ceph/debugfs.c +4
-0+} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +I'd like to hear more about how we expect users to use this facility. This debugfs file doesn't seem consistent with the rest of the UI, and I imagine if the box reboots you'd have to (manually) re-enable it after mount, right? Maybe this should be a mount option instead?
A mount option means we must do the unmounting to disable it.
I was thinking with the debugfs file we can do the debug or tuning even in the product setups at any time, usually this should be disabled since it will send it per second.
Or we could merge the "sending_metric" to "metrics" UI, just writing "enable"/"disable" to enable/disable sending the metrics to ceph, and just like the "reset" does to clean the metrics.
Then the "/sys/kernel/debug/ceph/XXX.clientYYY/metrics" could be writable with:
"reset" --> to clean and reset the metrics counters
"enable" --> enable sending metrics to ceph cluster
"disable" --> disable sending metrics to ceph cluster
Will this be better ?
[...]
Quoted 16 lines
/*
* delayed work -- periodically trim expired leases, renew caps with mds */fs/ceph/mds_client.c +9
-2+#define CEPH_WORK_DELAY_DEF 5static void schedule_delayed (struct ceph_mds_client *mdsc) { -int delay =5 ; -unsigned hz =round_jiffies_relative (HZ * delay); +unsigned int hz; +int delay =CEPH_WORK_DELAY_DEF ; + +mutex_lock (&mdsc->mutex); +if (mdsc->sending_metrics) + delay =1 ; +mutex_unlock (&mdsc->mutex); +
Quoted 3 lines
The
mdsc->mutexis dropped in the callers a little before this is called, so this is a little too mutex-thrashy. I think you'd be better off changing this function to be called with the mutex still held.
Will fix it.
[...]
Quoted 7 lines
fs/ceph/metric.h line 22
Quoted 6 lines
fs/ceph/metric.h +6
-0+/* metric caps header */ +struct ceph_metric_cap { +__le32 type;/* ceph metric type */ + +__u8 ver; +__u8 campat;I think you meant "compat" here.
Will fix it.
[...]
Quoted 7 lines
Quoted 7 lines
fs/ceph/metric.h +7
-0+/* metric metadata latency header */ +struct ceph_metric_metadata_latency { +__le32 type;/* ceph metric type */ + +__u8 ver; +__u8 campat; +
Quoted 8 lines
fs/ceph/metric.h line 80
Quoted 7 lines
fs/ceph/metric.h +7
-0+__le32 data_len;/* length of sizeof(sec + nsec) */ +__le32 sec; +__le32 nsec; +} __attribute__ ((packed)); + +struct ceph_metric_head { +__le32 num;/* the number of metrics will be sent */ "the number of metrics that will be sent"
Will fix it.
Thanks,
Quoted 29 lines
Quoted 29 lines
fs/ceph/metric.h +2
-0+} __attribute__ ((packed)); +/* This is the global metrics */ struct ceph_client_metric {atomic64_t total_dentries;fs/ceph/super.h +1
-0@@ -128,6 +128,7 @@ struct ceph_fs_client {struct dentry *debugfs_congestion_kb;struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap; +struct dentry *debugfs_sending_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif include/linux/ceph/ceph_fs.h +1
-0@@ -130,6 +130,7 @@ struct ceph_dir_layout {#define CEPH_MSG_CLIENT_REQUEST 24#define CEPH_MSG_CLIENT_REQUEST_FORWARD 25#define CEPH_MSG_CLIENT_REPLY 26 +#define CEPH_MSG_CLIENT_METRICS 29#define CEPH_MSG_CLIENT_CAPS 0x310#define CEPH_MSG_CLIENT_LEASE 0x311#define CEPH_MSG_CLIENT_SNAP 0x312
Re: [PATCH resend v5 08/11] ceph: periodically send perf metrics to MDS
Quoted 23 lines · Xiubo Li
Quoted 1 line · Jeff Layton
On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
[...]
Quoted 11 lines
Quoted 11 lines
fs/ceph/debugfs.c +11
-0+ +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc; + +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ;Quoted 8 lines
fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 160
Quoted 4 lines
fs/ceph/debugfs.c +4
-0+} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +I'd like to hear more about how we expect users to use this facility. This debugfs file doesn't seem consistent with the rest of the UI, and I imagine if the box reboots you'd have to (manually) re-enable it after mount, right? Maybe this should be a mount option instead?
Quoted 2 lines
A mount option means we must do the unmounting to disable it.
Technically, no. You could wire it up so that you could enable and disable it via -o remount. For example:
# mount -o remount,metrics=disabled
Another option might be a module parameter if this is something that you really want to be global (and not per-mount or per-session).
Quoted 4 lines
I was thinking with the debugfs file we can do the debug or tuning even in the product setups at any time, usually this should be disabled since it will send it per second.
Meh, one frame per second doesn't seem like it'll add much overhead.
Also, why one update per second? Should that interval be tunable?
Quoted 7 lines
Or we could merge the "
sending_metric" to "metrics" UI, just writing "enable"/"disable" to enable/disable sending the metrics to ceph, and just like the "reset" does to clean the metrics.Then the "
/sys/kernel/debug/ceph/XXX.clientYYY/metrics" could be writable with:
Quoted 8 lines
"reset" --> to clean and reset the metrics counters
"enable" --> enable sending metrics to ceph cluster
"disable" --> disable sending metrics to ceph cluster
Will this be better ?
I guess it's not clear to me how you intend for this to be used.
A debugfs switch means that this is being enabled and disabled on a per- session basis. Is the user supposed to turn this on for all, or just one session? How do they know?
Is this something we expect people to just turn on briefly when they are experiencing a problem, or is this something that we expect to be turned on and left on for long periods of time?
If it's the latter then setting up a mount in /etc/fstab is not going to be sufficient for an admin. She'll have to write a script or something that goes in after the mount and enables this by writing to debugfs after rebooting. Yuck.
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 08/11] ceph: periodically send perf metrics to MDS
Quoted 23 lines · Jeff Layton
Quoted 22 lines · Xiubo Li
Quoted 1 line · Jeff Layton
On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
[...]
Quoted 19 lines
Quoted 5 lines
fs/ceph/debugfs.c +5
-0+ +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc;fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 160
Quoted 10 lines
fs/ceph/debugfs.c +10
-0+ +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +I'd like to hear more about how we expect users to use this facility. This debugfs file doesn't seem consistent with the rest of the UI, and I imagine if the box reboots you'd have to (manually) re-enable it after mount, right? Maybe this should be a mount option instead?
Quoted 6 lines
Quoted 2 lines
A mount option means we must do the unmounting to disable it.
Technically, no. You could wire it up so that you could enable and disable it via -o remount. For example:
# mount -o remount,metrics=disabled
Yeah, this is cool.
Quoted 9 lines
Another option might be a module parameter if this is something that you really want to be global (and not per-mount or per-session).
Quoted 4 lines
I was thinking with the debugfs file we can do the debug or tuning even in the product setups at any time, usually this should be disabled since it will send it per second.
Meh, one frame per second doesn't seem like it'll add much overhead.
Okay.
Quoted 2 lines
Also, why one update per second? Should that interval be tunable?
Per second just keep it the same with the fuse client.
Quoted 13 lines
Quoted 13 lines
Or we could merge the "
sending_metric" to "metrics" UI, just writing "enable"/"disable" to enable/disable sending the metrics to ceph, and just like the "reset" does to clean the metrics.Then the "
/sys/kernel/debug/ceph/XXX.clientYYY/metrics" could be writable with:"reset" --> to clean and reset the metrics counters
"enable" --> enable sending metrics to ceph cluster
"disable" --> disable sending metrics to ceph cluster
Quoted 7 lines
Quoted 2 lines
Will this be better ?
I guess it's not clear to me how you intend for this to be used.
A debugfs switch means that this is being enabled and disabled on a per- session basis. Is the user supposed to turn this on for all, or just one session? How do they know?
Not for all, just per-superblock.
Quoted 4 lines
Is this something we expect people to just turn on briefly when they are experiencing a problem, or is this something that we expect to be turned on and left on for long periods of time?
If this won't add much overhead even per second, let's keep sending the metrics to ceph always and the mount option for this switch is not needed any more.
And there is already a switch to enable/disable showing the metrics in the ceph side, if here add another switch per client, it will be also yucky for admins .
Let's make the update interval tunable and per second as default. Maybe we should make this as a global UI for all clients ?
Is this okay ?
Thanks.
Quoted 5 lines
If it's the latter then setting up a mount in
/etc/fstabis not going to be sufficient for an admin. She'll have to write a script or something that goes in after the mount and enables this by writing to debugfs after rebooting. Yuck.
Re: [PATCH resend v5 08/11] ceph: periodically send perf metrics to MDS
Quoted 40 lines · Xiubo Li
Quoted 23 lines · Jeff Layton
Quoted 22 lines · Xiubo Li
Quoted 1 line · Jeff Layton
On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
[...]
Quoted 19 lines
Quoted 5 lines
fs/ceph/debugfs.c +5
-0+ +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc;fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 160
Quoted 10 lines
fs/ceph/debugfs.c +10
-0+ +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +I'd like to hear more about how we expect users to use this facility. This debugfs file doesn't seem consistent with the rest of the UI, and I imagine if the box reboots you'd have to (manually) re-enable it after mount, right? Maybe this should be a mount option instead?
Quoted 6 lines
Quoted 2 lines
A mount option means we must do the unmounting to disable it.
Technically, no. You could wire it up so that you could enable and disable it via -o remount. For example:
# mount -o remount,metrics=disabledYeah, this is cool.
Quoted 7 lines
Another option might be a module parameter if this is something that you really want to be global (and not per-mount or per-session).
Quoted 4 lines
I was thinking with the debugfs file we can do the debug or tuning even in the product setups at any time, usually this should be disabled since it will send it per second.
Quoted 6 lines
Quoted 1 line
Meh, one frame per second doesn't seem like it'll add much overhead.
Okay.
Quoted 1 line
Also, why one update per second? Should that interval be tunable?
Per second just keep it the same with the fuse client.
Ok.
Quoted 16 lines
Quoted 16 lines
Quoted 16 lines
Or we could merge the "
sending_metric" to "metrics" UI, just writing "enable"/"disable" to enable/disable sending the metrics to ceph, and just like the "reset" does to clean the metrics.Then the "
/sys/kernel/debug/ceph/XXX.clientYYY/metrics" could be writable with:"reset" --> to clean and reset the metrics counters
"enable" --> enable sending metrics to ceph cluster
"disable" --> disable sending metrics to ceph cluster
Will this be better ?
Quoted 8 lines
Quoted 5 lines
I guess it's not clear to me how you intend for this to be used.
A debugfs switch means that this is being enabled and disabled on a per- session basis. Is the user supposed to turn this on for all, or just one session? How do they know?
Not for all, just per-superblock.
If it's per-superblock, then a debugfs-based switch seems particularly ill-suited for this, as that's really a per-session interface.
Quoted 8 lines
Quoted 3 lines
Is this something we expect people to just turn on briefly when they are experiencing a problem, or is this something that we expect to be turned on and left on for long periods of time?
If this won't add much overhead even per second, let's keep sending the metrics to ceph always and the mount option for this switch is not needed any more.
Note that I don't really _know_ that it won't be a problem, just that it doesn't sound too bad. I think we probably will want some mechanism to enable/disable this until we have some experience with it in the field.
Quoted 7 lines
And there is already a switch to enable/disable showing the metrics in the ceph side, if here add another switch per client, it will be also yucky for admins .
Let's make the update interval tunable and per second as default. Maybe we should make this as a global UI for all clients ?
If you want a global setting for the interval that would take effect on all ceph mounts, then maybe a "metric_send_interval" module parameter would be best. Make it an unsigned int, and allow the admin to set it to 0 to turn off stats transmission in the client.
We have a well-defined interface for setting module parameters on most distros (via /etc/modprobe.d/), so that would be better than monkeying around with debugfs here, IMO.
As to the default, it might be best to have this default to 0 initially. Once we have more experience with it we could make it default to 1 in a later release.
Signature · 1 line
Jeff Layton <[email protected]>
Re: [PATCH resend v5 08/11] ceph: periodically send perf metrics to MDS
Quoted 95 lines · Jeff Layton
Quoted 35 lines · Xiubo Li
Quoted 23 lines · Jeff Layton
Quoted 22 lines · Xiubo Li
Quoted 1 line · Jeff Layton
On Wed, 2020-01-29 at 03:27 -0500, [email protected] wrote:
[...]
Quoted 19 lines
Quoted 5 lines
fs/ceph/debugfs.c +5
-0+ +static int sending_metrics_get (void *data,u64 *val) +{ +struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; +struct ceph_mds_client *mdsc = fsc->mdsc;fs/ceph/debugfs.c static int mdsc_show(struct seq_file *s, void *p) · line 160
Quoted 10 lines
fs/ceph/debugfs.c +10
-0+ +mutex_lock (&mdsc->mutex); + *val = (u64 )mdsc->sending_metrics; +mutex_unlock (&mdsc->mutex); + +return 0 ; +} +DEFINE_SIMPLE_ATTRIBUTE (sending_metrics_fops, sending_metrics_get, + sending_metrics_set,"%llu\n" ); +I'd like to hear more about how we expect users to use this facility. This debugfs file doesn't seem consistent with the rest of the UI, and I imagine if the box reboots you'd have to (manually) re-enable it after mount, right? Maybe this should be a mount option instead?
Quoted 6 lines
Quoted 2 lines
A mount option means we must do the unmounting to disable it.
Technically, no. You could wire it up so that you could enable and disable it via -o remount. For example:
# mount -o remount,metrics=disabledYeah, this is cool.
Quoted 3 lines
Another option might be a module parameter if this is something that you really want to be global (and not per-mount or per-session).
Quoted 9 lines
Quoted 5 lines
Quoted 4 lines
I was thinking with the debugfs file we can do the debug or tuning even in the product setups at any time, usually this should be disabled since it will send it per second.
Meh, one frame per second doesn't seem like it'll add much overhead.
Okay.
Quoted 1 line
Also, why one update per second? Should that interval be tunable?
Per second just keep it the same with the fuse client.
Ok.
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
Or we could merge the "
sending_metric" to "metrics" UI, just writing "enable"/"disable" to enable/disable sending the metrics to ceph, and just like the "reset" does to clean the metrics.Then the "
/sys/kernel/debug/ceph/XXX.clientYYY/metrics" could be writable with:"reset" --> to clean and reset the metrics counters
"enable" --> enable sending metrics to ceph cluster
"disable" --> disable sending metrics to ceph cluster
Quoted 9 lines
Quoted 7 lines
Quoted 2 lines
Will this be better ?
I guess it's not clear to me how you intend for this to be used.
A debugfs switch means that this is being enabled and disabled on a per- session basis. Is the user supposed to turn this on for all, or just one session? How do they know?
Not for all, just per-superblock.
If it's per-superblock, then a debugfs-based switch seems particularly ill-suited for this, as that's really a per-session interface.
Quoted 7 lines
Quoted 3 lines
Is this something we expect people to just turn on briefly when they are experiencing a problem, or is this something that we expect to be turned on and left on for long periods of time?
If this won't add much overhead even per second, let's keep sending the metrics to ceph always and the mount option for this switch is not needed any more.
Note that I don't really _know_ that it won't be a problem, just that it doesn't sound too bad. I think we probably will want some mechanism to enable/disable this until we have some experience with it in the field.
Quoted 7 lines
And there is already a switch to enable/disable showing the metrics in the ceph side, if here add another switch per client, it will be also yucky for admins .
Let's make the update interval tunable and per second as default. Maybe we should make this as a global UI for all clients ?
If you want a global setting for the interval that would take effect on all ceph mounts, then maybe a "
metric_send_interval" module parameter would be best. Make it an unsigned int, and allow the admin to set it to 0 to turn off stats transmission in the client.
Quoted 7 lines
We have a well-defined interface for setting module parameters on most distros (via
/etc/modprobe.d/), so that would be better than monkeying around with debugfs here, IMO.As to the default, it might be best to have this default to 0 initially. Once we have more experience with it we could make it default to 1 in a later release.
Yeah, this makes sense.
Let's switch to the module parameter "metric_send_interval", at the same time this will also act as a switch.
0 means off, >0 will be the interval.
Thanks,
PATCH RESEND v5 9/11 ceph: add CEPH_DEFINE_RW_FUNC helper support
In reply to [email protected]From: Xiubo Li <[email protected]>
This will support the string store.
--- include/linux/ceph/debugfs.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+)
include/linux/ceph/debugfs.h +14 -0
@@ -18,6 +18,20 @@ static const struct file_operations name##_fops = { \ .release = single_release, \ }; +#define CEPH_DEFINE_RW_FUNC (name) \ +static int name##_open(struct inode *inode, struct file *file) \ +{ \ + return single_open(file, name##_show, inode->i_private); \ +} \ + \ +static const struct file_operations name##_fops = { \ + .open = name##_open, \ + .read = seq_read, \ + .write = name##_store, \ + .llseek = seq_lseek, \ + .release = single_release, \ +}; +/* debugfs.c */ extern void ceph_debugfs_init (void );extern void ceph_debugfs_cleanup (void );
Signature · 1 line
2.21.0
PATCH RESEND v5 10/11 ceph: add reset metrics support
In reply to [email protected]From: Xiubo Li <[email protected]>
This will reset the most metric counters, except the cap and dentry total numbers.
Sometimes we need to discard the old metrics and start to get new metrics.
URL: https://tracker.ceph.com/issues/43215
--- fs/ceph/debugfs.c | 53 +++++++++++++++++++++++++++++++++++++++++++++-- fs/ceph/super.h | 1 + 2 files changed, 52 insertions(+), 2 deletions(-)
fs/ceph/debugfs.c +51 -2
@@ -217,6 +217,55 @@ static int metric_show(struct seq_file *s, void *p)return 0 ; } +static ssize_t metric_store (struct file *file,const char __user *user_buf, +size_t count,loff_t *ppos) +{ +struct seq_file *s = file->private_data; +struct ceph_fs_client *fsc = s->private; +struct ceph_mds_client *mdsc = fsc->mdsc; +struct ceph_client_metric *metric = &mdsc->metric; +char buf[8 ]; +int i; + +if (copy_from_user (buf, user_buf,8 )) +return -EFAULT ; + +if (strcmp (buf,"reset" )) { +pr_err ("Invalid set value '%s', only 'reset' is valid\n" , buf); +return -EINVAL ; + } + +percpu_counter_set (&metric->d_lease_hit,0 ); +percpu_counter_set (&metric->d_lease_mis,0 ); + +percpu_counter_set (&metric->read_latency_sum,0 ); +percpu_counter_set (&metric->total_reads,0 ); + +percpu_counter_set (&metric->write_latency_sum,0 ); +percpu_counter_set (&metric->total_writes,0 ); + +percpu_counter_set (&metric->metadata_latency_sum,0 ); +percpu_counter_set (&metric->total_metadatas,0 ); + +mutex_lock (&mdsc->mutex); +for (i =0 ; i < mdsc->max_sessions; i++) { +struct ceph_mds_session *session; + + session =__ceph_lookup_mds_session (mdsc, i); +if (!session) +continue ; +percpu_counter_set (&session->i_caps_hit,0 ); +percpu_counter_set (&session->i_caps_mis,0 ); +ceph_put_mds_session (session); + } + +mutex_unlock (&mdsc->mutex); + +return 0 ; +} + +CEPH_DEFINE_RW_FUNC (metric) + static int caps_show_cb(struct inode *inode, struct ceph_cap *cap, void *p) {struct seq_file *s = p;
@@ -313,7 +362,6 @@ static int mds_sessions_show(struct seq_file *s, void *ptr) CEPH_DEFINE_SHOW_FUNC(mdsmap_show) CEPH_DEFINE_SHOW_FUNC(mdsc_show) -CEPH_DEFINE_SHOW_FUNC(metric_show) CEPH_DEFINE_SHOW_FUNC(caps_show) CEPH_DEFINE_SHOW_FUNC(mds_sessions_show)
@@ -349,6 +397,7 @@ void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc)debugfs_remove (fsc->debugfs_caps);debugfs_remove (fsc->debugfs_metric);debugfs_remove (fsc->debugfs_sending_metrics); +debugfs_remove (fsc->debugfs_reset_metrics);debugfs_remove (fsc->debugfs_mdsc); }
@@ -400,7 +449,7 @@ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc)0400 , fsc->client->debugfs_dir, fsc, - &metric_show_fops); + &metric_fops); fsc->debugfs_caps = debugfs_create_file("caps", 0400,
fs/ceph/super.h +1 -0
@@ -129,6 +129,7 @@ struct ceph_fs_client {struct dentry *debugfs_bdi;struct dentry *debugfs_mdsc, *debugfs_mdsmap;struct dentry *debugfs_sending_metrics; +struct dentry *debugfs_reset_metrics;struct dentry *debugfs_metric;struct dentry *debugfs_mds_sessions;#endif
Signature · 1 line
2.21.0
PATCH RESEND v5 11/11 ceph: send client provided metric flags in client metadata
In reply to [email protected]From: Xiubo Li <[email protected]>
Will send the metric flags to MDS, currently it supports the cap, dentry lease, read latency, write latency and metadata latency.
URL: https://tracker.ceph.com/issues/43435
--- fs/ceph/mds_client.c | 47 ++++++++++++++++++++++++++++++++++++++++++-- fs/ceph/metric.h | 14 +++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-)
fs/ceph/mds_client.c +45 -2
@@ -1096,6 +1096,41 @@ static void encode_supported_features(void **p, void *end) } } +static const unsigned char metric_bits[] =CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED ; +#define METRIC_BYTES (cnt) (DIV_ROUND_UP((size_t)metric_bits[cnt - 1] + 1, 64) * 8) +static void encode_metric_spec (void **p,void *end) +{ +static const size_t count =ARRAY_SIZE (metric_bits); + +/* header */ +BUG_ON (*p +2 > end); +ceph_encode_8 (p,1 );/* version */ +ceph_encode_8 (p,1 );/* compat */ + +if (count >0 ) { +size_t i; +size_t size =METRIC_BYTES (count); + +BUG_ON (*p +4 +4 + size > end); + +/* metric spec info length */ +ceph_encode_32 (p,4 + size); + +/* metric spec */ +ceph_encode_32 (p, size); +memset (*p,0 , size); +for (i =0 ; i < count; i++) + ((unsigned char *)(*p))[i /8 ] |=BIT (metric_bits[i] %8 ); + *p += size; + }else { +BUG_ON (*p +4 +4 > end); +/* metric spec info length */ +ceph_encode_32 (p,4 ); +/* metric spec */ +ceph_encode_32 (p,0 ); + } +} +/* * session message, specialization for CEPH_SESSION_REQUEST_OPEN * to include additional client metadata fields.
@@ -1135,6 +1170,13 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6 size =FEATURE_BYTES (count); extra_bytes +=4 + size; +/* metric spec */ + size =0 ; + count =ARRAY_SIZE (metric_bits); +if (count >0 ) + size =METRIC_BYTES (count); + extra_bytes +=2 +4 +4 + size; +/* Allocate the message */ msg =ceph_msg_new (CEPH_MSG_CLIENT_SESSION ,sizeof (*h) + extra_bytes,GFP_NOFS , false);
@@ -1153,9 +1195,9 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6* Serialize client metadata into waiting buffer space, using * the format that userspace expects for map<string, string> * -* ClientSession messages with metadata are v3 +* ClientSession messages with metadata are v4 */ - msg->hdr.version =cpu_to_le16 (3 ); + msg->hdr.version =cpu_to_le16 (4 ); msg->hdr.compat_version =cpu_to_le16 (1 );/* The write pointer, following the session_head structure */
@@ -1178,6 +1220,7 @@ static struct ceph_msg *create_session_open_msg(struct ceph_mds_client *mdsc, u6 }encode_supported_features (&p, end); +encode_metric_spec (&p, end); msg->front.iov_len = p - msg->front.iov_base; msg->hdr.front_len =cpu_to_le32 (msg->front.iov_len);
fs/ceph/metric.h +14 -0
@@ -14,6 +14,20 @@ enum ceph_metric_type { CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_DENTRY_LEASE, }; +/* +* This will always have the highest metric bit value +* as the last element of the array. +*/ +#define CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED { \ + CLIENT_METRIC_TYPE_CAP_INFO, \ + CLIENT_METRIC_TYPE_READ_LATENCY, \ + CLIENT_METRIC_TYPE_WRITE_LATENCY, \ + CLIENT_METRIC_TYPE_METADATA_LATENCY, \ + CLIENT_METRIC_TYPE_DENTRY_LEASE, \ + \ + CLIENT_METRIC_TYPE_MAX, \ +} +/* metric caps header */ struct ceph_metric_cap {__le32 type;/* ceph metric type */
Signature · 1 line
2.21.0