libceph: add osd op counter metric support
PATCH v2 1/1 libceph: add osd op counter metric support
From: Xiubo Li <[email protected]>
The logic is the same with osdc/Objecter.cc in ceph in user space.
URL: https://tracker.ceph.com/issues/48053
---
V2:
- remove other not used counter metrics
include/linux/ceph/osd_client.h | 9 ++++++ net/ceph/debugfs.c | 13 ++++++++ net/ceph/osd_client.c | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+)
include/linux/ceph/osd_client.h +9 -0
@@ -339,6 +339,13 @@ struct ceph_osd_backoff {struct ceph_hobject_id *end; }; +struct ceph_osd_metric { +struct percpu_counter op_ops; +struct percpu_counter op_rmw; +struct percpu_counter op_r; +struct percpu_counter op_w; +}; +#define CEPH_LINGER_ID_START 0xffff000000000000ULLstruct ceph_osd_client {
@@ -371,6 +378,8 @@ struct ceph_osd_client {struct ceph_msgpool msgpool_op;struct ceph_msgpool msgpool_op_reply; +struct ceph_osd_metric metric; +struct workqueue_struct *notify_wq;struct workqueue_struct *completion_wq; };
net/ceph/debugfs.c +13 -0
@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd)mutex_unlock (&osd->lock); } +static void dump_op_metric (struct seq_file *s,struct ceph_osd_client *osdc) +{ +struct ceph_osd_metric *m = &osdc->metric; + +seq_printf (s," op_ops: %lld\n" ,percpu_counter_sum (&m->op_ops)); +seq_printf (s," op_rmw: %lld\n" ,percpu_counter_sum (&m->op_rmw)); +seq_printf (s," op_r: %lld\n" ,percpu_counter_sum (&m->op_r)); +seq_printf (s," op_w: %lld\n" ,percpu_counter_sum (&m->op_w)); +} +static int osdc_show (struct seq_file *s,void *pp) {struct ceph_client *client = s->private;
@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp) }up_read (&osdc->lock); + +seq_puts (s,"OP METRIC:\n" ); +dump_op_metric (s, osdc);return 0 ; }
net/ceph/osd_client.c +56 -0
92 lines not on this page · raw message
Signature · 1 line
2.27.0
Quoted 103 lines · On Tue, 2020-11-10 at 19:01 +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
The logic is the same with osdc/Objecter.cc in ceph in user space.
URL: https://tracker.ceph.com/issues/48053
Signed-off-by Xiubo Li <[email protected]>---
V2:
- remove other not used counter metricsinclude/linux/ceph/osd_client.h | 9 ++++++ net/ceph/debugfs.c | 13 ++++++++ net/ceph/osd_client.c | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+)include/linux/ceph/osd_client.h +0
-0@@ -339,6 +339,13 @@ struct ceph_osd_backoff { struct ceph_hobject_id *end; };include/linux/ceph/osd_client.h +7
-0+struct ceph_osd_metric { +struct percpu_counter op_ops; +struct percpu_counter op_rmw; +struct percpu_counter op_r; +struct percpu_counter op_w; +}; +#define
CEPH_LINGER_ID_START0xffff000000000000ULLstruct
ceph_osd_client{include/linux/ceph/osd_client.h +0
-0@@ -371,6 +378,8 @@ struct ceph_osd_client { struct ceph_msgpool msgpool_op; struct ceph_msgpool msgpool_op_reply;include/linux/ceph/osd_client.h +2
-0+struct ceph_osd_metric metric; +struct
workqueue_struct*notify_wq;
structworkqueue_struct*completion_wq;
};net/ceph/debugfs.c +0
-0@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd) mutex_unlock(&osd->lock); }net/ceph/debugfs.c +10
-0+static void dump_op_metric (struct seq_file *s,struct ceph_osd_client *osdc) +{ +struct ceph_osd_metric *m = &osdc->metric; + +seq_printf (s," op_ops: %lld\n" ,percpu_counter_sum (&m->op_ops)); +seq_printf (s," op_rmw: %lld\n" ,percpu_counter_sum (&m->op_rmw)); +seq_printf (s," op_r: %lld\n" ,percpu_counter_sum (&m->op_r)); +seq_printf (s," op_w: %lld\n" ,percpu_counter_sum (&m->op_w)); +} +static int
osdc_show(struct seq_file *s, void *pp){
structceph_client*client =s->private;net/ceph/debugfs.c +0
-0@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp) }
up_read(&osdc->lock);net/ceph/debugfs.c +3
-0+ +seq_puts (s,"OP METRIC:\n" ); +dump_op_metric (s, osdc);return 0;
}net/ceph/osd_client.c +0
-0@@ -2424,6 +2424,21 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked) goto again; }
net/ceph/osd_client.c static void __submit_request(struct ceph_osd_request *req, bool wrlocked) · line 2435
Quoted 9 lines
net/ceph/osd_client.c +9
-0+static void osd_acount_op_metric (struct ceph_osd_request *req) +{ +struct ceph_osd_metric *m = &req->r_osdc->metric; + +percpu_counter_inc (&m->op_ops); + +if ((req->r_flags & (CEPH_OSD_FLAG_READ |CEPH_OSD_FLAG_READ )) + == (CEPH_OSD_FLAG_READ |CEPH_OSD_FLAG_READ )) +percpu_counter_inc (&m->op_rmw);
What's the point of or'ing the same flag together, and how is this different from the read one below? Was it supposed to be or'ed with CEPH_OSD_FLAG_WRITE ?
Quoted 106 lines
net/ceph/osd_client.c +6
-0+if (req->r_flags &CEPH_OSD_FLAG_READ ) +percpu_counter_inc (&m->op_r); +else if (req->r_flags &CEPH_OSD_FLAG_WRITE ) +percpu_counter_inc (&m->op_w); +} +static void
account_request(struct ceph_osd_request *req)
{
WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK));net/ceph/osd_client.c +0
-0@@ -2434,6 +2449,8 @@ static void account_request(struct ceph_osd_request *req)
req->r_start_stamp= jiffies;
req->r_start_latency=ktime_get();net/ceph/osd_client.c +2
-0+ +osd_acount_op_metric (req);}
static void
submit_request(struct ceph_osd_request *req, bool wrlocked)net/ceph/osd_client.c +0
-0@@ -5205,6 +5222,39 @@ void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc) up_write(&osdc->lock); }net/ceph/osd_client.c +33
-0+static void ceph_metric_destroy (struct ceph_osd_metric *m) +{ +percpu_counter_destroy (&m->op_ops); +percpu_counter_destroy (&m->op_rmw); +percpu_counter_destroy (&m->op_r); +percpu_counter_destroy (&m->op_w); +} + +static int ceph_metric_init (struct ceph_osd_metric *m) +{ +int ret; + +memset (m,0 ,sizeof (*m)); + + ret =percpu_counter_init (&m->op_ops,0 ,GFP_NOIO ); +if (ret) +return ret; + ret =percpu_counter_init (&m->op_rmw,0 ,GFP_NOIO ); +if (ret) + goto err; + ret =percpu_counter_init (&m->op_r,0 ,GFP_NOIO ); +if (ret) + goto err; + ret =percpu_counter_init (&m->op_w,0 ,GFP_NOIO ); +if (ret) + goto err; +return 0 ; + +err: +ceph_metric_destroy (m); +return ret; +} +/*
* init, shutdown
*/net/ceph/osd_client.c +0
-0@@ -5257,6 +5307,9 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) if (!osdc->completion_wq) goto out_notify_wq;net/ceph/osd_client.c +3
-0+if (ceph_metric_init (&osdc->metric) <0 ) + goto out_completion_wq; +
schedule_delayed_work(&osdc->timeout_work, osdc->client->options->osd_keepalive_timeout); schedule_delayed_work(&osdc->osds_timeout_work,net/ceph/osd_client.c +0
-0@@ -5264,6 +5317,8 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)return 0;
net/ceph/osd_client.c +2
-0+out_completion_wq: +destroy_workqueue (osdc->completion_wq);out_notify_wq:
destroy_workqueue(osdc->notify_wq);
out_msgpool_reply:net/ceph/osd_client.c +0
-0@@ -5302,6 +5357,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc) WARN_ON(atomic_read(&osdc->num_requests)); WARN_ON(atomic_read(&osdc->num_homeless));net/ceph/osd_client.c +1
-0+ceph_metric_destroy (&osdc->metric);
ceph_osdmap_destroy(osdc->osdmap);
mempool_destroy(osdc->req_mempool);
ceph_msgpool_destroy(&osdc->msgpool_op);
Signature · 1 line
Jeff Layton <[email protected]>
Quoted 104 lines · Jeff Layton
Quoted 103 lines · On Tue, 2020-11-10 at 19:01 +0800, [email protected] wrote:
From: Xiubo Li <[email protected]>
The logic is the same with osdc/Objecter.cc in ceph in user space.
URL: https://tracker.ceph.com/issues/48053
Signed-off-by Xiubo Li <[email protected]>---
V2:
- remove other not used counter metricsinclude/linux/ceph/osd_client.h | 9 ++++++ net/ceph/debugfs.c | 13 ++++++++ net/ceph/osd_client.c | 56 +++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+)include/linux/ceph/osd_client.h +7
-0@@ -339,6 +339,13 @@ struct ceph_osd_backoff { struct ceph_hobject_id *end; }; +struct ceph_osd_metric { +struct percpu_counter op_ops; +struct percpu_counter op_rmw; +struct percpu_counter op_r; +struct percpu_counter op_w; +}; +#define
CEPH_LINGER_ID_START0xffff000000000000ULLstruct
ceph_osd_client{include/linux/ceph/osd_client.h +2
-0@@ -371,6 +378,8 @@ struct ceph_osd_client { struct ceph_msgpool msgpool_op; struct ceph_msgpool msgpool_op_reply; +struct ceph_osd_metric metric; +struct
workqueue_struct*notify_wq;
structworkqueue_struct*completion_wq;
};net/ceph/debugfs.c +10
-0@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd) mutex_unlock(&osd->lock); } +static void dump_op_metric (struct seq_file *s,struct ceph_osd_client *osdc) +{ +struct ceph_osd_metric *m = &osdc->metric; + +seq_printf (s," op_ops: %lld\n" ,percpu_counter_sum (&m->op_ops)); +seq_printf (s," op_rmw: %lld\n" ,percpu_counter_sum (&m->op_rmw)); +seq_printf (s," op_r: %lld\n" ,percpu_counter_sum (&m->op_r)); +seq_printf (s," op_w: %lld\n" ,percpu_counter_sum (&m->op_w)); +} +static int
osdc_show(struct seq_file *s, void *pp){
structceph_client*client =s->private;net/ceph/debugfs.c +3
-0@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp) } up_read(&osdc->lock); + +seq_puts (s,"OP METRIC:\n" ); +dump_op_metric (s, osdc);return 0;
}net/ceph/osd_client.c +0
-0@@ -2424,6 +2424,21 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked) goto again; }
Quoted 12 lines
net/ceph/osd_client.c static void __submit_request(struct ceph_osd_request *req, bool wrlocked) · line 2435
Quoted 9 lines
net/ceph/osd_client.c +9
-0+static void osd_acount_op_metric (struct ceph_osd_request *req) +{ +struct ceph_osd_metric *m = &req->r_osdc->metric; + +percpu_counter_inc (&m->op_ops); + +if ((req->r_flags & (CEPH_OSD_FLAG_READ |CEPH_OSD_FLAG_READ )) + == (CEPH_OSD_FLAG_READ |CEPH_OSD_FLAG_READ )) +percpu_counter_inc (&m->op_rmw);What's the point of or'ing the same flag together, and how is this different from the read one below? Was it supposed to be or'ed with
CEPH_OSD_FLAG_WRITE?
Yeah, It should be READ | WRITE instead.
Will fix it.
Thanks
Quoted 107 lines
Quoted 107 lines
net/ceph/osd_client.c +6
-0+if (req->r_flags &CEPH_OSD_FLAG_READ ) +percpu_counter_inc (&m->op_r); +else if (req->r_flags &CEPH_OSD_FLAG_WRITE ) +percpu_counter_inc (&m->op_w); +} +static void
account_request(struct ceph_osd_request *req)
{
WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK));net/ceph/osd_client.c +2
-0@@ -2434,6 +2449,8 @@ static void account_request(struct ceph_osd_request *req) req->r_start_stamp = jiffies; req->r_start_latency = ktime_get(); + +osd_acount_op_metric (req);}
static void
submit_request(struct ceph_osd_request *req, bool wrlocked)net/ceph/osd_client.c +33
-0@@ -5205,6 +5222,39 @@ void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc) up_write(&osdc->lock); } +static void ceph_metric_destroy (struct ceph_osd_metric *m) +{ +percpu_counter_destroy (&m->op_ops); +percpu_counter_destroy (&m->op_rmw); +percpu_counter_destroy (&m->op_r); +percpu_counter_destroy (&m->op_w); +} + +static int ceph_metric_init (struct ceph_osd_metric *m) +{ +int ret; + +memset (m,0 ,sizeof (*m)); + + ret =percpu_counter_init (&m->op_ops,0 ,GFP_NOIO ); +if (ret) +return ret; + ret =percpu_counter_init (&m->op_rmw,0 ,GFP_NOIO ); +if (ret) + goto err; + ret =percpu_counter_init (&m->op_r,0 ,GFP_NOIO ); +if (ret) + goto err; + ret =percpu_counter_init (&m->op_w,0 ,GFP_NOIO ); +if (ret) + goto err; +return 0 ; + +err: +ceph_metric_destroy (m); +return ret; +} +/*
* init, shutdown
*/net/ceph/osd_client.c +3
-0@@ -5257,6 +5307,9 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) if (!osdc->completion_wq) goto out_notify_wq; +if (ceph_metric_init (&osdc->metric) <0 ) + goto out_completion_wq; +
schedule_delayed_work(&osdc->timeout_work, osdc->client->options->osd_keepalive_timeout); schedule_delayed_work(&osdc->osds_timeout_work,net/ceph/osd_client.c +3
-0@@ -5264,6 +5317,8 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client) return 0; +out_completion_wq: +destroy_workqueue (osdc->completion_wq); out_notify_wq: destroy_workqueue(osdc->notify_wq); out_msgpool_reply:@@ -5302,6 +5357,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc) WARN_ON(atomic_read(&osdc->num_requests)); WARN_ON(atomic_read(&osdc->num_homeless)); +ceph_metric_destroy (&osdc->metric);
ceph_osdmap_destroy(osdc->osdmap);
mempool_destroy(osdc->req_mempool);
ceph_msgpool_destroy(&osdc->msgpool_op);