Skip to contentSkip to conversation

libceph: add osd op counter metric support

PATCH Superseded [email protected] 2 participants 3 messages Nov 10, 2020 Ceph common code (libceph)

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 libceph: add osd op counter metric support

From: Xiubo Li <[email protected]>

The logic is the same with osdc/Objecter.cc in ceph in user space.

URL: https://tracker.ceph.com/issues/48053

---

V2:
- remove other not used counter metrics

 include/linux/ceph/osd_client.h |  9 ++++++
 net/ceph/debugfs.c              | 13 ++++++++
 net/ceph/osd_client.c           | 56 +++++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)
include/linux/ceph/osd_client.h +9 -0
index 83fa08a06507..24301513b186 100644
@@ -339,6 +339,13 @@ struct ceph_osd_backoff {
 	struct ceph_hobject_id *end;
 };
 
+struct ceph_osd_metric {
+	struct percpu_counter op_ops;
+	struct percpu_counter op_rmw;
+	struct percpu_counter op_r;
+	struct percpu_counter op_w;
+};
+
 #define CEPH_LINGER_ID_START	0xffff000000000000ULL
 
 struct ceph_osd_client {
@@ -371,6 +378,8 @@ struct ceph_osd_client {
 	struct ceph_msgpool	msgpool_op;
 	struct ceph_msgpool	msgpool_op_reply;
 
+	struct ceph_osd_metric  metric;
+
 	struct workqueue_struct	*notify_wq;
 	struct workqueue_struct	*completion_wq;
 };
net/ceph/debugfs.c +13 -0
index 2110439f8a24..af90019386ab 100644
@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd)
 	mutex_unlock(&osd->lock);
 }
 
+static void dump_op_metric(struct seq_file *s, struct ceph_osd_client *osdc)
+{
+	struct ceph_osd_metric *m = &osdc->metric;
+
+	seq_printf(s, "  op_ops: %lld\n", percpu_counter_sum(&m->op_ops));
+	seq_printf(s, "  op_rmw: %lld\n", percpu_counter_sum(&m->op_rmw));
+	seq_printf(s, "  op_r:   %lld\n", percpu_counter_sum(&m->op_r));
+	seq_printf(s, "  op_w:   %lld\n", percpu_counter_sum(&m->op_w));
+}
+
 static int osdc_show(struct seq_file *s, void *pp)
 {
 	struct ceph_client *client = s->private;
@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp)
 	}
 
 	up_read(&osdc->lock);
+
+	seq_puts(s, "OP METRIC:\n");
+	dump_op_metric(s, osdc);
 	return 0;
 }
 
net/ceph/osd_client.c +56 -0

92 lines not on this page · raw message

Signature · 1 line
2.27.0
#1 lore raw
Quoted 103 lines · On Tue, 2020-11-10 at 19:01 +0800, [email protected] wrote:

From: Xiubo Li <[email protected]>

The logic is the same with osdc/Objecter.cc in ceph in user space.

URL: https://tracker.ceph.com/issues/48053

---

V2:
- remove other not used counter metrics

 include/linux/ceph/osd_client.h |  9 ++++++
 net/ceph/debugfs.c              | 13 ++++++++
 net/ceph/osd_client.c           | 56 +++++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)
include/linux/ceph/osd_client.h +0 -0
index 83fa08a06507..24301513b186 100644
@@ -339,6 +339,13 @@ struct ceph_osd_backoff {
  	struct ceph_hobject_id *end;
  };

include/linux/ceph/osd_client.h +7 -0
+struct ceph_osd_metric {
+	struct percpu_counter op_ops;
+	struct percpu_counter op_rmw;
+	struct percpu_counter op_r;
+	struct percpu_counter op_w;
+};
+

 #define CEPH_LINGER_ID_START 0xffff000000000000ULL

 struct ceph_osd_client {

include/linux/ceph/osd_client.h +0 -0
@@ -371,6 +378,8 @@ struct ceph_osd_client {
  	struct ceph_msgpool	msgpool_op;
  	struct ceph_msgpool	msgpool_op_reply;

include/linux/ceph/osd_client.h +2 -0
+	struct ceph_osd_metric  metric;
+

  struct workqueue_struct *notify_wq;
  struct workqueue_struct *completion_wq;
 };

net/ceph/debugfs.c +0 -0
index 2110439f8a24..af90019386ab 100644
@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd)
  	mutex_unlock(&osd->lock);
  }

net/ceph/debugfs.c +10 -0
+static void dump_op_metric(struct seq_file *s, struct ceph_osd_client *osdc)
+{
+	struct ceph_osd_metric *m = &osdc->metric;
+
+	seq_printf(s, "  op_ops: %lld\n", percpu_counter_sum(&m->op_ops));
+	seq_printf(s, "  op_rmw: %lld\n", percpu_counter_sum(&m->op_rmw));
+	seq_printf(s, "  op_r:   %lld\n", percpu_counter_sum(&m->op_r));
+	seq_printf(s, "  op_w:   %lld\n", percpu_counter_sum(&m->op_w));
+}
+

 static int osdc_show(struct seq_file *s, void *pp) {
  struct ceph_client *client = s->private;

net/ceph/debugfs.c +0 -0
@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp)
  	}

  up_read(&osdc->lock);

net/ceph/debugfs.c +3 -0
+
+	seq_puts(s, "OP METRIC:\n");
+	dump_op_metric(s, osdc);

  return 0;
 }

net/ceph/osd_client.c +0 -0
index 7901ab6c79fd..66774b2bc584 100644
@@ -2424,6 +2424,21 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
  	goto again;
  }

net/ceph/osd_client.c static void __submit_request(struct ceph_osd_request *req, bool wrlocked) · line 2435

Quoted 9 lines
net/ceph/osd_client.c +9 -0
+static void osd_acount_op_metric(struct ceph_osd_request *req)
+{
+	struct ceph_osd_metric *m = &req->r_osdc->metric;
+
+	percpu_counter_inc(&m->op_ops);
+
+	if ((req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_READ))
+	    == (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_READ))
+		percpu_counter_inc(&m->op_rmw);

What's the point of or'ing the same flag together, and how is this different from the read one below? Was it supposed to be or'ed with CEPH_OSD_FLAG_WRITE ?

Quoted 106 lines
net/ceph/osd_client.c +6 -0
+	if (req->r_flags & CEPH_OSD_FLAG_READ)
+		percpu_counter_inc(&m->op_r);
+	else if (req->r_flags & CEPH_OSD_FLAG_WRITE)
+		percpu_counter_inc(&m->op_w);
+}
+

 static void account_request(struct ceph_osd_request *req)
 {
  WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK));

net/ceph/osd_client.c +0 -0
@@ -2434,6 +2449,8 @@ static void account_request(struct ceph_osd_request *req)

  req->r_start_stamp = jiffies;
  req->r_start_latency = ktime_get();

net/ceph/osd_client.c +2 -0
+
+	osd_acount_op_metric(req);

 }

 static void submit_request(struct ceph_osd_request *req, bool wrlocked)

net/ceph/osd_client.c +0 -0
@@ -5205,6 +5222,39 @@ void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc)
  	up_write(&osdc->lock);
  }

net/ceph/osd_client.c +33 -0
+static void ceph_metric_destroy(struct ceph_osd_metric *m)
+{
+	percpu_counter_destroy(&m->op_ops);
+	percpu_counter_destroy(&m->op_rmw);
+	percpu_counter_destroy(&m->op_r);
+	percpu_counter_destroy(&m->op_w);
+}
+
+static int ceph_metric_init(struct ceph_osd_metric *m)
+{
+	int ret;
+
+	memset(m, 0, sizeof(*m));
+
+	ret = percpu_counter_init(&m->op_ops, 0, GFP_NOIO);
+	if (ret)
+		return ret;
+	ret = percpu_counter_init(&m->op_rmw, 0, GFP_NOIO);
+	if (ret)
+		goto err;
+	ret = percpu_counter_init(&m->op_r, 0, GFP_NOIO);
+	if (ret)
+		goto err;
+	ret = percpu_counter_init(&m->op_w, 0, GFP_NOIO);
+	if (ret)
+		goto err;
+	return 0;
+
+err:
+	ceph_metric_destroy(m);
+	return ret;
+}
+

 /*
  * init, shutdown
  */

net/ceph/osd_client.c +0 -0
@@ -5257,6 +5307,9 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
  	if (!osdc->completion_wq)
  		goto out_notify_wq;

net/ceph/osd_client.c +3 -0
+	if (ceph_metric_init(&osdc->metric) < 0)
+		goto out_completion_wq;
+

  schedule_delayed_work(&osdc->timeout_work, osdc->client->options->osd_keepalive_timeout); schedule_delayed_work(&osdc->osds_timeout_work,

net/ceph/osd_client.c +0 -0
@@ -5264,6 +5317,8 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)

  return 0;

net/ceph/osd_client.c +2 -0
+out_completion_wq:
+	destroy_workqueue(osdc->completion_wq);

 out_notify_wq:
  destroy_workqueue(osdc->notify_wq);
 out_msgpool_reply:

net/ceph/osd_client.c +0 -0
@@ -5302,6 +5357,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc)
  	WARN_ON(atomic_read(&osdc->num_requests));
  	WARN_ON(atomic_read(&osdc->num_homeless));

net/ceph/osd_client.c +1 -0
+	ceph_metric_destroy(&osdc->metric);

  ceph_osdmap_destroy(osdc->osdmap);
  mempool_destroy(osdc->req_mempool);
  ceph_msgpool_destroy(&osdc->msgpool_op);

Signature · 1 line
Jeff Layton <[email protected]>
author
#2 lore raw
Quoted 104 lines · Jeff Layton
Quoted 103 lines · On Tue, 2020-11-10 at 19:01 +0800, [email protected] wrote:

From: Xiubo Li <[email protected]>

The logic is the same with osdc/Objecter.cc in ceph in user space.

URL: https://tracker.ceph.com/issues/48053

---

V2:
- remove other not used counter metrics

  include/linux/ceph/osd_client.h |  9 ++++++
  net/ceph/debugfs.c              | 13 ++++++++
  net/ceph/osd_client.c           | 56 +++++++++++++++++++++++++++++++++
  3 files changed, 78 insertions(+)
include/linux/ceph/osd_client.h +7 -0
index 83fa08a06507..24301513b186 100644
@@ -339,6 +339,13 @@ struct ceph_osd_backoff {
  	struct ceph_hobject_id *end;
  };
 
 
 
 
+struct ceph_osd_metric {
+	struct percpu_counter op_ops;
+	struct percpu_counter op_rmw;
+	struct percpu_counter op_r;
+	struct percpu_counter op_w;
+};
+

 #define CEPH_LINGER_ID_START 0xffff000000000000ULL

 struct ceph_osd_client {

include/linux/ceph/osd_client.h +2 -0
@@ -371,6 +378,8 @@ struct ceph_osd_client {
  	struct ceph_msgpool	msgpool_op;
  	struct ceph_msgpool	msgpool_op_reply;
 
 
 
 
+	struct ceph_osd_metric  metric;
+

  struct workqueue_struct *notify_wq;
  struct workqueue_struct *completion_wq;
 };

net/ceph/debugfs.c +10 -0
index 2110439f8a24..af90019386ab 100644
@@ -339,6 +339,16 @@ static void dump_backoffs(struct seq_file *s, struct ceph_osd *osd)
  	mutex_unlock(&osd->lock);
  }
 
 
 
 
+static void dump_op_metric(struct seq_file *s, struct ceph_osd_client *osdc)
+{
+	struct ceph_osd_metric *m = &osdc->metric;
+
+	seq_printf(s, "  op_ops: %lld\n", percpu_counter_sum(&m->op_ops));
+	seq_printf(s, "  op_rmw: %lld\n", percpu_counter_sum(&m->op_rmw));
+	seq_printf(s, "  op_r:   %lld\n", percpu_counter_sum(&m->op_r));
+	seq_printf(s, "  op_w:   %lld\n", percpu_counter_sum(&m->op_w));
+}
+

 static int osdc_show(struct seq_file *s, void *pp) {
  struct ceph_client *client = s->private;

net/ceph/debugfs.c +3 -0
@@ -372,6 +382,9 @@ static int osdc_show(struct seq_file *s, void *pp)
  	}
 
 
 
 
  	up_read(&osdc->lock);
+
+	seq_puts(s, "OP METRIC:\n");
+	dump_op_metric(s, osdc);

  return 0;
 }

net/ceph/osd_client.c +0 -0
index 7901ab6c79fd..66774b2bc584 100644
@@ -2424,6 +2424,21 @@ static void __submit_request(struct ceph_osd_request *req, bool wrlocked)
  	goto again;
  }
 
 
 
 
Quoted 12 lines

net/ceph/osd_client.c static void __submit_request(struct ceph_osd_request *req, bool wrlocked) · line 2435

Quoted 9 lines
net/ceph/osd_client.c +9 -0
+static void osd_acount_op_metric(struct ceph_osd_request *req)
+{
+	struct ceph_osd_metric *m = &req->r_osdc->metric;
+
+	percpu_counter_inc(&m->op_ops);
+
+	if ((req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_READ))
+	    == (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_READ))
+		percpu_counter_inc(&m->op_rmw);

What's the point of or'ing the same flag together, and how is this different from the read one below? Was it supposed to be or'ed with CEPH_OSD_FLAG_WRITE ?

Yeah, It should be READ | WRITE instead.

Will fix it.

Thanks

Quoted 107 lines
Quoted 107 lines
net/ceph/osd_client.c +6 -0
+	if (req->r_flags & CEPH_OSD_FLAG_READ)
+		percpu_counter_inc(&m->op_r);
+	else if (req->r_flags & CEPH_OSD_FLAG_WRITE)
+		percpu_counter_inc(&m->op_w);
+}
+

 static void account_request(struct ceph_osd_request *req)
 {
  WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK));

net/ceph/osd_client.c +2 -0
@@ -2434,6 +2449,8 @@ static void account_request(struct ceph_osd_request *req)
 
 
 
 
  	req->r_start_stamp = jiffies;
  	req->r_start_latency = ktime_get();
+
+	osd_acount_op_metric(req);

 }

 static void submit_request(struct ceph_osd_request *req, bool wrlocked)

net/ceph/osd_client.c +33 -0
@@ -5205,6 +5222,39 @@ void ceph_osdc_reopen_osds(struct ceph_osd_client *osdc)
  	up_write(&osdc->lock);
  }
 
 
 
 
+static void ceph_metric_destroy(struct ceph_osd_metric *m)
+{
+	percpu_counter_destroy(&m->op_ops);
+	percpu_counter_destroy(&m->op_rmw);
+	percpu_counter_destroy(&m->op_r);
+	percpu_counter_destroy(&m->op_w);
+}
+
+static int ceph_metric_init(struct ceph_osd_metric *m)
+{
+	int ret;
+
+	memset(m, 0, sizeof(*m));
+
+	ret = percpu_counter_init(&m->op_ops, 0, GFP_NOIO);
+	if (ret)
+		return ret;
+	ret = percpu_counter_init(&m->op_rmw, 0, GFP_NOIO);
+	if (ret)
+		goto err;
+	ret = percpu_counter_init(&m->op_r, 0, GFP_NOIO);
+	if (ret)
+		goto err;
+	ret = percpu_counter_init(&m->op_w, 0, GFP_NOIO);
+	if (ret)
+		goto err;
+	return 0;
+
+err:
+	ceph_metric_destroy(m);
+	return ret;
+}
+

 /*
  * init, shutdown
  */

net/ceph/osd_client.c +3 -0
@@ -5257,6 +5307,9 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
  	if (!osdc->completion_wq)
  		goto out_notify_wq;
 
 
 
 
+	if (ceph_metric_init(&osdc->metric) < 0)
+		goto out_completion_wq;
+

  schedule_delayed_work(&osdc->timeout_work, osdc->client->options->osd_keepalive_timeout); schedule_delayed_work(&osdc->osds_timeout_work,

net/ceph/osd_client.c +3 -0
@@ -5264,6 +5317,8 @@ int ceph_osdc_init(struct ceph_osd_client *osdc, struct ceph_client *client)
 
 
 
 
  	return 0;
 
 
 
 
+out_completion_wq:
+	destroy_workqueue(osdc->completion_wq);
  out_notify_wq:
  	destroy_workqueue(osdc->notify_wq);
  out_msgpool_reply:
@@ -5302,6 +5357,7 @@ void ceph_osdc_stop(struct ceph_osd_client *osdc)
  	WARN_ON(atomic_read(&osdc->num_requests));
  	WARN_ON(atomic_read(&osdc->num_homeless));
 
 
 
 
+	ceph_metric_destroy(&osdc->metric);

  ceph_osdmap_destroy(osdc->osdmap);
  mempool_destroy(osdc->req_mempool);
  ceph_msgpool_destroy(&osdc->msgpool_op);