Skip to contentSkip to conversation

libceph: move away from global osd_req_flags

PATCH In review Ilya Dryomov 3 participants 5 messages Jun 8, 2020 – Jun 16, 2020

Subsystems (3)
lore
Share & tools
lore

Ilya Dryomov

author maintainer
#0 lore raw

PATCH 1/1 libceph: move away from global osd_req_flags

osd_req_flags is overly general and doesn't suit its only user (read_from_replica option) well:

- applying osd_req_flags in account_request() affects all OSD requests, including linger (i.e. watch and notify). However, linger requests should always go to the primary even though some of them are reads (e.g. notify has side effects but it is a read because it doesn't result in mutation on the OSDs).

- calls to class methods that are reads are allowed to go to the replica, but most such calls issued for "rbd map" and/or exclusive lock transitions are requested to be resent to the primary via EAGAIN, doubling the latency.

Get rid of global osd_req_flags and set read_from_replica flag only on specific OSD requests instead.

Fixes 8ad44d5e0d1e ("libceph: read_from_replica option")
---
 drivers/block/rbd.c          |  4 +++-
 include/linux/ceph/libceph.h |  4 ++--
 net/ceph/ceph_common.c       | 14 ++++++--------
 net/ceph/osd_client.c        |  3 +--
 4 files changed, 12 insertions(+), 13 deletions(-)
drivers/block/rbd.c +3 -1
index e02089d550a4..b2bb2f10562a 100644
@@ -1451,8 +1451,10 @@ static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
 static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
 {
 	struct rbd_obj_request *obj_request = osd_req->r_priv;
+	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
+	struct ceph_options *opt = rbd_dev->rbd_client->client->options;
 
-	osd_req->r_flags = CEPH_OSD_FLAG_READ;
+	osd_req->r_flags = CEPH_OSD_FLAG_READ | opt->read_from_replica;
 	osd_req->r_snapid = obj_request->img_request->snap_id;
 }
 
include/linux/ceph/libceph.h +2 -2
index 2247e71beb83..e5ed1c541e7f 100644
@@ -52,8 +52,7 @@ struct ceph_options {
 	unsigned long osd_idle_ttl;		/* jiffies */
 	unsigned long osd_keepalive_timeout;	/* jiffies */
 	unsigned long osd_request_timeout;	/* jiffies */
-
-	u32 osd_req_flags;  /* CEPH_OSD_FLAG_*, applied to each OSD request */
+	u32 read_from_replica;  /* CEPH_OSD_FLAG_BALANCE/LOCALIZE_READS */
 
 	/*
 	 * any type that can't be simply compared or doesn't need
@@ -76,6 +75,7 @@ struct ceph_options {
 #define CEPH_OSD_KEEPALIVE_DEFAULT	msecs_to_jiffies(5 * 1000)
 #define CEPH_OSD_IDLE_TTL_DEFAULT	msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
+#define CEPH_READ_FROM_REPLICA_DEFAULT	0  /* read from primary */
 
 #define CEPH_MONC_HUNT_INTERVAL		msecs_to_jiffies(3 * 1000)
 #define CEPH_MONC_PING_INTERVAL		msecs_to_jiffies(10 * 1000)
net/ceph/ceph_common.c +6 -8
index 9bab3e9a039b..b7705e91ae9a 100644
@@ -333,6 +333,7 @@ struct ceph_options *ceph_alloc_options(void)
 	opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
 	opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
 	opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
+	opt->read_from_replica = CEPH_READ_FROM_REPLICA_DEFAULT;
 	return opt;
 }
 EXPORT_SYMBOL(ceph_alloc_options);
@@ -491,16 +492,13 @@ int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt,
 	case Opt_read_from_replica:
 		switch (result.uint_32) {
 		case Opt_read_from_replica_no:
-			opt->osd_req_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS |
-						CEPH_OSD_FLAG_LOCALIZE_READS);
+			opt->read_from_replica = 0;
 			break;
 		case Opt_read_from_replica_balance:
-			opt->osd_req_flags |= CEPH_OSD_FLAG_BALANCE_READS;
-			opt->osd_req_flags &= ~CEPH_OSD_FLAG_LOCALIZE_READS;
+			opt->read_from_replica = CEPH_OSD_FLAG_BALANCE_READS;
 			break;
 		case Opt_read_from_replica_localize:
-			opt->osd_req_flags |= CEPH_OSD_FLAG_LOCALIZE_READS;
-			opt->osd_req_flags &= ~CEPH_OSD_FLAG_BALANCE_READS;
+			opt->read_from_replica = CEPH_OSD_FLAG_LOCALIZE_READS;
 			break;
 		default:
 			BUG();
@@ -614,9 +612,9 @@ int ceph_print_client_options(struct seq_file *m, struct ceph_client *client,
 		}
 		seq_putc(m, ',');
 	}
-	if (opt->osd_req_flags & CEPH_OSD_FLAG_BALANCE_READS) {
+	if (opt->read_from_replica == CEPH_OSD_FLAG_BALANCE_READS) {
 		seq_puts(m, "read_from_replica=balance,");
-	} else if (opt->osd_req_flags & CEPH_OSD_FLAG_LOCALIZE_READS) {
+	} else if (opt->read_from_replica == CEPH_OSD_FLAG_LOCALIZE_READS) {
 		seq_puts(m, "read_from_replica=localize,");
 	}
 
net/ceph/osd_client.c +1 -2
index 4fea3c33af2a..b151fe163821 100644
@@ -1117,10 +1117,10 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,
 				       truncate_size, truncate_seq);
 	}
 
-	req->r_flags = flags;
 	req->r_base_oloc.pool = layout->pool_id;
 	req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns);
 	ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum);
+	req->r_flags = flags | osdc->client->options->read_from_replica;
 
 	req->r_snapid = vino.snap;
 	if (flags & CEPH_OSD_FLAG_WRITE)
@@ -2437,7 +2437,6 @@ static void account_request(struct ceph_osd_request *req)
 	WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE)));
 
 	req->r_flags |= CEPH_OSD_FLAG_ONDISK;
-	req->r_flags |= osdc->client->options->osd_req_flags;
 	atomic_inc(&osdc->num_requests);
 
 	req->r_start_stamp = jiffies;
Signature · 1 line
2.19.2
#1 lore raw
Quoted 14 lines · Ilya Dryomov

osd_req_flags is overly general and doesn't suit its only user (read_from_replica option) well:

- applying osd_req_flags in account_request() affects all OSD requests, including linger (i.e. watch and notify). However, linger requests should always go to the primary even though some of them are reads (e.g. notify has side effects but it is a read because it doesn't result in mutation on the OSDs).

- calls to class methods that are reads are allowed to go to the replica, but most such calls issued for "rbd map" and/or exclusive lock transitions are requested to be resent to the primary via EAGAIN, doubling the latency.

Quoted 4 lines

Get rid of global osd_req_flags and set read_from_replica flag only on specific OSD requests instead.

Fixes 8ad44d5e0d1e ("libceph: read_from_replica option")

Since this isn't yet in mainline, can't you simply merge these two patches?

Quoted 59 lines
---
 drivers/block/rbd.c          |  4 +++-
 include/linux/ceph/libceph.h |  4 ++--
 net/ceph/ceph_common.c       | 14 ++++++--------
 net/ceph/osd_client.c        |  3 +--
 4 files changed, 12 insertions(+), 13 deletions(-)
drivers/block/rbd.c +3 -1
index e02089d550a4..b2bb2f10562a 100644
@@ -1451,8 +1451,10 @@ static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
 static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
 {
 	struct rbd_obj_request *obj_request = osd_req->r_priv;
+	struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
+	struct ceph_options *opt = rbd_dev->rbd_client->client->options;
 
-	osd_req->r_flags = CEPH_OSD_FLAG_READ;
+	osd_req->r_flags = CEPH_OSD_FLAG_READ | opt->read_from_replica;
 	osd_req->r_snapid = obj_request->img_request->snap_id;
 }
 
include/linux/ceph/libceph.h +2 -2
index 2247e71beb83..e5ed1c541e7f 100644
@@ -52,8 +52,7 @@ struct ceph_options {
 	unsigned long osd_idle_ttl;		/* jiffies */
 	unsigned long osd_keepalive_timeout;	/* jiffies */
 	unsigned long osd_request_timeout;	/* jiffies */
-
-	u32 osd_req_flags;  /* CEPH_OSD_FLAG_*, applied to each OSD request */
+	u32 read_from_replica;  /* CEPH_OSD_FLAG_BALANCE/LOCALIZE_READS */
 
 	/*
 	 * any type that can't be simply compared or doesn't need
@@ -76,6 +75,7 @@ struct ceph_options {
 #define CEPH_OSD_KEEPALIVE_DEFAULT	msecs_to_jiffies(5 * 1000)
 #define CEPH_OSD_IDLE_TTL_DEFAULT	msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
+#define CEPH_READ_FROM_REPLICA_DEFAULT	0  /* read from primary */
 
 #define CEPH_MONC_HUNT_INTERVAL		msecs_to_jiffies(3 * 1000)
 #define CEPH_MONC_PING_INTERVAL		msecs_to_jiffies(10 * 1000)
net/ceph/ceph_common.c +1 -0
index 9bab3e9a039b..b7705e91ae9a 100644
@@ -333,6 +333,7 @@ struct ceph_options *ceph_alloc_options(void)
 	opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
 	opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
 	opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
+	opt->read_from_replica = CEPH_READ_FROM_REPLICA_DEFAULT;
 	return opt;
 }
 EXPORT_SYMBOL(ceph_alloc_options);

net/ceph/ceph_common.c int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, · line 495

Quoted 6 lines
net/ceph/ceph_common.c +1 -2
@@ -491,16 +492,13 @@ int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt,
 	case Opt_read_from_replica:
 		switch (result.uint_32) {
 		case Opt_read_from_replica_no:
-			opt->osd_req_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS |
-						CEPH_OSD_FLAG_LOCALIZE_READS);
+			opt->read_from_replica = 0;

CEPH_READ_FROM_REPLICA_DEFAULT?

Quoted 42 lines
net/ceph/ceph_common.c +4 -6
 			break;
 		case Opt_read_from_replica_balance:
-			opt->osd_req_flags |= CEPH_OSD_FLAG_BALANCE_READS;
-			opt->osd_req_flags &= ~CEPH_OSD_FLAG_LOCALIZE_READS;
+			opt->read_from_replica = CEPH_OSD_FLAG_BALANCE_READS;
 			break;
 		case Opt_read_from_replica_localize:
-			opt->osd_req_flags |= CEPH_OSD_FLAG_LOCALIZE_READS;
-			opt->osd_req_flags &= ~CEPH_OSD_FLAG_BALANCE_READS;
+			opt->read_from_replica = CEPH_OSD_FLAG_LOCALIZE_READS;
 			break;
 		default:
 			BUG();
@@ -614,9 +612,9 @@ int ceph_print_client_options(struct seq_file *m, struct ceph_client *client,
 		}
 		seq_putc(m, ',');
 	}
-	if (opt->osd_req_flags & CEPH_OSD_FLAG_BALANCE_READS) {
+	if (opt->read_from_replica == CEPH_OSD_FLAG_BALANCE_READS) {
 		seq_puts(m, "read_from_replica=balance,");
-	} else if (opt->osd_req_flags & CEPH_OSD_FLAG_LOCALIZE_READS) {
+	} else if (opt->read_from_replica == CEPH_OSD_FLAG_LOCALIZE_READS) {
 		seq_puts(m, "read_from_replica=localize,");
 	}
 
net/ceph/osd_client.c +1 -1
index 4fea3c33af2a..b151fe163821 100644
@@ -1117,10 +1117,10 @@ struct ceph_osd_request *ceph_osdc_new_request(struct ceph_osd_client *osdc,
 				       truncate_size, truncate_seq);
 	}
 
-	req->r_flags = flags;
 	req->r_base_oloc.pool = layout->pool_id;
 	req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns);
 	ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum);
+	req->r_flags = flags | osdc->client->options->read_from_replica;
 
 	req->r_snapid = vino.snap;
 	if (flags & CEPH_OSD_FLAG_WRITE)

net/ceph/osd_client.c static void account_request(struct ceph_osd_request *req) · line 2442

Quoted 7 lines
net/ceph/osd_client.c +0 -1
@@ -2437,7 +2437,6 @@ static void account_request(struct ceph_osd_request *req)
 	WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE)));
 
 	req->r_flags |= CEPH_OSD_FLAG_ONDISK;
-	req->r_flags |= osdc->client->options->osd_req_flags;
 	atomic_inc(&osdc->num_requests);
 
 	req->r_start_stamp = jiffies;

Cheers,

Signature · 1 line
Luis
author maintainer
#2 lore raw
Quoted 17 lines · Luis Henriques
Quoted 14 lines · Ilya Dryomov

osd_req_flags is overly general and doesn't suit its only user (read_from_replica option) well:

- applying osd_req_flags in account_request() affects all OSD requests, including linger (i.e. watch and notify). However, linger requests should always go to the primary even though some of them are reads (e.g. notify has side effects but it is a read because it doesn't result in mutation on the OSDs).

- calls to class methods that are reads are allowed to go to the replica, but most such calls issued for "rbd map" and/or exclusive lock transitions are requested to be resent to the primary via EAGAIN, doubling the latency.

Quoted 7 lines
Quoted 4 lines

Get rid of global osd_req_flags and set read_from_replica flag only on specific OSD requests instead.

Fixes 8ad44d5e0d1e ("libceph: read_from_replica option")

Since this isn't yet in mainline, can't you simply merge these two patches?

Hi Luis,

The pull request has already been put together and I'd rather not rebase.

Quoted 60 lines
Quoted 60 lines
---
 drivers/block/rbd.c          |  4 +++-
 include/linux/ceph/libceph.h |  4 ++--
 net/ceph/ceph_common.c       | 14 ++++++--------
 net/ceph/osd_client.c        |  3 +--
 4 files changed, 12 insertions(+), 13 deletions(-)
drivers/block/rbd.c +3 -1
index e02089d550a4..b2bb2f10562a 100644
@@ -1451,8 +1451,10 @@ static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
 static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
 {
      struct rbd_obj_request *obj_request = osd_req->r_priv;
+     struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
+     struct ceph_options *opt = rbd_dev->rbd_client->client->options;
 
-     osd_req->r_flags = CEPH_OSD_FLAG_READ;
+     osd_req->r_flags = CEPH_OSD_FLAG_READ | opt->read_from_replica;
      osd_req->r_snapid = obj_request->img_request->snap_id;
 }
 
include/linux/ceph/libceph.h +2 -2
index 2247e71beb83..e5ed1c541e7f 100644
@@ -52,8 +52,7 @@ struct ceph_options {
      unsigned long osd_idle_ttl;             /* jiffies */
      unsigned long osd_keepalive_timeout;    /* jiffies */
      unsigned long osd_request_timeout;      /* jiffies */
-
-     u32 osd_req_flags;  /* CEPH_OSD_FLAG_*, applied to each OSD request */
+     u32 read_from_replica;  /* CEPH_OSD_FLAG_BALANCE/LOCALIZE_READS */
 
      /*
       * any type that can't be simply compared or doesn't need
@@ -76,6 +75,7 @@ struct ceph_options {
 #define CEPH_OSD_KEEPALIVE_DEFAULT   msecs_to_jiffies(5 * 1000)
 #define CEPH_OSD_IDLE_TTL_DEFAULT    msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
+#define CEPH_READ_FROM_REPLICA_DEFAULT       0  /* read from primary */
 
 #define CEPH_MONC_HUNT_INTERVAL              msecs_to_jiffies(3 * 1000)
 #define CEPH_MONC_PING_INTERVAL              msecs_to_jiffies(10 * 1000)
net/ceph/ceph_common.c +1 -0
index 9bab3e9a039b..b7705e91ae9a 100644
@@ -333,6 +333,7 @@ struct ceph_options *ceph_alloc_options(void)
      opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
      opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
      opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
+     opt->read_from_replica = CEPH_READ_FROM_REPLICA_DEFAULT;
      return opt;
 }
 EXPORT_SYMBOL(ceph_alloc_options);
@@ -491,16 +492,13 @@ int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt,
Quoted 8 lines

net/ceph/ceph_common.c int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, · line 495

Quoted 6 lines
net/ceph/ceph_common.c +1 -2
      case Opt_read_from_replica:
              switch (result.uint_32) {
              case Opt_read_from_replica_no:
-                     opt->osd_req_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS |
-                                             CEPH_OSD_FLAG_LOCALIZE_READS);
+                     opt->read_from_replica = 0;

CEPH_READ_FROM_REPLICA_DEFAULT?

No. If the default changes, read_from_replica=no should still mean "don't go to the replica, use the primary".

Thanks,

Ilya

#3 lore raw
Quoted 23 lines · Ilya Dryomov
Quoted 22 lines · Luis Henriques
Quoted 18 lines · Ilya Dryomov

osd_req_flags is overly general and doesn't suit its only user (read_from_replica option) well:

- applying osd_req_flags in account_request() affects all OSD requests, including linger (i.e. watch and notify). However, linger requests should always go to the primary even though some of them are reads (e.g. notify has side effects but it is a read because it doesn't result in mutation on the OSDs).

- calls to class methods that are reads are allowed to go to the replica, but most such calls issued for "rbd map" and/or exclusive lock transitions are requested to be resent to the primary via EAGAIN, doubling the latency.

Get rid of global osd_req_flags and set read_from_replica flag only on specific OSD requests instead.

Fixes 8ad44d5e0d1e ("libceph: read_from_replica option")
Quoted 7 lines
Quoted 2 lines

Since this isn't yet in mainline, can't you simply merge these two patches?

Hi Luis,

The pull request has already been put together and I'd rather not rebase.

Ah, ok. It's a bit more churn but that's not a big deal I guess.

Quoted 69 lines
Quoted 60 lines
Quoted 60 lines
---
 drivers/block/rbd.c          |  4 +++-
 include/linux/ceph/libceph.h |  4 ++--
 net/ceph/ceph_common.c       | 14 ++++++--------
 net/ceph/osd_client.c        |  3 +--
 4 files changed, 12 insertions(+), 13 deletions(-)
drivers/block/rbd.c +3 -1
index e02089d550a4..b2bb2f10562a 100644
@@ -1451,8 +1451,10 @@ static void rbd_osd_req_callback(struct ceph_osd_request *osd_req)
 static void rbd_osd_format_read(struct ceph_osd_request *osd_req)
 {
      struct rbd_obj_request *obj_request = osd_req->r_priv;
+     struct rbd_device *rbd_dev = obj_request->img_request->rbd_dev;
+     struct ceph_options *opt = rbd_dev->rbd_client->client->options;
 
-     osd_req->r_flags = CEPH_OSD_FLAG_READ;
+     osd_req->r_flags = CEPH_OSD_FLAG_READ | opt->read_from_replica;
      osd_req->r_snapid = obj_request->img_request->snap_id;
 }
 
include/linux/ceph/libceph.h +2 -2
index 2247e71beb83..e5ed1c541e7f 100644
@@ -52,8 +52,7 @@ struct ceph_options {
      unsigned long osd_idle_ttl;             /* jiffies */
      unsigned long osd_keepalive_timeout;    /* jiffies */
      unsigned long osd_request_timeout;      /* jiffies */
-
-     u32 osd_req_flags;  /* CEPH_OSD_FLAG_*, applied to each OSD request */
+     u32 read_from_replica;  /* CEPH_OSD_FLAG_BALANCE/LOCALIZE_READS */
 
      /*
       * any type that can't be simply compared or doesn't need
@@ -76,6 +75,7 @@ struct ceph_options {
 #define CEPH_OSD_KEEPALIVE_DEFAULT   msecs_to_jiffies(5 * 1000)
 #define CEPH_OSD_IDLE_TTL_DEFAULT    msecs_to_jiffies(60 * 1000)
 #define CEPH_OSD_REQUEST_TIMEOUT_DEFAULT 0  /* no timeout */
+#define CEPH_READ_FROM_REPLICA_DEFAULT       0  /* read from primary */
 
 #define CEPH_MONC_HUNT_INTERVAL              msecs_to_jiffies(3 * 1000)
 #define CEPH_MONC_PING_INTERVAL              msecs_to_jiffies(10 * 1000)
net/ceph/ceph_common.c +1 -0
index 9bab3e9a039b..b7705e91ae9a 100644
@@ -333,6 +333,7 @@ struct ceph_options *ceph_alloc_options(void)
      opt->mount_timeout = CEPH_MOUNT_TIMEOUT_DEFAULT;
      opt->osd_idle_ttl = CEPH_OSD_IDLE_TTL_DEFAULT;
      opt->osd_request_timeout = CEPH_OSD_REQUEST_TIMEOUT_DEFAULT;
+     opt->read_from_replica = CEPH_READ_FROM_REPLICA_DEFAULT;
      return opt;
 }
 EXPORT_SYMBOL(ceph_alloc_options);
@@ -491,16 +492,13 @@ int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt,
Quoted 8 lines

net/ceph/ceph_common.c int ceph_parse_param(struct fs_parameter *param, struct ceph_options *opt, · line 495

Quoted 6 lines
net/ceph/ceph_common.c +1 -2
      case Opt_read_from_replica:
              switch (result.uint_32) {
              case Opt_read_from_replica_no:
-                     opt->osd_req_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS |
-                                             CEPH_OSD_FLAG_LOCALIZE_READS);
+                     opt->read_from_replica = 0;

CEPH_READ_FROM_REPLICA_DEFAULT?

Quoted 2 lines

No. If the default changes, read_from_replica=no should still mean "don't go to the replica, use the primary".

Makes sense. Thanks.

Cheers,

Signature · 1 line
Luis
Reviewed-by on v1 1/1
#4 lore raw
In reply to Ilya Dryomov
Quoted 17 lines · Ilya Dryomov

osd_req_flags is overly general and doesn't suit its only user (read_from_replica option) well:

- applying osd_req_flags in account_request() affects all OSD requests, including linger (i.e. watch and notify). However, linger requests should always go to the primary even though some of them are reads (e.g. notify has side effects but it is a read because it doesn't result in mutation on the OSDs).

- calls to class methods that are reads are allowed to go to the replica, but most such calls issued for "rbd map" and/or exclusive lock transitions are requested to be resent to the primary via EAGAIN, doubling the latency.

Get rid of global osd_req_flags and set read_from_replica flag only on specific OSD requests instead.

Quoted 8 lines
Fixes 8ad44d5e0d1e ("libceph: read_from_replica option")
---
 drivers/block/rbd.c          |  4 +++-
 include/linux/ceph/libceph.h |  4 ++--
 net/ceph/ceph_common.c       | 14 ++++++--------
 net/ceph/osd_client.c        |  3 +--
 4 files changed, 12 insertions(+), 13 deletions(-)

Looks reasonable: