libceph: cleanups preparing for state cleanup

The ceph connection state "DEAD" is never set and is therefore not needed.  Eliminate it.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/messenger.h |    1 -
 net/ceph/messenger.c           |    6 ------

 2 files changed, 0 insertions(+), 7 deletions(-)

include/linux/ceph/messenger.h +0 -1

index 2521a95..aa506ca 100644
@@ -119,7 +119,6 @@ struct ceph_msg_pos {
  #define CLOSED         10 /* we've closed the connection */
  #define SOCK_CLOSED    11 /* socket state changed to closed */
  #define OPENING         13 /* open connection w/ (possibly new) peer */
-#define DEAD            14 /* dead, about to kfree */
  #define BACKOFF         15
 
  /*

net/ceph/messenger.c +0 -7

index 1a80907..42ca8aa 100644
@@ -2087,12 +2087,6 @@ bad_tag:
  */
  static void queue_con(struct ceph_connection *con)
  {
-       if (test_bit(DEAD, &con->state)) {
-               dout("queue_con %p ignoring: DEAD\n",
-                    con);
-               return;
-       }
-
        if (!con->ops->get(con)) {
                dout("queue_con %p ref count 0\n", con);
                return;
--

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

No code sets a bad_proto method in its ceph connection operations vector, so just get rid of it.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/messenger.h |    3 ---
 net/ceph/messenger.c           |    5 -----

 2 files changed, 0 insertions(+), 8 deletions(-)

include/linux/ceph/messenger.h +0 -0

index aa506ca..74f6c9b 100644
@@ -31,9 +31,6 @@ struct ceph_connection_operations {

       int (*verify_authorizer_reply) (struct ceph_connection *con, int len);
       int (*invalidate_authorizer)(struct ceph_connection *con);

-       /* protocol version mismatch */
-       void (*bad_proto) (struct ceph_connection *con);
-
       /* there was some error on the socket (disconnect, whatever) */ void (*fault) (struct ceph_connection *con);

net/ceph/messenger.c +0 -0

index 42ca8aa..07af994 100644
@@ -1356,11 +1356,6 @@ static void fail_protocol(struct ceph_connection
 *con)
  {
        reset_connection(con);

       set_bit(CLOSED, &con->state);  /* in case there's queued work */ -
-       mutex_unlock(&con->mutex);
-       if (con->ops->bad_proto)
-               con->ops->bad_proto(con);
-       mutex_lock(&con->mutex);
 }

 static int process_connect(struct ceph_connection *con) --
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

In con_close_socket(), SOCK_CLOSED is set in the connection state, then cleared again after shutting down the socket. Nothing between the setting and clearing of that bit will ever be affected by it, so there's no point in setting/clearing it at all. So don't.

Signed-off-by Alex Elder <[email protected]>

---
net/ceph/messenger.c | 2 --

1 files changed, 0 insertions(+), 2 deletions(-)

net/ceph/messenger.c +0 -2

index 07af994..fe3c2a1 100644
@@ -338,11 +338,9 @@ static int con_close_socket(struct ceph_connection
 *con)
 dout("con_close_socket on %p sock %p\n", con, con->sock);
 if (!con->sock)
 return 0;
- set_bit(SOCK_CLOSED, &con->state);
 rc = con->sock->ops->shutdown(con->sock, SHUT_RDWR);
 sock_release(con->sock);
 con->sock = NULL;
- clear_bit(SOCK_CLOSED, &con->state);
 return rc;
 }

Change the names of the three socket callback functions to make it more obvious they're specifically associated with a connection's socket (not the ceph connection that uses it).

Signed-off-by Alex Elder <[email protected]>

---
 net/ceph/messenger.c |   28 ++++++++++++++--------------

 1 files changed, 14 insertions(+), 14 deletions(-)

net/ceph/messenger.c +8 -8

index fe3c2a1..5ad1f0a 100644
@@ -153,46 +153,46 @@ EXPORT_SYMBOL(ceph_msgr_flush);
  */
 
  /* data available on socket, or listen socket received a connect */
-static void ceph_data_ready(struct sock *sk, int count_unused)
+static void ceph_sock_data_ready(struct sock *sk, int count_unused)
  {
        struct ceph_connection *con = sk->sk_user_data;
 
        if (sk->sk_state != TCP_CLOSE_WAIT) {
-               dout("ceph_data_ready on %p state = %lu, queueing work\n",
+               dout("%s on %p state = %lu, queueing work\n", __func__,
                     con, con->state);
                queue_con(con);
        }
  }
 
  /* socket has buffer space for writing */
-static void ceph_write_space(struct sock *sk)
+static void ceph_sock_write_space(struct sock *sk)
  {
        struct ceph_connection *con = sk->sk_user_data;
 
        /* only queue to workqueue if there is data we want to write,
         * and there is sufficient space in the socket buffer to accept
-        * more data.  clear SOCK_NOSPACE so that ceph_write_space()
+        * more data.  clear SOCK_NOSPACE so that ceph_sock_write_space()
         * doesn't get called again until try_write() fills the socket
         * buffer. See net/ipv4/tcp_input.c:tcp_check_space()
         * and net/core/stream.c:sk_stream_write_space().
         */
        if (test_bit(WRITE_PENDING, &con->state)) {
                if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
-                       dout("ceph_write_space %p queueing write work\n",
 con);
+                       dout("%s %p queueing write work\n", __func__, con);
                        clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
                        queue_con(con);
                }
        } else {
-               dout("ceph_write_space %p nothing to write\n", con);
+               dout("%s %p nothing to write\n", __func__, con);
        }
  }
 
  /* socket's state has changed */
-static void ceph_state_change(struct sock *sk)
+static void ceph_sock_state_change(struct sock *sk)
  {
        struct ceph_connection *con = sk->sk_user_data;
 
-       dout("ceph_state_change %p state = %lu sk_state = %u\n",
+       dout("%s %p state = %lu sk_state = %u\n", __func__,
             con, con->state, sk->sk_state);
 

       if (test_bit(CLOSED, &con->state))

net/ceph/messenger.c +0 -0

@@ -200,9 +200,9 @@ static void ceph_state_change(struct sock *sk)
 
        switch (sk->sk_state) {

       case TCP_CLOSE:
-               dout("ceph_state_change TCP_CLOSE\n");
+               dout("%s TCP_CLOSE\n", __func__);
       case TCP_CLOSE_WAIT:
-               dout("ceph_state_change TCP_CLOSE_WAIT\n");
+               dout("%s TCP_CLOSE_WAIT\n", __func__);
               if (test_and_set_bit(SOCK_CLOSED, &con->state) == 0) { if (test_bit(CONNECTING, &con->state)) con->error_msg = "connection failed";

net/ceph/messenger.c +0 -0

@@ -212,7 +212,7 @@ static void ceph_state_change(struct sock *sk)
                }
                break;

       case TCP_ESTABLISHED:
-               dout("ceph_state_change TCP_ESTABLISHED\n");
+               dout("%s TCP_ESTABLISHED\n", __func__); queue_con(con);
               break;
       default:        /* Everything else is uninteresting */

net/ceph/messenger.c +3 -4

@@ -228,9 +228,9 @@ static void set_sock_callbacks(struct socket *sock,
  {
        struct sock *sk = sock->sk;
        sk->sk_user_data = con;
-       sk->sk_data_ready = ceph_data_ready;
-       sk->sk_write_space = ceph_write_space;
-       sk->sk_state_change = ceph_state_change;
+       sk->sk_data_ready = ceph_sock_data_ready;
+       sk->sk_write_space = ceph_sock_write_space;
+       sk->sk_state_change = ceph_sock_state_change;
  }
 
 
--

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Change the names of the three socket callback functions to make it more obvious they're specifically associated with a connection's socket (not the ceph connection that uses it).

Signed-off-by Alex Elder <[email protected]>

---
 net/ceph/messenger.c |   28 ++++++++++++++--------------
 1 files changed, 14 insertions(+), 14 deletions(-)

net/ceph/messenger.c +0 -0

index fe3c2a1..5ad1f0a 100644
@@ -153,46 +153,46 @@ EXPORT_SYMBOL(ceph_msgr_flush);
  */

net/ceph/messenger.c +6 -6

 		     con, con->state);
 		queue_con(con);
 	}
 }
 
 /* socket has buffer space for writing */
-static void ceph_write_space(struct sock *sk)
+static void ceph_sock_write_space(struct sock *sk)
 {
 	struct ceph_connection *con = sk->sk_user_data;
 
 	/* only queue to workqueue if there is data we want to write,
 	 * and there is sufficient space in the socket buffer to accept
-	 * more data.  clear SOCK_NOSPACE so that ceph_write_space()
+	 * more data.  clear SOCK_NOSPACE so that ceph_sock_write_space()
 	 * doesn't get called again until try_write() fills the socket
 	 * buffer. See net/ipv4/tcp_input.c:tcp_check_space()
 	 * and net/core/stream.c:sk_stream_write_space().
 	 */
 	if (test_bit(WRITE_PENDING, &con->state)) {
 		if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
-			dout("ceph_write_space %p queueing write work\n",
 con);
+			dout("%s %p queueing write work\n", __func__, con);
 			clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 			queue_con(con);
 		}
 	} else {
-		dout("ceph_write_space %p nothing to write\n", con);
+		dout("%s %p nothing to write\n", __func__, con);
 	}
 }
 
 /* socket's state has changed */
-static void ceph_state_change(struct sock *sk)
+static void ceph_sock_state_change(struct sock *sk)
 {
 	struct ceph_connection *con = sk->sk_user_data;
 
-	dout("ceph_state_change %p state = %lu sk_state = %u\n",
+	dout("%s %p state = %lu sk_state = %u\n", __func__,
 	     con, con->state, sk->sk_state);
 

net/ceph/messenger.c +6 -6

 	if (test_bit(CLOSED, &con->state))

@@ -200,9 +200,9 @@ static void ceph_state_change(struct sock *sk)
 
 	switch (sk->sk_state) {
 	case TCP_CLOSE:
-		dout("ceph_state_change TCP_CLOSE\n");
+		dout("%s TCP_CLOSE\n", __func__);
 	case TCP_CLOSE_WAIT:
-		dout("ceph_state_change TCP_CLOSE_WAIT\n");
+		dout("%s TCP_CLOSE_WAIT\n", __func__);
 		if (test_and_set_bit(SOCK_CLOSED, &con->state) == 0) {
 			if (test_bit(CONNECTING, &con->state))
 				con->error_msg = "connection failed";

@@ -212,7 +212,7 @@ static void ceph_state_change(struct sock *sk)
 		}
 		break;
 	case TCP_ESTABLISHED:
-		dout("ceph_state_change TCP_ESTABLISHED\n");
+		dout("%s TCP_ESTABLISHED\n", __func__);
 		queue_con(con);
 		break;
 	default:	/* Everything else is uninteresting */

@@ -228,9 +228,9 @@ static void set_sock_callbacks(struct socket *sock,
 {
 	struct sock *sk = sock->sk;
 	sk->sk_user_data = con;
-	sk->sk_data_ready = ceph_data_ready;
-	sk->sk_write_space = ceph_write_space;
-	sk->sk_state_change = ceph_state_change;
+	sk->sk_data_ready = ceph_sock_data_ready;
+	sk->sk_write_space = ceph_sock_write_space;
+	sk->sk_state_change = ceph_sock_state_change;
 }
 
 

Signature · 6 lines

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

The functions ceph_con_out_kvec_reset() and ceph_con_out_kvec_add() are entirely private functions, so drop the "ceph_" prefix in their name to make them slightly more wieldy.

Signed-off-by Alex Elder <[email protected]>

---
 net/ceph/messenger.c |   48
++++++++++++++++++++++++------------------------

 1 files changed, 24 insertions(+), 24 deletions(-)

net/ceph/messenger.c +2 -2

index 5ad1f0a..2e9054f 100644
@@ -484,14 +484,14 @@ static u32 get_global_seq(struct ceph_messenger *msgr,
 u32 gt)
        return ret;
  }
 
-static void ceph_con_out_kvec_reset(struct ceph_connection *con)
+static void con_out_kvec_reset(struct ceph_connection *con)
  {
        con->out_kvec_left = 0;
        con->out_kvec_bytes = 0;
        con->out_kvec_cur = &con->out_kvec[0];
  }
 
-static void ceph_con_out_kvec_add(struct ceph_connection *con,
+static void con_out_kvec_add(struct ceph_connection *con,
                                size_t size, void *data)
  {

       int index;

net/ceph/messenger.c +7 -7

@@ -532,7 +532,7 @@ static void prepare_write_message(struct ceph_connection
 *con)
        struct ceph_msg *m;
        u32 crc;
 
-       ceph_con_out_kvec_reset(con);
+       con_out_kvec_reset(con);
        con->out_kvec_is_msg = true;
        con->out_msg_done = false;

@@ -540,9 +540,9 @@ static void prepare_write_message(struct ceph_connection
 *con)
         * TCP packet that's a good thing. */
        if (con->in_seq > con->in_seq_acked) {
                con->in_seq_acked = con->in_seq;
-               ceph_con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
+               con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
                con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
-               ceph_con_out_kvec_add(con, sizeof (con->out_temp_ack),
+               con_out_kvec_add(con, sizeof (con->out_temp_ack),
                        &con->out_temp_ack);
        }

@@ -570,12 +570,12 @@ static void prepare_write_message(struct
 ceph_connection *con)
        BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);
 
        /* tag + hdr + front + middle */
-       ceph_con_out_kvec_add(con, sizeof (tag_msg), &tag_msg);
-       ceph_con_out_kvec_add(con, sizeof (m->hdr), &m->hdr);
-       ceph_con_out_kvec_add(con, m->front.iov_len, m->front.iov_base);
+       con_out_kvec_add(con, sizeof (tag_msg), &tag_msg);
+       con_out_kvec_add(con, sizeof (m->hdr), &m->hdr);
+       con_out_kvec_add(con, m->front.iov_len, m->front.iov_base);
 
        if (m->middle)
-               ceph_con_out_kvec_add(con, m->middle->vec.iov_len,
+               con_out_kvec_add(con, m->middle->vec.iov_len,
                        m->middle->vec.iov_base);
 

       /* fill in crc (except data pages), footer */

net/ceph/messenger.c +3 -3

@@ -624,12 +624,12 @@ static void prepare_write_ack(struct ceph_connection
 *con)
             con->in_seq_acked, con->in_seq);
        con->in_seq_acked = con->in_seq;
 
-       ceph_con_out_kvec_reset(con);
+       con_out_kvec_reset(con);
 
-       ceph_con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
+       con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
 
        con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
-       ceph_con_out_kvec_add(con, sizeof (con->out_temp_ack),
+       con_out_kvec_add(con, sizeof (con->out_temp_ack),
                                &con->out_temp_ack);
 

       con->out_more = 1;  /* more will follow.. eventually.. */

net/ceph/messenger.c +4 -4

@@ -642,8 +642,8 @@ static void prepare_write_ack(struct ceph_connection
 *con)
  static void prepare_write_keepalive(struct ceph_connection *con)
  {
        dout("prepare_write_keepalive %p\n", con);
-       ceph_con_out_kvec_reset(con);
-       ceph_con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive);
+       con_out_kvec_reset(con);
+       con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive);
        set_bit(WRITE_PENDING, &con->state);
  }

@@ -688,8 +688,8 @@ static struct ceph_auth_handshake
 *get_connect_authorizer(struct ceph_connection
  */
  static void prepare_write_banner(struct ceph_connection *con)
  {
-       ceph_con_out_kvec_add(con, strlen(CEPH_BANNER), CEPH_BANNER);
-       ceph_con_out_kvec_add(con, sizeof (con->msgr->my_enc_addr),
+       con_out_kvec_add(con, strlen(CEPH_BANNER), CEPH_BANNER);
+       con_out_kvec_add(con, sizeof (con->msgr->my_enc_addr),
                                        &con->msgr->my_enc_addr);
 

       con->out_more = 0;

net/ceph/messenger.c +0 -0

@@ -736,10 +736,10 @@ static int prepare_write_connect(struct
 ceph_connection *con)

       con->out_connect.authorizer_len = auth ? cpu_to_le32(auth->authorizer_buf_len) : 0;

-       ceph_con_out_kvec_add(con, sizeof (con->out_connect),
+       con_out_kvec_add(con, sizeof (con->out_connect), &con->out_connect); if (auth && auth->authorizer_buf_len)
-               ceph_con_out_kvec_add(con, auth->authorizer_buf_len,
+               con_out_kvec_add(con, auth->authorizer_buf_len, auth->authorizer_buf);

       con->out_more = 0;

net/ceph/messenger.c +1 -1

@@ -933,7 +933,7 @@ static int write_partial_msg_pages(struct
 ceph_connection *con)
        /* prepare and queue up footer, too */
        if (!do_datacrc)
                con->out_msg->footer.flags |= CEPH_MSG_FOOTER_NOCRC;
-       ceph_con_out_kvec_reset(con);
+       con_out_kvec_reset(con);
        prepare_write_message_footer(con);
        ret = 1;

 out:

net/ceph/messenger.c +1 -1

@@ -1396,7 +1396,7 @@ static int process_connect(struct ceph_connection
 *con)
                        return -1;
                }
                con->auth_retry = 1;
-               ceph_con_out_kvec_reset(con);
+               con_out_kvec_reset(con);
                ret = prepare_write_connect(con);
                if (ret < 0)

                       return ret;

net/ceph/messenger.c +1 -1

@@ -1417,7 +1417,7 @@ static int process_connect(struct ceph_connection
 *con)
                       ENTITY_NAME(con->peer_name),
                       ceph_pr_addr(&con->peer_addr.in_addr));
                reset_connection(con);
-               ceph_con_out_kvec_reset(con);
+               con_out_kvec_reset(con);
                ret = prepare_write_connect(con);
                if (ret < 0)

                       return ret;

net/ceph/messenger.c +1 -1

@@ -1443,7 +1443,7 @@ static int process_connect(struct ceph_connection
 *con)
                     le32_to_cpu(con->out_connect.connect_seq),
                     le32_to_cpu(con->in_connect.connect_seq));
                con->connect_seq = le32_to_cpu(con->in_connect.connect_seq);
-               ceph_con_out_kvec_reset(con);
+               con_out_kvec_reset(con);
                ret = prepare_write_connect(con);
                if (ret < 0)

                       return ret;

net/ceph/messenger.c +1 -1

@@ -1460,7 +1460,7 @@ static int process_connect(struct ceph_connection
 *con)
                     le32_to_cpu(con->in_connect.global_seq));
                get_global_seq(con->msgr,
                               le32_to_cpu(con->in_connect.global_seq));
-               ceph_con_out_kvec_reset(con);
+               con_out_kvec_reset(con);
                ret = prepare_write_connect(con);
                if (ret < 0)

                       return ret;

net/ceph/messenger.c +1 -2

@@ -1867,7 +1867,7 @@ more:
 
        /* open the socket first? */
        if (con->sock == NULL) {
-               ceph_con_out_kvec_reset(con);
+               con_out_kvec_reset(con);
                prepare_write_banner(con);
                ret = prepare_write_connect(con);
                if (ret < 0)
--

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

The functions ceph_con_out_kvec_reset() and ceph_con_out_kvec_add() are entirely private functions, so drop the "ceph_" prefix in their name to make them slightly more wieldy.

Signed-off-by Alex Elder <[email protected]>

---
 net/ceph/messenger.c |   48 ++++++++++++++++++++++++------------------------
 1 files changed, 24 insertions(+), 24 deletions(-)

net/ceph/messenger.c +2 -2

index 5ad1f0a..2e9054f 100644
@@ -484,14 +484,14 @@ static u32 get_global_seq(struct ceph_messenger *msgr,
 u32 gt)
 	return ret;
 }
 
-static void ceph_con_out_kvec_reset(struct ceph_connection *con)
+static void con_out_kvec_reset(struct ceph_connection *con)
 {
 	con->out_kvec_left = 0;
 	con->out_kvec_bytes = 0;
 	con->out_kvec_cur = &con->out_kvec[0];
 }
 
-static void ceph_con_out_kvec_add(struct ceph_connection *con,
+static void con_out_kvec_add(struct ceph_connection *con,
 				size_t size, void *data)
 {

net/ceph/messenger.c +7 -7

 	int index;

@@ -532,7 +532,7 @@ static void prepare_write_message(struct ceph_connection
 *con)
 	struct ceph_msg *m;
 	u32 crc;
 
-	ceph_con_out_kvec_reset(con);
+	con_out_kvec_reset(con);
 	con->out_kvec_is_msg = true;
 	con->out_msg_done = false;

@@ -540,9 +540,9 @@ static void prepare_write_message(struct ceph_connection
 *con)
 	 * TCP packet that's a good thing. */
 	if (con->in_seq > con->in_seq_acked) {
 		con->in_seq_acked = con->in_seq;
-		ceph_con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
+		con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
 		con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
-		ceph_con_out_kvec_add(con, sizeof (con->out_temp_ack),
+		con_out_kvec_add(con, sizeof (con->out_temp_ack),
 			&con->out_temp_ack);
 	}

@@ -570,12 +570,12 @@ static void prepare_write_message(struct ceph_connection
 *con)
 	BUG_ON(le32_to_cpu(m->hdr.front_len) != m->front.iov_len);
 
 	/* tag + hdr + front + middle */
-	ceph_con_out_kvec_add(con, sizeof (tag_msg), &tag_msg);
-	ceph_con_out_kvec_add(con, sizeof (m->hdr), &m->hdr);
-	ceph_con_out_kvec_add(con, m->front.iov_len, m->front.iov_base);
+	con_out_kvec_add(con, sizeof (tag_msg), &tag_msg);
+	con_out_kvec_add(con, sizeof (m->hdr), &m->hdr);
+	con_out_kvec_add(con, m->front.iov_len, m->front.iov_base);
 
 	if (m->middle)
-		ceph_con_out_kvec_add(con, m->middle->vec.iov_len,
+		con_out_kvec_add(con, m->middle->vec.iov_len,
 			m->middle->vec.iov_base);
 

net/ceph/messenger.c +3 -3

 	/* fill in crc (except data pages), footer */

@@ -624,12 +624,12 @@ static void prepare_write_ack(struct ceph_connection
 *con)
 	     con->in_seq_acked, con->in_seq);
 	con->in_seq_acked = con->in_seq;
 
-	ceph_con_out_kvec_reset(con);
+	con_out_kvec_reset(con);
 
-	ceph_con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
+	con_out_kvec_add(con, sizeof (tag_ack), &tag_ack);
 
 	con->out_temp_ack = cpu_to_le64(con->in_seq_acked);
-	ceph_con_out_kvec_add(con, sizeof (con->out_temp_ack),
+	con_out_kvec_add(con, sizeof (con->out_temp_ack),
 				&con->out_temp_ack);
 

net/ceph/messenger.c +4 -4

 	con->out_more = 1;  /* more will follow.. eventually.. */

@@ -642,8 +642,8 @@ static void prepare_write_ack(struct ceph_connection *con)
 static void prepare_write_keepalive(struct ceph_connection *con)
 {
 	dout("prepare_write_keepalive %p\n", con);
-	ceph_con_out_kvec_reset(con);
-	ceph_con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive);
+	con_out_kvec_reset(con);
+	con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive);
 	set_bit(WRITE_PENDING, &con->state);
 }
 

@@ -688,8 +688,8 @@ static struct ceph_auth_handshake
 *get_connect_authorizer(struct ceph_connection
  */
 static void prepare_write_banner(struct ceph_connection *con)
 {
-	ceph_con_out_kvec_add(con, strlen(CEPH_BANNER), CEPH_BANNER);
-	ceph_con_out_kvec_add(con, sizeof (con->msgr->my_enc_addr),
+	con_out_kvec_add(con, strlen(CEPH_BANNER), CEPH_BANNER);
+	con_out_kvec_add(con, sizeof (con->msgr->my_enc_addr),
 					&con->msgr->my_enc_addr);
 

net/ceph/messenger.c +2 -2

 	con->out_more = 0;

@@ -736,10 +736,10 @@ static int prepare_write_connect(struct ceph_connection
 *con)
 	con->out_connect.authorizer_len = auth ?
 		cpu_to_le32(auth->authorizer_buf_len) : 0;
 
-	ceph_con_out_kvec_add(con, sizeof (con->out_connect),
+	con_out_kvec_add(con, sizeof (con->out_connect),
 					&con->out_connect);
 	if (auth && auth->authorizer_buf_len)
-		ceph_con_out_kvec_add(con, auth->authorizer_buf_len,
+		con_out_kvec_add(con, auth->authorizer_buf_len,
 					auth->authorizer_buf);
 

net/ceph/messenger.c +1 -1

 	con->out_more = 0;

@@ -933,7 +933,7 @@ static int write_partial_msg_pages(struct ceph_connection
 *con)
 	/* prepare and queue up footer, too */
 	if (!do_datacrc)
 		con->out_msg->footer.flags |= CEPH_MSG_FOOTER_NOCRC;
-	ceph_con_out_kvec_reset(con);
+	con_out_kvec_reset(con);
 	prepare_write_message_footer(con);
 	ret = 1;

out:

net/ceph/messenger.c +5 -5

@@ -1396,7 +1396,7 @@ static int process_connect(struct ceph_connection *con)
 			return -1;
 		}
 		con->auth_retry = 1;
-		ceph_con_out_kvec_reset(con);
+		con_out_kvec_reset(con);
 		ret = prepare_write_connect(con);
 		if (ret < 0)
 			return ret;

@@ -1417,7 +1417,7 @@ static int process_connect(struct ceph_connection *con)
 		       ENTITY_NAME(con->peer_name),
 		       ceph_pr_addr(&con->peer_addr.in_addr));
 		reset_connection(con);
-		ceph_con_out_kvec_reset(con);
+		con_out_kvec_reset(con);
 		ret = prepare_write_connect(con);
 		if (ret < 0)
 			return ret;

@@ -1443,7 +1443,7 @@ static int process_connect(struct ceph_connection *con)
 		     le32_to_cpu(con->out_connect.connect_seq),
 		     le32_to_cpu(con->in_connect.connect_seq));
 		con->connect_seq = le32_to_cpu(con->in_connect.connect_seq);
-		ceph_con_out_kvec_reset(con);
+		con_out_kvec_reset(con);
 		ret = prepare_write_connect(con);
 		if (ret < 0)
 			return ret;

@@ -1460,7 +1460,7 @@ static int process_connect(struct ceph_connection *con)
 		     le32_to_cpu(con->in_connect.global_seq));
 		get_global_seq(con->msgr,
 			       le32_to_cpu(con->in_connect.global_seq));
-		ceph_con_out_kvec_reset(con);
+		con_out_kvec_reset(con);
 		ret = prepare_write_connect(con);
 		if (ret < 0)
 			return ret;

@@ -1867,7 +1867,7 @@ more:
 
 	/* open the socket first? */
 	if (con->sock == NULL) {
-		ceph_con_out_kvec_reset(con);
+		con_out_kvec_reset(con);
 		prepare_write_banner(con);
 		ret = prepare_write_connect(con);
 		if (ret < 0)

Signature · 6 lines

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A ceph client has a pointer to a ceph messenger structure in it. There is always exactly one ceph messenger for a ceph client, so there is no need to allocate it separate from the ceph client structure.

Switch the ceph_client structure to embed its ceph_messenger structure.

Signed-off-by Alex Elder <[email protected]>

---
 fs/ceph/mds_client.c           |    2 +- include/linux/ceph/libceph.h   |    2 +- include/linux/ceph/messenger.h |    9 +++++---- net/ceph/ceph_common.c         |   18 +++++-------------

 net/ceph/messenger.c           |   30 +++++++++---------------------
 net/ceph/mon_client.c          |    6 +++---
 net/ceph/osd_client.c          |    4 ++--
 7 files changed, 26 insertions(+), 45 deletions(-)

fs/ceph/mds_client.c +1 -1

index 200bc87..ad30261 100644
@@ -394,7 +394,7 @@ static struct ceph_mds_session *register_session(struct
 ceph_mds_client *mdsc,
        s->s_seq = 0;
        mutex_init(&s->s_mutex);
 
-       ceph_con_init(mdsc->fsc->client->msgr, &s->s_con);
+       ceph_con_init(&mdsc->fsc->client->msgr, &s->s_con);
        s->s_con.private = s;
        s->s_con.ops = &mds_con_ops;

       s->s_con.peer_name.type = CEPH_ENTITY_TYPE_MDS;

include/linux/ceph/libceph.h +1 -1

index 92eef7c..927361c 100644
@@ -131,7 +131,7 @@ struct ceph_client {
        u32 supported_features;
        u32 required_features;
 
-       struct ceph_messenger *msgr;   /* messenger instance */
+       struct ceph_messenger msgr;   /* messenger instance */
        struct ceph_mon_client monc;
        struct ceph_osd_client osdc;
 

include/linux/ceph/messenger.h +5 -4

index 74f6c9b..3fbd4be 100644
@@ -211,10 +211,11 @@ extern int ceph_msgr_init(void);
  extern void ceph_msgr_exit(void);
  extern void ceph_msgr_flush(void);
 
-extern struct ceph_messenger *ceph_messenger_create(
-       struct ceph_entity_addr *myaddr,
-       u32 features, u32 required);
-extern void ceph_messenger_destroy(struct ceph_messenger *);
+extern void ceph_messenger_init(struct ceph_messenger *msgr,
+                       struct ceph_entity_addr *myaddr,
+                       u32 supported_features,
+                       u32 required_features,
+                       bool nocrc);
 
  extern void ceph_con_init(struct ceph_messenger *msgr,
                          struct ceph_connection *con);

net/ceph/ceph_common.c +5 -9

index cc91319..2de3ea1 100644
@@ -468,19 +468,15 @@ struct ceph_client *ceph_create_client(struct
 ceph_options *opt, void *private,
        /* msgr */
        if (ceph_test_opt(client, MYIP))
                myaddr = &client->options->my_addr;
-       client->msgr = ceph_messenger_create(myaddr,
-                                            client->supported_features,
-                                            client->required_features);
-       if (IS_ERR(client->msgr)) {
-               err = PTR_ERR(client->msgr);
-               goto fail;
-       }
-       client->msgr->nocrc = ceph_test_opt(client, NOCRC);
+       ceph_messenger_init(&client->msgr, myaddr,
+               client->supported_features,
+               client->required_features,
+               ceph_test_opt(client, NOCRC));
 
        /* subsystems */
        err = ceph_monc_init(&client->monc, client);
        if (err < 0)
-               goto fail_msgr;
+               goto fail;
        err = ceph_osdc_init(&client->osdc, client);
        if (err < 0)

               goto fail_monc;

net/ceph/ceph_common.c +0 -0

@@ -489,8 +485,6 @@ struct ceph_client *ceph_create_client(struct
 ceph_options *opt, void *private,
 

 fail_monc:
       ceph_monc_stop(&client->monc);
-fail_msgr:
-       ceph_messenger_destroy(client->msgr);
 fail:
       kfree(client);
       return ERR_PTR(err);

net/ceph/ceph_common.c +0 -2

@@ -515,8 +509,6 @@ void ceph_destroy_client(struct ceph_client *client)
 
        ceph_debugfs_client_cleanup(client);
 
-       ceph_messenger_destroy(client->msgr);
-
        ceph_destroy_options(client->options);
 
        kfree(client);

net/ceph/messenger.c +9 -21

index 2e9054f..19f1948 100644
@@ -2243,18 +2243,14 @@ out:
 
 
  /*
- * create a new messenger instance
+ * initialize a new messenger instance
  */
-struct ceph_messenger *ceph_messenger_create(struct ceph_entity_addr
 *myaddr,
-                                            u32 supported_features,
-                                            u32 required_features)
+void ceph_messenger_init(struct ceph_messenger *msgr,
+                       struct ceph_entity_addr *myaddr,
+                       u32 supported_features,
+                       u32 required_features,
+                       bool nocrc)
  {
-       struct ceph_messenger *msgr;
-
-       msgr = kzalloc(sizeof(*msgr), GFP_KERNEL);
-       if (msgr == NULL)
-               return ERR_PTR(-ENOMEM);
-
        msgr->supported_features = supported_features;
        msgr->required_features = required_features;

@@ -2267,19 +2263,11 @@ struct ceph_messenger *ceph_messenger_create(struct
 ceph_entity_addr *myaddr,
        msgr->inst.addr.type = 0;
        get_random_bytes(&msgr->inst.addr.nonce,
 sizeof(msgr->inst.addr.nonce));
        encode_my_addr(msgr);
+       msgr->nocrc = nocrc;
 
-       dout("messenger_create %p\n", msgr);
-       return msgr;
-}
-EXPORT_SYMBOL(ceph_messenger_create);
-
-void ceph_messenger_destroy(struct ceph_messenger *msgr)
-{
-       dout("destroy %p\n", msgr);
-       kfree(msgr);
-       dout("destroyed messenger %p\n", msgr);
+       dout("%s %p\n", __func__, msgr);
  }
-EXPORT_SYMBOL(ceph_messenger_destroy);
+EXPORT_SYMBOL(ceph_messenger_init);
 

 static void clear_standby(struct ceph_connection *con) {

net/ceph/mon_client.c +3 -3

index 1845cde..704dc95 100644
@@ -763,7 +763,7 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct
 ceph_client *cl)
        monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL);
        if (!monc->con)
                goto out_monmap;
-       ceph_con_init(monc->client->msgr, monc->con);
+       ceph_con_init(&monc->client->msgr, monc->con);
        monc->con->private = monc;
        monc->con->ops = &mon_con_ops;

@@ -880,8 +880,8 @@ static void handle_auth_reply(struct ceph_mon_client
 *monc,
        } else if (!was_auth &&
 monc->auth->ops->is_authenticated(monc->auth)) {
                dout("authenticated, starting session\n");
 
-               monc->client->msgr->inst.name.type =
 CEPH_ENTITY_TYPE_CLIENT;
-               monc->client->msgr->inst.name.num =
+               monc->client->msgr.inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
+               monc->client->msgr.inst.name.num =

                                       cpu_to_le64(monc->auth->global_id);

               __send_subscribe(monc);

net/ceph/osd_client.c +1 -1

index b098e7b..cca4c7f 100644
@@ -639,7 +639,7 @@ static struct ceph_osd *create_osd(struct
 ceph_osd_client *osdc)
        INIT_LIST_HEAD(&osd->o_osd_lru);
        osd->o_incarnation = 1;
 
-       ceph_con_init(osdc->client->msgr, &osd->o_con);
+       ceph_con_init(&osdc->client->msgr, &osd->o_con);
        osd->o_con.private = osd;
        osd->o_con.ops = &osd_con_ops;

       osd->o_con.peer_name.type = CEPH_ENTITY_TYPE_OSD;

net/ceph/osd_client.c +1 -1

@@ -1391,7 +1391,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client
 *osdc, struct ceph_msg *msg)
                             epoch, maplen);
                        newmap = osdmap_apply_incremental(&p, next,
                                                          osdc->osdmap,
-
 osdc->client->msgr);
+
 &osdc->client->msgr);

                       if (IS_ERR(newmap)) { err = PTR_ERR(newmap); goto bad;
--
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A ceph client has a pointer to a ceph messenger structure in it. There is always exactly one ceph messenger for a ceph client, so there is no need to allocate it separate from the ceph client structure.

Switch the ceph_client structure to embed its ceph_messenger structure.

Signed-off-by Alex Elder <[email protected]>

---
 fs/ceph/mds_client.c           |    2 +-
 include/linux/ceph/libceph.h   |    2 +-
 include/linux/ceph/messenger.h |    9 +++++----
 net/ceph/ceph_common.c         |   18 +++++-------------
 net/ceph/messenger.c           |   30 +++++++++---------------------
 net/ceph/mon_client.c          |    6 +++---
 net/ceph/osd_client.c          |    4 ++--
 7 files changed, 26 insertions(+), 45 deletions(-)

fs/ceph/mds_client.c +1 -1

index 200bc87..ad30261 100644
@@ -394,7 +394,7 @@ static struct ceph_mds_session *register_session(struct
 ceph_mds_client *mdsc,
 	s->s_seq = 0;
 	mutex_init(&s->s_mutex);
 
-	ceph_con_init(mdsc->fsc->client->msgr, &s->s_con);
+	ceph_con_init(&mdsc->fsc->client->msgr, &s->s_con);
 	s->s_con.private = s;
 	s->s_con.ops = &mds_con_ops;

fs/ceph/mds_client.c +0 -0

 	s->s_con.peer_name.type = CEPH_ENTITY_TYPE_MDS;

include/linux/ceph/libceph.h +1 -1

index 92eef7c..927361c 100644
@@ -131,7 +131,7 @@ struct ceph_client {
 	u32 supported_features;
 	u32 required_features;
 
-	struct ceph_messenger *msgr;   /* messenger instance */
+	struct ceph_messenger msgr;   /* messenger instance */
 	struct ceph_mon_client monc;
 	struct ceph_osd_client osdc;
 

include/linux/ceph/messenger.h +5 -4

index 74f6c9b..3fbd4be 100644
@@ -211,10 +211,11 @@ extern int ceph_msgr_init(void);
 extern void ceph_msgr_exit(void);
 extern void ceph_msgr_flush(void);
 
-extern struct ceph_messenger *ceph_messenger_create(
-	struct ceph_entity_addr *myaddr,
-	u32 features, u32 required);
-extern void ceph_messenger_destroy(struct ceph_messenger *);
+extern void ceph_messenger_init(struct ceph_messenger *msgr,
+			struct ceph_entity_addr *myaddr,
+			u32 supported_features,
+			u32 required_features,
+			bool nocrc);
 
 extern void ceph_con_init(struct ceph_messenger *msgr,
 			  struct ceph_connection *con);

net/ceph/ceph_common.c +5 -9

index cc91319..2de3ea1 100644
@@ -468,19 +468,15 @@ struct ceph_client *ceph_create_client(struct
 ceph_options *opt, void *private,
 	/* msgr */
 	if (ceph_test_opt(client, MYIP))
 		myaddr = &client->options->my_addr;
-	client->msgr = ceph_messenger_create(myaddr,
-					     client->supported_features,
-					     client->required_features);
-	if (IS_ERR(client->msgr)) {
-		err = PTR_ERR(client->msgr);
-		goto fail;
-	}
-	client->msgr->nocrc = ceph_test_opt(client, NOCRC);
+	ceph_messenger_init(&client->msgr, myaddr,
+		client->supported_features,
+		client->required_features,
+		ceph_test_opt(client, NOCRC));
 
 	/* subsystems */
 	err = ceph_monc_init(&client->monc, client);
 	if (err < 0)
-		goto fail_msgr;
+		goto fail;
 	err = ceph_osdc_init(&client->osdc, client);
 	if (err < 0)

net/ceph/ceph_common.c +0 -2

 		goto fail_monc;

@@ -489,8 +485,6 @@ struct ceph_client *ceph_create_client(struct ceph_options
 *opt, void *private,
 
 fail_monc:
 	ceph_monc_stop(&client->monc);
-fail_msgr:
-	ceph_messenger_destroy(client->msgr);
 fail:
 	kfree(client);

net/ceph/ceph_common.c +0 -2

 	return ERR_PTR(err);

@@ -515,8 +509,6 @@ void ceph_destroy_client(struct ceph_client *client)
 
 	ceph_debugfs_client_cleanup(client);
 
-	ceph_messenger_destroy(client->msgr);
-
 	ceph_destroy_options(client->options);
 
 	kfree(client);

net/ceph/messenger.c +9 -21

index 2e9054f..19f1948 100644
@@ -2243,18 +2243,14 @@ out:
 
 
 /*
- * create a new messenger instance
+ * initialize a new messenger instance
  */
-struct ceph_messenger *ceph_messenger_create(struct ceph_entity_addr *myaddr,
-					     u32 supported_features,
-					     u32 required_features)
+void ceph_messenger_init(struct ceph_messenger *msgr,
+			struct ceph_entity_addr *myaddr,
+			u32 supported_features,
+			u32 required_features,
+			bool nocrc)
 {
-	struct ceph_messenger *msgr;
-
-	msgr = kzalloc(sizeof(*msgr), GFP_KERNEL);
-	if (msgr == NULL)
-		return ERR_PTR(-ENOMEM);
-
 	msgr->supported_features = supported_features;
 	msgr->required_features = required_features;
 

@@ -2267,19 +2263,11 @@ struct ceph_messenger *ceph_messenger_create(struct
 ceph_entity_addr *myaddr,
 	msgr->inst.addr.type = 0;
 	get_random_bytes(&msgr->inst.addr.nonce,
 sizeof(msgr->inst.addr.nonce));
 	encode_my_addr(msgr);
+	msgr->nocrc = nocrc;
 
-	dout("messenger_create %p\n", msgr);
-	return msgr;
-}
-EXPORT_SYMBOL(ceph_messenger_create);
-
-void ceph_messenger_destroy(struct ceph_messenger *msgr)
-{
-	dout("destroy %p\n", msgr);
-	kfree(msgr);
-	dout("destroyed messenger %p\n", msgr);
+	dout("%s %p\n", __func__, msgr);
 }
-EXPORT_SYMBOL(ceph_messenger_destroy);
+EXPORT_SYMBOL(ceph_messenger_init);
 

static void clear_standby(struct ceph_connection *con) {

net/ceph/mon_client.c +3 -3

index 1845cde..704dc95 100644
@@ -763,7 +763,7 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct
 ceph_client *cl)
 	monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL);
 	if (!monc->con)
 		goto out_monmap;
-	ceph_con_init(monc->client->msgr, monc->con);
+	ceph_con_init(&monc->client->msgr, monc->con);
 	monc->con->private = monc;
 	monc->con->ops = &mon_con_ops;

@@ -880,8 +880,8 @@ static void handle_auth_reply(struct ceph_mon_client
 *monc,
 	} else if (!was_auth && monc->auth->ops->is_authenticated(monc->auth))
 {
 		dout("authenticated, starting session\n");
 
-		monc->client->msgr->inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
-		monc->client->msgr->inst.name.num =
+		monc->client->msgr.inst.name.type = CEPH_ENTITY_TYPE_CLIENT;
+		monc->client->msgr.inst.name.num =
 					cpu_to_le64(monc->auth->global_id);

net/ceph/mon_client.c +0 -0

 		__send_subscribe(monc);

net/ceph/osd_client.c +1 -1

index b098e7b..cca4c7f 100644
@@ -639,7 +639,7 @@ static struct ceph_osd *create_osd(struct ceph_osd_client
 *osdc)
 	INIT_LIST_HEAD(&osd->o_osd_lru);
 	osd->o_incarnation = 1;
 
-	ceph_con_init(osdc->client->msgr, &osd->o_con);
+	ceph_con_init(&osdc->client->msgr, &osd->o_con);
 	osd->o_con.private = osd;
 	osd->o_con.ops = &osd_con_ops;

net/ceph/osd_client.c +1 -1

 	osd->o_con.peer_name.type = CEPH_ENTITY_TYPE_OSD;

@@ -1391,7 +1391,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc,
 struct ceph_msg *msg)
 			     epoch, maplen);
 			newmap = osdmap_apply_incremental(&p, next,
 							  osdc->osdmap,
-							  osdc->client->msgr);
+
 &osdc->client->msgr);
 			if (IS_ERR(newmap)) {

net/ceph/osd_client.c +0 -0

 				err = PTR_ERR(newmap);
 				goto bad;

Signature · 6 lines

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

A monitor client has a pointer to a ceph connection structure in it. This is the only one of the three ceph client types that do it this way; the OSD and MDS clients embed the connection into their main structures. There is always exactly one ceph connection for a monitor client, so there is no need to allocate it separate from the monitor client structure.

So switch the ceph_mon_client structure to embed its ceph_connection structure.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/mon_client.h |    2 +-
 net/ceph/mon_client.c           |   47 ++++++++++++++++----------------------
 2 files changed, 21 insertions(+), 28 deletions(-)

include/linux/ceph/mon_client.h +1 -1

index 545f859..2113e38 100644
@@ -70,7 +70,7 @@ struct ceph_mon_client {
 	bool hunting;
 	int cur_mon;                       /* last monitor i contacted */
 	unsigned long sub_sent, sub_renew_after;
-	struct ceph_connection *con;
+	struct ceph_connection con;
 	bool have_fsid;
 
 	/* pending generic requests */

net/ceph/mon_client.c +2 -2

index 704dc95..ac4d6b1 100644
@@ -106,9 +106,9 @@ static void __send_prepared_auth_request(struct
 ceph_mon_client *monc, int len)
 	monc->pending_auth = 1;
 	monc->m_auth->front.iov_len = len;
 	monc->m_auth->hdr.front_len = cpu_to_le32(len);
-	ceph_con_revoke(monc->con, monc->m_auth);
+	ceph_con_revoke(&monc->con, monc->m_auth);
 	ceph_msg_get(monc->m_auth);  /* keep our ref */
-	ceph_con_send(monc->con, monc->m_auth);
+	ceph_con_send(&monc->con, monc->m_auth);
 }
 

/*

net/ceph/mon_client.c +2 -2

@@ -117,8 +117,8 @@ static void __send_prepared_auth_request(struct
 ceph_mon_client *monc, int len)
 static void __close_session(struct ceph_mon_client *monc)
 {
 	dout("__close_session closing mon%d\n", monc->cur_mon);
-	ceph_con_revoke(monc->con, monc->m_auth);
-	ceph_con_close(monc->con);
+	ceph_con_revoke(&monc->con, monc->m_auth);
+	ceph_con_close(&monc->con);
 	monc->cur_mon = -1;
 	monc->pending_auth = 0;

net/ceph/mon_client.c +6 -6

 	ceph_auth_reset(monc->auth);

@@ -142,9 +142,9 @@ static int __open_session(struct ceph_mon_client *monc)
 		monc->want_next_osdmap = !!monc->want_next_osdmap;
 
 		dout("open_session mon%d opening\n", monc->cur_mon);
-		monc->con->peer_name.type = CEPH_ENTITY_TYPE_MON;
-		monc->con->peer_name.num = cpu_to_le64(monc->cur_mon);
-		ceph_con_open(monc->con,
+		monc->con.peer_name.type = CEPH_ENTITY_TYPE_MON;
+		monc->con.peer_name.num = cpu_to_le64(monc->cur_mon);
+		ceph_con_open(&monc->con,
 			      &monc->monmap->mon_inst[monc->cur_mon].addr);
 
 		/* initiatiate authentication handshake */

@@ -226,8 +226,8 @@ static void __send_subscribe(struct ceph_mon_client *monc)
 
 		msg->front.iov_len = p - msg->front.iov_base;
 		msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
-		ceph_con_revoke(monc->con, msg);
-		ceph_con_send(monc->con, ceph_msg_get(msg));
+		ceph_con_revoke(&monc->con, msg);
+		ceph_con_send(&monc->con, ceph_msg_get(msg));
 
 		monc->sub_sent = jiffies | 1;  /* never 0 */
 	}

@@ -247,7 +247,7 @@ static void handle_subscribe_ack(struct ceph_mon_client
 *monc,
 	if (monc->hunting) {
 		pr_info("mon%d %s session established\n",
 			monc->cur_mon,
-			ceph_pr_addr(&monc->con->peer_addr.in_addr));
+			ceph_pr_addr(&monc->con.peer_addr.in_addr));
 		monc->hunting = false;
 	}

net/ceph/mon_client.c +1 -1

 	dout("handle_subscribe_ack after %d seconds\n", seconds);

@@ -461,7 +461,7 @@ static int do_generic_request(struct ceph_mon_client
 *monc,
 	req->request->hdr.tid = cpu_to_le64(req->tid);
 	__insert_generic_request(monc, req);
 	monc->num_generic_requests++;
-	ceph_con_send(monc->con, ceph_msg_get(req->request));
+	ceph_con_send(&monc->con, ceph_msg_get(req->request));
 	mutex_unlock(&monc->mutex);
 

net/ceph/mon_client.c +7 -10

 	err = wait_for_completion_interruptible(&req->completion);

@@ -684,8 +684,8 @@ static void __resend_generic_request(struct
 ceph_mon_client *monc)
 
 	for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) {
 		req = rb_entry(p, struct ceph_mon_generic_request, node);
-		ceph_con_revoke(monc->con, req->request);
-		ceph_con_send(monc->con, ceph_msg_get(req->request));
+		ceph_con_revoke(&monc->con, req->request);
+		ceph_con_send(&monc->con, ceph_msg_get(req->request));
 	}
 }

@@ -705,7 +705,7 @@ static void delayed_work(struct work_struct *work)
 		__close_session(monc);
 		__open_session(monc);  /* continue hunting */
 	} else {
-		ceph_con_keepalive(monc->con);
+		ceph_con_keepalive(&monc->con);
 
 		__validate_auth(monc);
 

@@ -760,19 +760,16 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct
 ceph_client *cl)
 		goto out;
 
 	/* connection */
-	monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL);
-	if (!monc->con)
-		goto out_monmap;
-	ceph_con_init(&monc->client->msgr, monc->con);
-	monc->con->private = monc;
-	monc->con->ops = &mon_con_ops;
+	ceph_con_init(&monc->client->msgr, &monc->con);
+	monc->con.private = monc;
+	monc->con.ops = &mon_con_ops;
 
 	/* authentication */
 	monc->auth = ceph_auth_init(cl->options->name,
 				    cl->options->key);
 	if (IS_ERR(monc->auth)) {
 		err = PTR_ERR(monc->auth);
-		goto out_con;
+		goto out_monmap;
 	}
 	monc->auth->want_keys =

net/ceph/mon_client.c +0 -0

 		CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |

net/ceph/mon_client.c +2 -4

 out_monmap:
 	kfree(monc->monmap);
 out:

@@ -841,9 +836,7 @@ void ceph_monc_stop(struct ceph_mon_client *monc)
 	mutex_lock(&monc->mutex);
 	__close_session(monc);
 
-	monc->con->private = NULL;
-	monc->con->ops->put(monc->con);
-	monc->con = NULL;
+	monc->con.private = NULL;
 
 	mutex_unlock(&monc->mutex);
 

@@ -1021,7 +1014,7 @@ static void mon_fault(struct ceph_connection *con)
 	if (!monc->hunting)
 		pr_info("mon%d %s session lost, "
 			"hunting for new mon\n", monc->cur_mon,
-			ceph_pr_addr(&monc->con->peer_addr.in_addr));
+			ceph_pr_addr(&monc->con.peer_addr.in_addr));
 
 	__close_session(monc);
 	if (!monc->hunting) {

Signature · 6 lines

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 149 lines · Alex Elder

A monitor client has a pointer to a ceph connection structure in it. This is the only one of the three ceph client types that do it this way; the OSD and MDS clients embed the connection into their main structures. There is always exactly one ceph connection for a monitor client, so there is no need to allocate it separate from the monitor client structure.

So switch the ceph_mon_client structure to embed its ceph_connection structure.

Signed-off-by Alex Elder<[email protected]>

---
  include/linux/ceph/mon_client.h |    2 +-
  net/ceph/mon_client.c           |   47 ++++++++++++++++----------------------
  2 files changed, 21 insertions(+), 28 deletions(-)

include/linux/ceph/mon_client.h +1 -1

index 545f859..2113e38 100644
@@ -70,7 +70,7 @@ struct ceph_mon_client {
  	bool hunting;
  	int cur_mon;                       /* last monitor i contacted */
  	unsigned long sub_sent, sub_renew_after;
-	struct ceph_connection *con;
+	struct ceph_connection con;
  	bool have_fsid;
 
  	/* pending generic requests */

net/ceph/mon_client.c +2 -2

index 704dc95..ac4d6b1 100644
@@ -106,9 +106,9 @@ static void __send_prepared_auth_request(struct
 ceph_mon_client *monc, int len)
  	monc->pending_auth = 1;
  	monc->m_auth->front.iov_len = len;
  	monc->m_auth->hdr.front_len = cpu_to_le32(len);
-	ceph_con_revoke(monc->con, monc->m_auth);
+	ceph_con_revoke(&monc->con, monc->m_auth);
  	ceph_msg_get(monc->m_auth);  /* keep our ref */
-	ceph_con_send(monc->con, monc->m_auth);
+	ceph_con_send(&monc->con, monc->m_auth);
  }
 

/*

net/ceph/mon_client.c +2 -2

@@ -117,8 +117,8 @@ static void __send_prepared_auth_request(struct
 ceph_mon_client *monc, int len)
  static void __close_session(struct ceph_mon_client *monc)
  {
  	dout("__close_session closing mon%d\n", monc->cur_mon);
-	ceph_con_revoke(monc->con, monc->m_auth);
-	ceph_con_close(monc->con);
+	ceph_con_revoke(&monc->con, monc->m_auth);
+	ceph_con_close(&monc->con);
  	monc->cur_mon = -1;
  	monc->pending_auth = 0;

  	ceph_auth_reset(monc->auth);

net/ceph/mon_client.c +6 -6

@@ -142,9 +142,9 @@ static int __open_session(struct ceph_mon_client *monc)
  		monc->want_next_osdmap = !!monc->want_next_osdmap;
 
  		dout("open_session mon%d opening\n", monc->cur_mon);
-		monc->con->peer_name.type = CEPH_ENTITY_TYPE_MON;
-		monc->con->peer_name.num = cpu_to_le64(monc->cur_mon);
-		ceph_con_open(monc->con,
+		monc->con.peer_name.type = CEPH_ENTITY_TYPE_MON;
+		monc->con.peer_name.num = cpu_to_le64(monc->cur_mon);
+		ceph_con_open(&monc->con,
  			&monc->monmap->mon_inst[monc->cur_mon].addr);
 
  		/* initiatiate authentication handshake */

@@ -226,8 +226,8 @@ static void __send_subscribe(struct ceph_mon_client *monc)
 
  		msg->front.iov_len = p - msg->front.iov_base;
  		msg->hdr.front_len = cpu_to_le32(msg->front.iov_len);
-		ceph_con_revoke(monc->con, msg);
-		ceph_con_send(monc->con, ceph_msg_get(msg));
+		ceph_con_revoke(&monc->con, msg);
+		ceph_con_send(&monc->con, ceph_msg_get(msg));
 
  		monc->sub_sent = jiffies | 1;  /* never 0 */
  	}

@@ -247,7 +247,7 @@ static void handle_subscribe_ack(struct ceph_mon_client
 *monc,
  	if (monc->hunting) {
  		pr_info("mon%d %s session established\n",
  			monc->cur_mon,
-			ceph_pr_addr(&monc->con->peer_addr.in_addr));
+			ceph_pr_addr(&monc->con.peer_addr.in_addr));
  		monc->hunting = false;
  	}

  	dout("handle_subscribe_ack after %d seconds\n", seconds);

net/ceph/mon_client.c +1 -1

@@ -461,7 +461,7 @@ static int do_generic_request(struct ceph_mon_client
 *monc,
  	req->request->hdr.tid = cpu_to_le64(req->tid);
  	__insert_generic_request(monc, req);
  	monc->num_generic_requests++;
-	ceph_con_send(monc->con, ceph_msg_get(req->request));
+	ceph_con_send(&monc->con, ceph_msg_get(req->request));
  	mutex_unlock(&monc->mutex);
 

  	err = wait_for_completion_interruptible(&req->completion);

net/ceph/mon_client.c +7 -10

@@ -684,8 +684,8 @@ static void __resend_generic_request(struct
 ceph_mon_client *monc)
 
  	for (p = rb_first(&monc->generic_request_tree); p; p = rb_next(p)) {
  		req = rb_entry(p, struct ceph_mon_generic_request, node);
-		ceph_con_revoke(monc->con, req->request);
-		ceph_con_send(monc->con, ceph_msg_get(req->request));
+		ceph_con_revoke(&monc->con, req->request);
+		ceph_con_send(&monc->con, ceph_msg_get(req->request));
  	}
  }

@@ -705,7 +705,7 @@ static void delayed_work(struct work_struct *work)
  		__close_session(monc);
  		__open_session(monc);  /* continue hunting */
  	} else {
-		ceph_con_keepalive(monc->con);
+		ceph_con_keepalive(&monc->con);
 
  		__validate_auth(monc);
 

@@ -760,19 +760,16 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct
 ceph_client *cl)
  		goto out;
 
  	/* connection */
-	monc->con = kmalloc(sizeof(*monc->con), GFP_KERNEL);
-	if (!monc->con)
-		goto out_monmap;
-	ceph_con_init(&monc->client->msgr, monc->con);
-	monc->con->private = monc;
-	monc->con->ops =&mon_con_ops;
+	ceph_con_init(&monc->client->msgr,&monc->con);
+	monc->con.private = monc;
+	monc->con.ops =&mon_con_ops;
 
  	/* authentication */
  	monc->auth = ceph_auth_init(cl->options->name,
  				    cl->options->key);
  	if (IS_ERR(monc->auth)) {
  		err = PTR_ERR(monc->auth);
-		goto out_con;
+		goto out_monmap;
  	}
  	monc->auth->want_keys =

CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |

net/ceph/mon_client.c out_subscribe_ack: · line 828

Quoted 5 lines

net/ceph/mon_client.c +0 -2

@@ -824,8 +821,6 @@ out_subscribe_ack:
  	ceph_msg_put(monc->m_subscribe_ack);
  out_auth:
  	ceph_auth_destroy(monc->auth);
-out_con:
-	monc->con->ops->put(monc->con);

AH!

This reminds me, these connections need to be refcounted. There's a ->get() and ->put() op defined so that you can refcount the containing structure. That means that this patch needs to alo change

static const struct ceph_connection_operations mon_con_ops = {
.get = ceph_con_get,
.put = ceph_con_put,

in mon_client.c. Hopefully the mon_client itself is refcounted, *or* we can ensure that it won't go away before the msgr workqueue is drained and the get/put ops can turn to no-ops.

Also: when poking around, I noticed that ceph_con_get() and put() are called directly from osd_client.c... that's a bug! Those connections have a get and put op defined that twiddles the containing ceph_osd struct's ref count.

I pushed several patches to your latest (wip-messenger-2) branch that fix these issues. Compile tested only! The first should probably be folded into this one, the others follow.

Quoted 33 lines

net/ceph/mon_client.c +2 -5

  out_monmap:
  	kfree(monc->monmap);
  out:

@@ -841,9 +836,7 @@ void ceph_monc_stop(struct ceph_mon_client *monc)
  	mutex_lock(&monc->mutex);
  	__close_session(monc);
 
-	monc->con->private = NULL;
-	monc->con->ops->put(monc->con);
-	monc->con = NULL;
+	monc->con.private = NULL;
 
  	mutex_unlock(&monc->mutex);
 

@@ -1021,7 +1014,7 @@ static void mon_fault(struct ceph_connection *con)
  	if (!monc->hunting)
  		pr_info("mon%d %s session lost, "
  			"hunting for new mon\n", monc->cur_mon,
-			ceph_pr_addr(&monc->con->peer_addr.in_addr));
+			ceph_pr_addr(&monc->con.peer_addr.in_addr));
 
  	__close_session(monc);
  	if (!monc->hunting) {
--

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 8 lines · Sage Weil

Quoted 7 lines · Alex Elder

A monitor client has a pointer to a ceph connection structure in it. This is the only one of the three ceph client types that do it this way; the OSD and MDS clients embed the connection into their main structures. There is always exactly one ceph connection for a monitor client, so there is no need to allocate it separate from the monitor client structure.

Quoted 4 lines

Quoted 4 lines

So switch the ceph_mon_client structure to embed its ceph_connection structure.

Signed-off-by Alex Elder<[email protected]>

Quoted 19 lines

Quoted 12 lines

/* authentication */
monc->auth = ceph_auth_init(cl->options->name,
cl->options->key);
if (IS_ERR(monc->auth)) {
err = PTR_ERR(monc->auth);
- goto out_con;
+ goto out_monmap;
}
monc->auth->want_keys =
CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |

net/ceph/mon_client.c +0 -0

@@ -824,8 +821,6 @@ out_subscribe_ack:
 ceph_msg_put(monc->m_subscribe_ack);

Quoted 4 lines

out_auth:
ceph_auth_destroy(monc->auth);
-out_con:
- monc->con->ops->put(monc->con);

AH!

Quoted 3 lines

This reminds me, these connections need to be refcounted. There's a ->get() and ->put() op defined so that you can refcount the containing structure. That means that this patch needs to alo change

Quoted 7 lines

static const struct ceph_connection_operations mon_con_ops = { .get = ceph_con_get,
.put = ceph_con_put,

in mon_client.c. Hopefully the mon_client itself is refcounted, *or* we can ensure that it won't go away before the msgr workqueue is drained and the get/put ops can turn to no-ops.

Earlier I looked at the ref counting stuff a bit and stopped myself from going off on that tangent. But it didn't look like it was used consistently and made a note to myself to revisit it.

Quoted 8 lines

Also: when poking around, I noticed that ceph_con_get() and put() are called directly from osd_client.c... that's a bug! Those connections have a get and put op defined that twiddles the containing ceph_osd struct's ref count.

I pushed several patches to your latest (wip-messenger-2) branch that fix these issues. Compile tested only! The first should probably be folded into this one, the others follow.

I'll look at your patches and incorporate them as appropriate. But at the moment I don't see them; whenever you are back online again perhaps you'll send me a link.

-Alex

Quoted 34 lines

Quoted 33 lines

out_monmap:
kfree(monc->monmap);
out:

net/ceph/mon_client.c +1 -3

@@ -841,9 +836,7 @@ void ceph_monc_stop(struct ceph_mon_client *monc)
 mutex_lock(&monc->mutex);
 __close_session(monc);
 
- monc->con->private = NULL;
- monc->con->ops->put(monc->con);
- monc->con = NULL;
+ monc->con.private = NULL;
 
 mutex_unlock(&monc->mutex);
 

@@ -1021,7 +1014,7 @@ static void mon_fault(struct ceph_connection *con)
 if (!monc->hunting)

pr_info("mon%d %s session lost, "
"hunting for new mon\n", monc->cur_mon,
- ceph_pr_addr(&monc->con->peer_addr.in_addr));
+ ceph_pr_addr(&monc->con.peer_addr.in_addr));

__close_session(monc);
if (!monc->hunting) {
--
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 9 lines · Alex Elder

Quoted 8 lines · Sage Weil

Quoted 7 lines · Alex Elder

A monitor client has a pointer to a ceph connection structure in it. This is the only one of the three ceph client types that do it this way; the OSD and MDS clients embed the connection into their main structures. There is always exactly one ceph connection for a monitor client, so there is no need to allocate it separate from the monitor client structure.

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

So switch the ceph_mon_client structure to embed its ceph_connection structure.

Signed-off-by Alex Elder<[email protected]>

. . .

Quoted 19 lines

Quoted 19 lines

Quoted 12 lines

/* authentication */
monc->auth = ceph_auth_init(cl->options->name,
cl->options->key);
if (IS_ERR(monc->auth)) {
err = PTR_ERR(monc->auth);
- goto out_con;
+ goto out_monmap;
}
monc->auth->want_keys =
CEPH_ENTITY_TYPE_AUTH | CEPH_ENTITY_TYPE_MON |

net/ceph/mon_client.c +0 -0

@@ -824,8 +821,6 @@ out_subscribe_ack:
 ceph_msg_put(monc->m_subscribe_ack);

Quoted 4 lines

out_auth:
ceph_auth_destroy(monc->auth);
-out_con:
- monc->con->ops->put(monc->con);

AH!

Quoted 3 lines

Quoted 3 lines

This reminds me, these connections need to be refcounted. There's a ->get() and ->put() op defined so that you can refcount the containing structure. That means that this patch needs to alo change

Looking at this again. Why do they need to be refcounted? If this patch is correct in embedding the connection into the containing structure, then the last reference to the containing structure is coincident with with the last reference to the connection. And the other connections are already embedded into other containing structures.

So--again assuming it's OK to embed the connection--I would rather see the ->get and ->put methods for the connection go away entirely and have the containing structure take care of its own damned ref counting...

This actually gets into another thing I wanted to do anyway (while digging through raw memory trying to figure out what's going on). I want every ceph_message to point back to the connection it is associated with. That way there's no need for the OSD (for example) to keep track of the connection--a revoke is simply an operation on the message, which would already know the connection from which it is being revoked.

If you think the above approach is good I'll gladly do it now rather than later. I think it might eliminate the need for any need to reference count the connections.

Anyway, when I'm done (if I ever finish!) the state of the connection will tell you whether any legitimate uses of a connection remain.

-Alex

Quoted 22 lines

Quoted 7 lines

static const struct ceph_connection_operations mon_con_ops = { .get = ceph_con_get,
.put = ceph_con_put,

in mon_client.c. Hopefully the mon_client itself is refcounted, *or* we can ensure that it won't go away before the msgr workqueue is drained and the get/put ops can turn to no-ops.

Earlier I looked at the ref counting stuff a bit and stopped myself from going off on that tangent. But it didn't look like it was used consistently and made a note to myself to revisit it.

Quoted 8 lines

Also: when poking around, I noticed that ceph_con_get() and put() are called directly from osd_client.c... that's a bug! Those connections have a get and put op defined that twiddles the containing ceph_osd struct's ref count.

I pushed several patches to your latest (wip-messenger-2) branch that fix these issues. Compile tested only! The first should probably be folded into this one, the others follow.

Quoted 41 lines

I'll look at your patches and incorporate them as appropriate. But at the moment I don't see them; whenever you are back online again perhaps you'll send me a link.

-Alex

Quoted 34 lines

Quoted 33 lines

out_monmap:
kfree(monc->monmap);
out:

net/ceph/mon_client.c +1 -3

@@ -841,9 +836,7 @@ void ceph_monc_stop(struct ceph_mon_client *monc)
 mutex_lock(&monc->mutex);
 __close_session(monc);
 
- monc->con->private = NULL;
- monc->con->ops->put(monc->con);
- monc->con = NULL;
+ monc->con.private = NULL;
 
 mutex_unlock(&monc->mutex);
 

@@ -1021,7 +1014,7 @@ static void mon_fault(struct ceph_connection *con)
 if (!monc->hunting)

pr_info("mon%d %s session lost, "
"hunting for new mon\n", monc->cur_mon,
- ceph_pr_addr(&monc->con->peer_addr.in_addr));
+ ceph_pr_addr(&monc->con.peer_addr.in_addr));

__close_session(monc);
if (!monc->hunting) {
--
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

The problem is that socket events queue work, which can take a while, and race with, say, osd_client getting an osdmap and dropping it's struct ceph_osd. The ->get and ->put ops just twiddle the containing struct's refcount, in that case, so the con_work will find the (now closed) ceph_connection and do nothing...

Quoted 5 lines · Sage Weil

The problem is that socket events queue work, which can take a while, and race with, say, osd_client getting an osdmap and dropping it's struct ceph_osd. The ->get and ->put ops just twiddle the containing struct's refcount, in that case, so the con_work will find the (now closed) ceph_connection and do nothing...

I think you're saying that the connection (or its socket) needs to be protected from its containing structure going away. So the connection needs to hold a reference to its container. If that's the case then the disposal of the ceph_osd needs to clean up the connection fully before it goes away.

Anyway, I think I see why there might be a need for the ref counts and they obviously won't go away if they're needed...

-Alex
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Also: when poking around, I noticed that ceph_con_get() and put() are called directly from osd_client.c... that's a bug! Those connections have a get and put op defined that twiddles the containing ceph_osd struct's ref count.

Quoted 4 lines · Sage Weil

Also: when poking around, I noticed that ceph_con_get() and put() are called directly from osd_client.c... that's a bug! Those connections have a get and put op defined that twiddles the containing ceph_osd struct's ref count.

So are you saying that the calls in "osd_client.c" to ceph_con_get() and ceph_con_put() should instead be calls to get_osd_con() and put_osd_con(), respectively? (Or more generally con->ops->get() and con->ops->put()?)

A ceph_connection holds a mixture of connection state (as in "state machine" state) and connection flags in a single "state" field. To make the distinction more clear, define a new "flags" field and use it rather than the "state" field to hold Boolean flag values.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/messenger.h |   18 +++++++++----
 net/ceph/messenger.c           |   50
++++++++++++++++++++--------------------
 2 files changed, 37 insertions(+), 31 deletions(-)

include/linux/ceph/messenger.h +10 -3

index 3fbd4be..920235e 100644
@@ -103,20 +103,25 @@ struct ceph_msg_pos {
 #define MAX_DELAY_INTERVAL	(5 * 60 * HZ)
 
 /*
- * ceph_connection state bit flags
+ * ceph_connection flag bits
  */
+
 #define LOSSYTX         0  /* we can close channel or drop messages on errors
 */
-#define CONNECTING	1
-#define NEGOTIATING	2
 #define KEEPALIVE_PENDING      3
 #define WRITE_PENDING	4  /* we have data ready to send */
+#define SOCK_CLOSED	11 /* socket state changed to closed */
+#define BACKOFF         15
+
+/*
+ * ceph_connection states
+ */
+#define CONNECTING	1
+#define NEGOTIATING	2
 #define STANDBY		8  /* no outgoing messages, socket closed.  we

keep

/*
* A single connection with another host.

include/linux/ceph/messenger.h +2 -1

@@ -133,7 +138,8 @@ struct ceph_connection {
 
 	struct ceph_messenger *msgr;
 	struct socket *sock;
-	unsigned long state;	/* connection state (see flags above) */
+	unsigned long flags;
+	unsigned long state;
 	const char *error_msg;  /* error message, if any */
 
 	struct ceph_entity_addr peer_addr; /* peer address */

net/ceph/messenger.c +6 -6

index 19f1948..29055df 100644
@@ -176,7 +176,7 @@ static void ceph_sock_write_space(struct sock *sk)
 	 * buffer. See net/ipv4/tcp_input.c:tcp_check_space()
 	 * and net/core/stream.c:sk_stream_write_space().
 	 */
-	if (test_bit(WRITE_PENDING, &con->state)) {
+	if (test_bit(WRITE_PENDING, &con->flags)) {
 		if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
 			dout("%s %p queueing write work\n", __func__, con);
 			clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);

@@ -203,7 +203,7 @@ static void ceph_sock_state_change(struct sock *sk)
 		dout("%s TCP_CLOSE\n", __func__);
 	case TCP_CLOSE_WAIT:
 		dout("%s TCP_CLOSE_WAIT\n", __func__);
-		if (test_and_set_bit(SOCK_CLOSED, &con->state) == 0) {
+		if (test_and_set_bit(SOCK_CLOSED, &con->flags) == 0) {
 			if (test_bit(CONNECTING, &con->state))
 				con->error_msg = "connection failed";
 			else

@@ -393,9 +393,9 @@ void ceph_con_close(struct ceph_connection *con)
 	     ceph_pr_addr(&con->peer_addr.in_addr));
 	set_bit(CLOSED, &con->state);  /* in case there's queued work */
 	clear_bit(STANDBY, &con->state);  /* avoid connect_seq bump */
-	clear_bit(LOSSYTX, &con->state);  /* so we retry next connect */
-	clear_bit(KEEPALIVE_PENDING, &con->state);
-	clear_bit(WRITE_PENDING, &con->state);
+	clear_bit(LOSSYTX, &con->flags);  /* so we retry next connect */
+	clear_bit(KEEPALIVE_PENDING, &con->flags);
+	clear_bit(WRITE_PENDING, &con->flags);
 	mutex_lock(&con->mutex);
 	reset_connection(con);
 	con->peer_global_seq = 0;

@@ -612,7 +612,7 @@ static void prepare_write_message(struct ceph_connection
 *con)
 		prepare_write_message_footer(con);
 	}
 
-	set_bit(WRITE_PENDING, &con->state);
+	set_bit(WRITE_PENDING, &con->flags);
 }
 

/*

net/ceph/messenger.c +2 -2

@@ -633,7 +633,7 @@ static void prepare_write_ack(struct ceph_connection *con)
 				&con->out_temp_ack);
 
 	con->out_more = 1;  /* more will follow.. eventually.. */
-	set_bit(WRITE_PENDING, &con->state);
+	set_bit(WRITE_PENDING, &con->flags);
 }
 
 /*

@@ -644,7 +644,7 @@ static void prepare_write_keepalive(struct ceph_connection
 *con)
 	dout("prepare_write_keepalive %p\n", con);
 	con_out_kvec_reset(con);
 	con_out_kvec_add(con, sizeof (tag_keepalive), &tag_keepalive);
-	set_bit(WRITE_PENDING, &con->state);
+	set_bit(WRITE_PENDING, &con->flags);
 }
 

/*

net/ceph/messenger.c +1 -1

@@ -673,7 +673,7 @@ static struct ceph_auth_handshake
 *get_connect_authorizer(struct ceph_connection
 
 	if (IS_ERR(auth))
 		return auth;
-	if (test_bit(CLOSED, &con->state) || test_bit(OPENING, &con->state))
+	if (test_bit(CLOSED, &con->state) || test_bit(OPENING, &con->flags))
 		return ERR_PTR(-EAGAIN);
 

net/ceph/messenger.c +1 -1

 	con->auth_reply_buf = auth->authorizer_reply_buf;

@@ -693,7 +693,7 @@ static void prepare_write_banner(struct ceph_connection
 *con)
 					&con->msgr->my_enc_addr);
 
 	con->out_more = 0;
-	set_bit(WRITE_PENDING, &con->state);
+	set_bit(WRITE_PENDING, &con->flags);
 }
 

static int prepare_write_connect(struct ceph_connection *con)

net/ceph/messenger.c +1 -1

@@ -743,7 +743,7 @@ static int prepare_write_connect(struct ceph_connection
 *con)
 					auth->authorizer_buf);
 
 	con->out_more = 0;
-	set_bit(WRITE_PENDING, &con->state);
+	set_bit(WRITE_PENDING, &con->flags);
 
 	return 0;

}

net/ceph/messenger.c +8 -8

@@ -1490,7 +1490,7 @@ static int process_connect(struct ceph_connection *con)
 			le32_to_cpu(con->in_reply.connect_seq));
 
 		if (con->in_reply.flags & CEPH_MSG_CONNECT_LOSSY)
-			set_bit(LOSSYTX, &con->state);
+			set_bit(LOSSYTX, &con->flags);
 
 		prepare_read_tag(con);
 		break;

@@ -1931,14 +1931,14 @@ do_next:
 			prepare_write_ack(con);
 			goto more;
 		}
-		if (test_and_clear_bit(KEEPALIVE_PENDING, &con->state)) {
+		if (test_and_clear_bit(KEEPALIVE_PENDING, &con->flags)) {
 			prepare_write_keepalive(con);
 			goto more;
 		}
 	}
 
 	/* Nothing to do! */
-	clear_bit(WRITE_PENDING, &con->state);
+	clear_bit(WRITE_PENDING, &con->flags);
 	dout("try_write nothing else to write.\n");
 	ret = 0;
 out:

@@ -2104,7 +2104,7 @@ static void con_work(struct work_struct *work)
 
 	mutex_lock(&con->mutex);
 restart:
-	if (test_and_clear_bit(BACKOFF, &con->state)) {
+	if (test_and_clear_bit(BACKOFF, &con->flags)) {
 		dout("con_work %p backing off\n", con);
 		if (queue_delayed_work(ceph_msgr_wq, &con->work,
 				       round_jiffies_relative(con->delay))) {

@@ -2133,7 +2133,7 @@ restart:
 		con_close_socket(con);
 	}
 
-	if (test_and_clear_bit(SOCK_CLOSED, &con->state))
+	if (test_and_clear_bit(SOCK_CLOSED, &con->flags))
 		goto fault;
 
 	ret = try_read(con);

@@ -2172,7 +2172,7 @@ static void ceph_fault(struct ceph_connection *con)
 	dout("fault %p state %lu to peer %s\n",
 	     con, con->state, ceph_pr_addr(&con->peer_addr.in_addr));
 
-	if (test_bit(LOSSYTX, &con->state)) {
+	if (test_bit(LOSSYTX, &con->flags)) {
 		dout("fault on LOSSYTX channel\n");
 		goto out;
 	}

@@ -2194,9 +2194,9 @@ static void ceph_fault(struct ceph_connection *con)
 	/* If there are no messages queued or keepalive pending, place
 	 * the connection in a STANDBY state */
 	if (list_empty(&con->out_queue) &&
-	    !test_bit(KEEPALIVE_PENDING, &con->state)) {
+	    !test_bit(KEEPALIVE_PENDING, &con->flags)) {
 		dout("fault %p setting STANDBY clearing WRITE_PENDING\n",
 con);
-		clear_bit(WRITE_PENDING, &con->state);
+		clear_bit(WRITE_PENDING, &con->flags);
 		set_bit(STANDBY, &con->state);
 	} else {

net/ceph/messenger.c +4 -4

 		/* retry after a delay. */

@@ -2220,7 +2220,7 @@ static void ceph_fault(struct ceph_connection *con)
 			 * that when con_work restarts we schedule the
 			 * delay then.
 			 */
-			set_bit(BACKOFF, &con->state);
+			set_bit(BACKOFF, &con->flags);
 		}
 	}
 

@@ -2276,8 +2276,8 @@ static void clear_standby(struct ceph_connection *con)
 		mutex_lock(&con->mutex);
 		dout("clear_standby %p and ++connect_seq\n", con);
 		con->connect_seq++;
-		WARN_ON(test_bit(WRITE_PENDING, &con->state));
-		WARN_ON(test_bit(KEEPALIVE_PENDING, &con->state));
+		WARN_ON(test_bit(WRITE_PENDING, &con->flags));
+		WARN_ON(test_bit(KEEPALIVE_PENDING, &con->flags));
 		mutex_unlock(&con->mutex);
 	}
 }

@@ -2315,7 +2315,7 @@ void ceph_con_send(struct ceph_connection *con, struct
 ceph_msg *msg)
 	/* if there wasn't anything waiting to send before, queue
 	 * new work */
 	clear_standby(con);
-	if (test_and_set_bit(WRITE_PENDING, &con->state) == 0)
+	if (test_and_set_bit(WRITE_PENDING, &con->flags) == 0)
 		queue_con(con);
 }

EXPORT_SYMBOL(ceph_con_send);

net/ceph/messenger.c +2 -2

@@ -2382,8 +2382,8 @@ void ceph_con_keepalive(struct ceph_connection *con)
 {
 	dout("con_keepalive %p\n", con);
 	clear_standby(con);
-	if (test_and_set_bit(KEEPALIVE_PENDING, &con->state) == 0 &&
-	    test_and_set_bit(WRITE_PENDING, &con->state) == 0)
+	if (test_and_set_bit(KEEPALIVE_PENDING, &con->flags) == 0 &&
+	    test_and_set_bit(WRITE_PENDING, &con->flags) == 0)
 		queue_con(con);
 }
 EXPORT_SYMBOL(ceph_con_keepalive);

Signature · 6 lines

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 40 lines · Alex Elder

A ceph_connection holds a mixture of connection state (as in "state machine" state) and connection flags in a single "state" field. To make the distinction more clear, define a new "flags" field and use it rather than the "state" field to hold Boolean flag values.

Signed-off-by Alex Elder<[email protected]>

---
  include/linux/ceph/messenger.h |   18 +++++++++----
  net/ceph/messenger.c           |   50
++++++++++++++++++++--------------------
  2 files changed, 37 insertions(+), 31 deletions(-)

include/linux/ceph/messenger.h +10 -3

index 3fbd4be..920235e 100644
@@ -103,20 +103,25 @@ struct ceph_msg_pos {
  #define MAX_DELAY_INTERVAL	(5 * 60 * HZ)
 
  /*
- * ceph_connection state bit flags
+ * ceph_connection flag bits
   */
+
  #define LOSSYTX         0  /* we can close channel or drop messages on errors
 */
-#define CONNECTING	1
-#define NEGOTIATING	2
  #define KEEPALIVE_PENDING      3
  #define WRITE_PENDING	4  /* we have data ready to send */
+#define SOCK_CLOSED	11 /* socket state changed to closed */
+#define BACKOFF         15
+
+/*
+ * ceph_connection states
+ */
+#define CONNECTING	1
+#define NEGOTIATING	2
  #define STANDBY		8  /* no outgoing messages, socket closed.  we

keep

include/linux/ceph/messenger.h

Quoted 6 lines

* the ceph_connection around to maintain shared
* state with the peer. */
#define CLOSED 10 /* we've closed the connection */

include/linux/ceph/messenger.h +0 -2

-#define SOCK_CLOSED	11 /* socket state changed to closed */
  #define OPENING         13 /* open connection w/ (possibly new) peer */
-#define BACKOFF         15

Later it might be work prefixing these with FLAG_ and/or STATE_.

Reviewed-by Sage Weil<[email protected]>

Quoted 223 lines

/*
* A single connection with another host.

include/linux/ceph/messenger.h +2 -1

@@ -133,7 +138,8 @@ struct ceph_connection {
 
  	struct ceph_messenger *msgr;
  	struct socket *sock;
-	unsigned long state;	/* connection state (see flags above) */
+	unsigned long flags;
+	unsigned long state;
  	const char *error_msg;  /* error message, if any */
 
  	struct ceph_entity_addr peer_addr; /* peer address */

net/ceph/messenger.c +6 -6

index 19f1948..29055df 100644
@@ -176,7 +176,7 @@ static void ceph_sock_write_space(struct sock *sk)
  	 * buffer. See net/ipv4/tcp_input.c:tcp_check_space()
  	 * and net/core/stream.c:sk_stream_write_space().
  	 */
-	if (test_bit(WRITE_PENDING,&con->state)) {
+	if (test_bit(WRITE_PENDING,&con->flags)) {
  		if (sk_stream_wspace(sk)>= sk_stream_min_wspace(sk)) {
  			dout("%s %p queueing write work\n", __func__, con);
  			clear_bit(SOCK_NOSPACE,&sk->sk_socket->flags);

@@ -203,7 +203,7 @@ static void ceph_sock_state_change(struct sock *sk)
  		dout("%s TCP_CLOSE\n", __func__);
  	case TCP_CLOSE_WAIT:
  		dout("%s TCP_CLOSE_WAIT\n", __func__);
-		if (test_and_set_bit(SOCK_CLOSED,&con->state) == 0) {
+		if (test_and_set_bit(SOCK_CLOSED,&con->flags) == 0) {
  			if (test_bit(CONNECTING,&con->state))
  				con->error_msg = "connection failed";
  			else

@@ -393,9 +393,9 @@ void ceph_con_close(struct ceph_connection *con)
  	     ceph_pr_addr(&con->peer_addr.in_addr));
  	set_bit(CLOSED,&con->state);  /* in case there's queued work */
  	clear_bit(STANDBY,&con->state);  /* avoid connect_seq bump */
-	clear_bit(LOSSYTX,&con->state);  /* so we retry next connect */
-	clear_bit(KEEPALIVE_PENDING,&con->state);
-	clear_bit(WRITE_PENDING,&con->state);
+	clear_bit(LOSSYTX,&con->flags);  /* so we retry next connect */
+	clear_bit(KEEPALIVE_PENDING,&con->flags);
+	clear_bit(WRITE_PENDING,&con->flags);
  	mutex_lock(&con->mutex);
  	reset_connection(con);
  	con->peer_global_seq = 0;

@@ -612,7 +612,7 @@ static void prepare_write_message(struct ceph_connection
 *con)
  		prepare_write_message_footer(con);
  	}
 
-	set_bit(WRITE_PENDING,&con->state);
+	set_bit(WRITE_PENDING,&con->flags);
  }
 

/*

net/ceph/messenger.c +2 -2

@@ -633,7 +633,7 @@ static void prepare_write_ack(struct ceph_connection *con)
  				&con->out_temp_ack);
 
  	con->out_more = 1;  /* more will follow.. eventually.. */
-	set_bit(WRITE_PENDING,&con->state);
+	set_bit(WRITE_PENDING,&con->flags);
  }
 
  /*

@@ -644,7 +644,7 @@ static void prepare_write_keepalive(struct ceph_connection
 *con)
  	dout("prepare_write_keepalive %p\n", con);
  	con_out_kvec_reset(con);
  	con_out_kvec_add(con, sizeof (tag_keepalive),&tag_keepalive);
-	set_bit(WRITE_PENDING,&con->state);
+	set_bit(WRITE_PENDING,&con->flags);
  }
 

/*

net/ceph/messenger.c +1 -1

@@ -673,7 +673,7 @@ static struct ceph_auth_handshake
 *get_connect_authorizer(struct ceph_connection
 
  	if (IS_ERR(auth))
  		return auth;
-	if (test_bit(CLOSED,&con->state) || test_bit(OPENING,&con->state))
+	if (test_bit(CLOSED,&con->state) || test_bit(OPENING,&con->flags))
  		return ERR_PTR(-EAGAIN);
 

  	con->auth_reply_buf = auth->authorizer_reply_buf;

net/ceph/messenger.c +1 -1

@@ -693,7 +693,7 @@ static void prepare_write_banner(struct ceph_connection
 *con)
  					&con->msgr->my_enc_addr);
 
  	con->out_more = 0;
-	set_bit(WRITE_PENDING,&con->state);
+	set_bit(WRITE_PENDING,&con->flags);
  }
 

static int prepare_write_connect(struct ceph_connection *con)

net/ceph/messenger.c +1 -1

@@ -743,7 +743,7 @@ static int prepare_write_connect(struct ceph_connection
 *con)
  					auth->authorizer_buf);
 
  	con->out_more = 0;
-	set_bit(WRITE_PENDING,&con->state);
+	set_bit(WRITE_PENDING,&con->flags);
 
  	return 0;

  }

net/ceph/messenger.c +8 -8

@@ -1490,7 +1490,7 @@ static int process_connect(struct ceph_connection *con)
  			le32_to_cpu(con->in_reply.connect_seq));
 
  		if (con->in_reply.flags&  CEPH_MSG_CONNECT_LOSSY)
-			set_bit(LOSSYTX,&con->state);
+			set_bit(LOSSYTX,&con->flags);
 
  		prepare_read_tag(con);
  		break;

@@ -1931,14 +1931,14 @@ do_next:
  			prepare_write_ack(con);
  			goto more;
  		}
-		if (test_and_clear_bit(KEEPALIVE_PENDING,&con->state)) {
+		if (test_and_clear_bit(KEEPALIVE_PENDING,&con->flags)) {
  			prepare_write_keepalive(con);
  			goto more;
  		}
  	}
 
  	/* Nothing to do! */
-	clear_bit(WRITE_PENDING,&con->state);
+	clear_bit(WRITE_PENDING,&con->flags);
  	dout("try_write nothing else to write.\n");
  	ret = 0;
  out:

@@ -2104,7 +2104,7 @@ static void con_work(struct work_struct *work)
 
  	mutex_lock(&con->mutex);
  restart:
-	if (test_and_clear_bit(BACKOFF,&con->state)) {
+	if (test_and_clear_bit(BACKOFF,&con->flags)) {
  		dout("con_work %p backing off\n", con);
  		if (queue_delayed_work(ceph_msgr_wq,&con->work,
  				       round_jiffies_relative(con->delay))) {

@@ -2133,7 +2133,7 @@ restart:
  		con_close_socket(con);
  	}
 
-	if (test_and_clear_bit(SOCK_CLOSED,&con->state))
+	if (test_and_clear_bit(SOCK_CLOSED,&con->flags))
  		goto fault;
 
  	ret = try_read(con);

@@ -2172,7 +2172,7 @@ static void ceph_fault(struct ceph_connection *con)
  	dout("fault %p state %lu to peer %s\n",
  	     con, con->state, ceph_pr_addr(&con->peer_addr.in_addr));
 
-	if (test_bit(LOSSYTX,&con->state)) {
+	if (test_bit(LOSSYTX,&con->flags)) {
  		dout("fault on LOSSYTX channel\n");
  		goto out;
  	}

@@ -2194,9 +2194,9 @@ static void ceph_fault(struct ceph_connection *con)
  	/* If there are no messages queued or keepalive pending, place
  	 * the connection in a STANDBY state */
  	if (list_empty(&con->out_queue)&&
-	    !test_bit(KEEPALIVE_PENDING,&con->state)) {
+	    !test_bit(KEEPALIVE_PENDING,&con->flags)) {
  		dout("fault %p setting STANDBY clearing WRITE_PENDING\n",
 con);
-		clear_bit(WRITE_PENDING,&con->state);
+		clear_bit(WRITE_PENDING,&con->flags);
  		set_bit(STANDBY,&con->state);
  	} else {

/* retry after a delay. */

net/ceph/messenger.c +4 -4

@@ -2220,7 +2220,7 @@ static void ceph_fault(struct ceph_connection *con)
  			 * that when con_work restarts we schedule the
  			 * delay then.
  			 */
-			set_bit(BACKOFF,&con->state);
+			set_bit(BACKOFF,&con->flags);
  		}
  	}
 

@@ -2276,8 +2276,8 @@ static void clear_standby(struct ceph_connection *con)
  		mutex_lock(&con->mutex);
  		dout("clear_standby %p and ++connect_seq\n", con);
  		con->connect_seq++;
-		WARN_ON(test_bit(WRITE_PENDING,&con->state));
-		WARN_ON(test_bit(KEEPALIVE_PENDING,&con->state));
+		WARN_ON(test_bit(WRITE_PENDING,&con->flags));
+		WARN_ON(test_bit(KEEPALIVE_PENDING,&con->flags));
  		mutex_unlock(&con->mutex);
  	}
  }

@@ -2315,7 +2315,7 @@ void ceph_con_send(struct ceph_connection *con, struct
 ceph_msg *msg)
  	/* if there wasn't anything waiting to send before, queue
  	 * new work */
  	clear_standby(con);
-	if (test_and_set_bit(WRITE_PENDING,&con->state) == 0)
+	if (test_and_set_bit(WRITE_PENDING,&con->flags) == 0)
  		queue_con(con);
  }

  EXPORT_SYMBOL(ceph_con_send);

net/ceph/messenger.c +2 -3

@@ -2382,8 +2382,8 @@ void ceph_con_keepalive(struct ceph_connection *con)
  {
  	dout("con_keepalive %p\n", con);
  	clear_standby(con);
-	if (test_and_set_bit(KEEPALIVE_PENDING,&con->state) == 0&&
-	    test_and_set_bit(WRITE_PENDING,&con->state) == 0)
+	if (test_and_set_bit(KEEPALIVE_PENDING,&con->flags) == 0&&
+	    test_and_set_bit(WRITE_PENDING,&con->flags) == 0)
  		queue_con(con);
  }
  EXPORT_SYMBOL(ceph_con_keepalive);
--

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Start explicitly keeping track of the state of a ceph connection's socket, separate from the state of the connection itself. Create placeholder functions to encapsulate the state transitions.

    --------
    | NEW* |  transient initial state
    --------
        | con_sock_state_init()
        v
    ----------
    | CLOSED |  initialized, but no socket (and no
    ----------  TCP connection)
     ^      \
     |       \ con_sock_state_connecting()
     |        ----------------------
     |                              \
     + con_sock_state_closed()       \
     |\                               \
     | \                               \
     |  -----------                     \
     |  | CLOSING |  socket event;       \
     |  -----------  await close          \
     |       ^                            |
     |       |                            |
     |       + con_sock_state_closing()   |
     |      / \                           |
     |     /   ---------------            |
     |    /                   \           v

     |   /                    --------------
     |  /    -----------------| CONNECTING |  socket created, TCP
     |  |   /                 --------------  connect initiated
     |  |   | con_sock_state_connected()
     |  |   v
    -------------
    | CONNECTED |  TCP connection established
    -------------

Make the socket state an atomic variable, reinforcing that it's a distinct transtion with no possible "intermediate/both" states. This is almost certainly overkill at this point, though the transitions into CONNECTED and CLOSING state do get called via socket callback (the rest of the transitions occur with the connection mutex held). We can back out the atomicity later.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/messenger.h |    8 ++++-
 net/ceph/messenger.c           |   63
++++++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 2 deletions(-)

include/linux/ceph/messenger.h +6 -2

index 920235e..5e852f4 100644
@@ -137,14 +137,18 @@ struct ceph_connection {
 	const struct ceph_connection_operations *ops;
 
 	struct ceph_messenger *msgr;
+
+	atomic_t sock_state;
 	struct socket *sock;
+	struct ceph_entity_addr peer_addr; /* peer address */
+	struct ceph_entity_addr peer_addr_for_me;
+
 	unsigned long flags;
 	unsigned long state;
 	const char *error_msg;  /* error message, if any */
 
-	struct ceph_entity_addr peer_addr; /* peer address */
 	struct ceph_entity_name peer_name; /* peer name */
-	struct ceph_entity_addr peer_addr_for_me;
+
 	unsigned peer_features;
 	u32 connect_seq;      /* identify the most recent connection
 				 attempt for this connection, client */

net/ceph/messenger.c +8 -0

index 29055df..7e11b07 100644
@@ -29,6 +29,14 @@
  * the sender.
  */
 
+/* State values for ceph_connection->sock_state; NEW is assumed to be 0 */
+
+#define CON_SOCK_STATE_NEW		0	/* -> CLOSED */
+#define CON_SOCK_STATE_CLOSED		1	/* -> CONNECTING */
+#define CON_SOCK_STATE_CONNECTING	2	/* -> CONNECTED or -> CLOSING
 */
+#define CON_SOCK_STATE_CONNECTED	3	/* -> CLOSING or -> CLOSED */
+#define CON_SOCK_STATE_CLOSING		4	/* -> CLOSED */
+
 /* static tag bytes (protocol control messages) */
 static char tag_msg = CEPH_MSGR_TAG_MSG;

static char tag_ack = CEPH_MSGR_TAG_ACK;

net/ceph/messenger.c +55 -0

@@ -147,6 +155,54 @@ void ceph_msgr_flush(void)
 }
 EXPORT_SYMBOL(ceph_msgr_flush);
 
+/* Connection socket state transition functions */
+
+static void con_sock_state_init(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+	if (WARN_ON(old_state != CON_SOCK_STATE_NEW))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_connecting(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTING);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CLOSED))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_connected(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTED);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_closing(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSING);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING &&
+			old_state != CON_SOCK_STATE_CONNECTED))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_closed(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTED &&
+			old_state != CON_SOCK_STATE_CLOSING))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
 
 /*
  * socket callback functions

@@ -203,6 +259,7 @@ static void ceph_sock_state_change(struct sock *sk)
 		dout("%s TCP_CLOSE\n", __func__);
 	case TCP_CLOSE_WAIT:
 		dout("%s TCP_CLOSE_WAIT\n", __func__);
+		con_sock_state_closing(con);
 		if (test_and_set_bit(SOCK_CLOSED, &con->flags) == 0) {
 			if (test_bit(CONNECTING, &con->state))
 				con->error_msg = "connection failed";

@@ -213,6 +270,7 @@ static void ceph_sock_state_change(struct sock *sk)
 		break;
 	case TCP_ESTABLISHED:
 		dout("%s TCP_ESTABLISHED\n", __func__);
+		con_sock_state_connected(con);
 		queue_con(con);
 		break;
 	default:	/* Everything else is uninteresting */

@@ -277,6 +335,7 @@ static int ceph_tcp_connect(struct ceph_connection *con)
 		return ret;
 	}
 	con->sock = sock;
+	con_sock_state_connecting(con);
 
 	return 0;
 }

@@ -341,6 +400,7 @@ static int con_close_socket(struct ceph_connection *con)
 	rc = con->sock->ops->shutdown(con->sock, SHUT_RDWR);
 	sock_release(con->sock);
 	con->sock = NULL;
+	con_sock_state_closed(con);
 	return rc;
 }
 

@@ -460,6 +520,9 @@ void ceph_con_init(struct ceph_messenger *msgr, struct
 ceph_connection *con)
 	memset(con, 0, sizeof(*con));
 	atomic_set(&con->nref, 1);
 	con->msgr = msgr;
+
+	con_sock_state_init(con);
+
 	mutex_init(&con->mutex);
 	INIT_LIST_HEAD(&con->out_queue);

INIT_LIST_HEAD(&con->out_sent);

Signature · 6 lines

1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 28 lines · Alex Elder

Start explicitly keeping track of the state of a ceph connection's socket, separate from the state of the connection itself. Create placeholder functions to encapsulate the state transitions.

     --------
     | NEW* |  transient initial state
     --------
         | con_sock_state_init()
         v
     ----------
     | CLOSED |  initialized, but no socket (and no
     ----------  TCP connection)
      ^      \
      |       \ con_sock_state_connecting()
      |        ----------------------
      |                              \
      + con_sock_state_closed()       \
      |\                               \
      | \                               \
      |  -----------                     \
      |  | CLOSING |  socket event;       \
      |  -----------  await close          \
      |       ^                            |
      |       |                            |
      |       + con_sock_state_closing()   |
      |      / \                           |
      |     /   ---------------            |
      |    /                   \           v

Quoted 8 lines

      |   /                    --------------
      |  /    -----------------| CONNECTING |  socket created, TCP
      |  |   /                 --------------  connect initiated
      |  |   | con_sock_state_connected()
      |  |   v
     -------------
     | CONNECTED |  TCP connection established
     -------------

Can we put this beautiful pictures in the header next to the states?

Reviewed-by Sage Weil<[email protected]>

Quoted 167 lines

Make the socket state an atomic variable, reinforcing that it's a distinct transtion with no possible "intermediate/both" states. This is almost certainly overkill at this point, though the transitions into CONNECTED and CLOSING state do get called via socket callback (the rest of the transitions occur with the connection mutex held). We can back out the atomicity later.

Signed-off-by Alex Elder<[email protected]>

---
  include/linux/ceph/messenger.h |    8 ++++-
  net/ceph/messenger.c           |   63
++++++++++++++++++++++++++++++++++++++++
  2 files changed, 69 insertions(+), 2 deletions(-)

include/linux/ceph/messenger.h +6 -2

index 920235e..5e852f4 100644
@@ -137,14 +137,18 @@ struct ceph_connection {
  	const struct ceph_connection_operations *ops;
 
  	struct ceph_messenger *msgr;
+
+	atomic_t sock_state;
  	struct socket *sock;
+	struct ceph_entity_addr peer_addr; /* peer address */
+	struct ceph_entity_addr peer_addr_for_me;
+
  	unsigned long flags;
  	unsigned long state;
  	const char *error_msg;  /* error message, if any */
 
-	struct ceph_entity_addr peer_addr; /* peer address */
  	struct ceph_entity_name peer_name; /* peer name */
-	struct ceph_entity_addr peer_addr_for_me;
+
  	unsigned peer_features;
  	u32 connect_seq;      /* identify the most recent connection
  				 attempt for this connection, client */

net/ceph/messenger.c +8 -0

index 29055df..7e11b07 100644
@@ -29,6 +29,14 @@
   * the sender.
   */
 
+/* State values for ceph_connection->sock_state; NEW is assumed to be 0 */
+
+#define CON_SOCK_STATE_NEW		0	/* ->  CLOSED */
+#define CON_SOCK_STATE_CLOSED		1	/* ->  CONNECTING */
+#define CON_SOCK_STATE_CONNECTING	2	/* ->  CONNECTED or ->  CLOSING
 */
+#define CON_SOCK_STATE_CONNECTED	3	/* ->  CLOSING or ->  CLOSED */
+#define CON_SOCK_STATE_CLOSING		4	/* ->  CLOSED */
+
  /* static tag bytes (protocol control messages) */
  static char tag_msg = CEPH_MSGR_TAG_MSG;

  static char tag_ack = CEPH_MSGR_TAG_ACK;

net/ceph/messenger.c +55 -0

@@ -147,6 +155,54 @@ void ceph_msgr_flush(void)
  }
  EXPORT_SYMBOL(ceph_msgr_flush);
 
+/* Connection socket state transition functions */
+
+static void con_sock_state_init(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+	if (WARN_ON(old_state != CON_SOCK_STATE_NEW))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_connecting(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTING);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CLOSED))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_connected(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTED);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_closing(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSING);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING&&
+			old_state != CON_SOCK_STATE_CONNECTED))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
+
+static void con_sock_state_closed(struct ceph_connection *con)
+{
+	int old_state;
+
+	old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+	if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTED&&
+			old_state != CON_SOCK_STATE_CLOSING))
+		printk("%s: unexpected old state %d\n", __func__, old_state);
+}
 
  /*
   * socket callback functions

@@ -203,6 +259,7 @@ static void ceph_sock_state_change(struct sock *sk)
  		dout("%s TCP_CLOSE\n", __func__);
  	case TCP_CLOSE_WAIT:
  		dout("%s TCP_CLOSE_WAIT\n", __func__);
+		con_sock_state_closing(con);
  		if (test_and_set_bit(SOCK_CLOSED,&con->flags) == 0) {
  			if (test_bit(CONNECTING,&con->state))
  				con->error_msg = "connection failed";

@@ -213,6 +270,7 @@ static void ceph_sock_state_change(struct sock *sk)
  		break;
  	case TCP_ESTABLISHED:
  		dout("%s TCP_ESTABLISHED\n", __func__);
+		con_sock_state_connected(con);
  		queue_con(con);
  		break;
  	default:	/* Everything else is uninteresting */

@@ -277,6 +335,7 @@ static int ceph_tcp_connect(struct ceph_connection *con)
  		return ret;
  	}
  	con->sock = sock;
+	con_sock_state_connecting(con);
 
  	return 0;
  }

@@ -341,6 +400,7 @@ static int con_close_socket(struct ceph_connection *con)
  	rc = con->sock->ops->shutdown(con->sock, SHUT_RDWR);
  	sock_release(con->sock);
  	con->sock = NULL;
+	con_sock_state_closed(con);
  	return rc;
  }
 

@@ -460,6 +520,9 @@ void ceph_con_init(struct ceph_messenger *msgr, struct
 ceph_connection *con)
  	memset(con, 0, sizeof(*con));
  	atomic_set(&con->nref, 1);
  	con->msgr = msgr;
+
+	con_sock_state_init(con);
+
  	mutex_init(&con->mutex);
  	INIT_LIST_HEAD(&con->out_queue);

INIT_LIST_HEAD(&con->out_sent);
--
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Start explicitly keeping track of the state of a ceph connection's socket, separate from the state of the connection itself.  Create placeholder functions to encapsulate the state transitions.

   --------
   | NEW* |  transient initial state
   --------
       | con_sock_state_init()
       v
   ----------
   | CLOSED |  initialized, but no socket (and no
   ----------  TCP connection)
    ^      \

    |       \ con_sock_state_connecting()
    |        ----------------------
    |                              \
    + con_sock_state_closed()       \
    |\                               \
    | \                               \
    |  -----------                     \
    |  | CLOSING |  socket event;       \
    |  -----------  await close          \
    |       ^                            |
    |       |                            |
    |       + con_sock_state_closing()   |
    |      / \                           |
    |     /   ---------------            |
    |    /                   \           v
    |   /                    --------------
    |  /    -----------------| CONNECTING |  socket created, TCP
    |  |   /                 --------------  connect initiated
    |  |   | con_sock_state_connected()
    |  |   v

   -------------
   | CONNECTED |  TCP connection established
   -------------

Make the socket state an atomic variable, reinforcing that it's a distinct transtion with no possible "intermediate/both" states. This is almost certainly overkill at this point, though the transitions into CONNECTED and CLOSING state do get called via socket callback (the rest of the transitions occur with the connection mutex held).  We can back out the atomicity later.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/messenger.h |    8 ++++-
 net/ceph/messenger.c           |   63
++++++++++++++++++++++++++++++++++++++++

 2 files changed, 69 insertions(+), 2 deletions(-)

include/linux/ceph/messenger.h +5 -0

index 920235e..5e852f4 100644
@@ -137,14 +137,18 @@ struct ceph_connection {
        const struct ceph_connection_operations *ops;
 
        struct ceph_messenger *msgr;
+
+       atomic_t sock_state;
        struct socket *sock;
+       struct ceph_entity_addr peer_addr; /* peer address */
+       struct ceph_entity_addr peer_addr_for_me;
+
        unsigned long flags;
        unsigned long state;

       const char *error_msg;  /* error message, if any */

-       struct ceph_entity_addr peer_addr; /* peer address */ struct ceph_entity_name peer_name; /* peer name */
-       struct ceph_entity_addr peer_addr_for_me;
+
       unsigned peer_features;
       u32 connect_seq;      /* identify the most recent connection attempt for this connection, client */

net/ceph/messenger.c +5 -0

index 29055df..7e11b07 100644
@@ -29,6 +29,14 @@
  * the sender.
  */
 
+/* State values for ceph_connection->sock_state; NEW is assumed to be 0 */
+
+#define CON_SOCK_STATE_NEW             0       /* -> CLOSED */
+#define CON_SOCK_STATE_CLOSED          1       /* -> CONNECTING */
+#define CON_SOCK_STATE_CONNECTING      2       /* -> CONNECTED or ->

CLOSING */
+#define CON_SOCK_STATE_CONNECTED       3       /* -> CLOSING or -> CLOSED */

net/ceph/messenger.c +2 -0

+#define CON_SOCK_STATE_CLOSING         4       /* -> CLOSED */
+

 /* static tag bytes (protocol control messages) */ static char tag_msg = CEPH_MSGR_TAG_MSG; static char tag_ack = CEPH_MSGR_TAG_ACK;

net/ceph/messenger.c +27 -0

@@ -147,6 +155,54 @@ void ceph_msgr_flush(void)
  }
  EXPORT_SYMBOL(ceph_msgr_flush);
 
+/* Connection socket state transition functions */
+
+static void con_sock_state_init(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+       if (WARN_ON(old_state != CON_SOCK_STATE_NEW))
+               printk("%s: unexpected old state %d\n", __func__,
 old_state);
+}
+
+static void con_sock_state_connecting(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state,
 CON_SOCK_STATE_CONNECTING);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CLOSED))
+               printk("%s: unexpected old state %d\n", __func__,
 old_state);
+}
+
+static void con_sock_state_connected(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTED);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING))
+               printk("%s: unexpected old state %d\n", __func__,

old_state);

net/ceph/messenger.c +10 -0

+}
+
+static void con_sock_state_closing(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSING);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING &&
+                       old_state != CON_SOCK_STATE_CONNECTED))
+               printk("%s: unexpected old state %d\n", __func__,

old_state);

net/ceph/messenger.c +10 -0

+}
+
+static void con_sock_state_closed(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTED &&
+                       old_state != CON_SOCK_STATE_CLOSING))
+               printk("%s: unexpected old state %d\n", __func__,

old_state);
+}

 /*
 * socket callback functions

net/ceph/messenger.c +0 -0

@@ -203,6 +259,7 @@ static void ceph_sock_state_change(struct sock *sk)
                dout("%s TCP_CLOSE\n", __func__);

       case TCP_CLOSE_WAIT:
               dout("%s TCP_CLOSE_WAIT\n", __func__);
+               con_sock_state_closing(con);
               if (test_and_set_bit(SOCK_CLOSED, &con->flags) == 0) { if (test_bit(CONNECTING, &con->state)) con->error_msg = "connection failed";

net/ceph/messenger.c +0 -0

@@ -213,6 +270,7 @@ static void ceph_sock_state_change(struct sock *sk)
                break;

       case TCP_ESTABLISHED:
               dout("%s TCP_ESTABLISHED\n", __func__);
+               con_sock_state_connected(con); queue_con(con);
               break;
       default:        /* Everything else is uninteresting */

net/ceph/messenger.c +5 -0

@@ -277,6 +335,7 @@ static int ceph_tcp_connect(struct ceph_connection *con)
                return ret;
        }
        con->sock = sock;
+       con_sock_state_connecting(con);
 
        return 0;
  }

@@ -341,6 +400,7 @@ static int con_close_socket(struct ceph_connection *con)
        rc = con->sock->ops->shutdown(con->sock, SHUT_RDWR);
        sock_release(con->sock);
        con->sock = NULL;
+       con_sock_state_closed(con);
        return rc;
  }
 

@@ -460,6 +520,9 @@ void ceph_con_init(struct ceph_messenger *msgr, struct
 ceph_connection *con)
        memset(con, 0, sizeof(*con));
        atomic_set(&con->nref, 1);
        con->msgr = msgr;
+
+       con_sock_state_init(con);
+
        mutex_init(&con->mutex);
        INIT_LIST_HEAD(&con->out_queue);

       INIT_LIST_HEAD(&con->out_sent);
--
1.7.5.4

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Quoted 153 lines · Alex Elder

Start explicitly keeping track of the state of a ceph connection's socket, separate from the state of the connection itself.  Create placeholder functions to encapsulate the state transitions.

   --------
   | NEW* |  transient initial state
   --------
       | con_sock_state_init()
       v
   ----------
   | CLOSED |  initialized, but no socket (and no
   ----------  TCP connection)
    ^      \

    |       \ con_sock_state_connecting()
    |        ----------------------
    |                              \
    + con_sock_state_closed()       \
    |\                               \
    | \                               \
    |  -----------                     \
    |  | CLOSING |  socket event;       \
    |  -----------  await close          \
    |       ^                            |
    |       |                            |
    |       + con_sock_state_closing()   |
    |      / \                           |
    |     /   ---------------            |
    |    /                   \           v
    |   /                    --------------
    |  /    -----------------| CONNECTING |  socket created, TCP
    |  |   /                 --------------  connect initiated
    |  |   | con_sock_state_connected()
    |  |   v

   -------------
   | CONNECTED |  TCP connection established
   -------------

Make the socket state an atomic variable, reinforcing that it's a distinct transtion with no possible "intermediate/both" states. This is almost certainly overkill at this point, though the transitions into CONNECTED and CLOSING state do get called via socket callback (the rest of the transitions occur with the connection mutex held).  We can back out the atomicity later.

Signed-off-by Alex Elder <[email protected]>

---
 include/linux/ceph/messenger.h |    8 ++++-
 net/ceph/messenger.c           |   63
++++++++++++++++++++++++++++++++++++++++

 2 files changed, 69 insertions(+), 2 deletions(-)

include/linux/ceph/messenger.h +5 -0

index 920235e..5e852f4 100644
@@ -137,14 +137,18 @@ struct ceph_connection {
        const struct ceph_connection_operations *ops;
 
        struct ceph_messenger *msgr;
+
+       atomic_t sock_state;
        struct socket *sock;
+       struct ceph_entity_addr peer_addr; /* peer address */
+       struct ceph_entity_addr peer_addr_for_me;
+
        unsigned long flags;
        unsigned long state;

       const char *error_msg;  /* error message, if any */

-       struct ceph_entity_addr peer_addr; /* peer address */ struct ceph_entity_name peer_name; /* peer name */
-       struct ceph_entity_addr peer_addr_for_me;
+
       unsigned peer_features;
       u32 connect_seq;      /* identify the most recent connection attempt for this connection, client */

net/ceph/messenger.c +5 -0

index 29055df..7e11b07 100644
@@ -29,6 +29,14 @@
  * the sender.
  */
 
+/* State values for ceph_connection->sock_state; NEW is assumed to be 0 */
+
+#define CON_SOCK_STATE_NEW             0       /* -> CLOSED */
+#define CON_SOCK_STATE_CLOSED          1       /* -> CONNECTING */
+#define CON_SOCK_STATE_CONNECTING      2       /* -> CONNECTED or ->

CLOSING */
+#define CON_SOCK_STATE_CONNECTED       3       /* -> CLOSING or -> CLOSED */

net/ceph/messenger.c +2 -0

+#define CON_SOCK_STATE_CLOSING         4       /* -> CLOSED */
+

 /* static tag bytes (protocol control messages) */ static char tag_msg = CEPH_MSGR_TAG_MSG; static char tag_ack = CEPH_MSGR_TAG_ACK;

net/ceph/messenger.c +27 -0

@@ -147,6 +155,54 @@ void ceph_msgr_flush(void)
  }
  EXPORT_SYMBOL(ceph_msgr_flush);
 
+/* Connection socket state transition functions */
+
+static void con_sock_state_init(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+       if (WARN_ON(old_state != CON_SOCK_STATE_NEW))
+               printk("%s: unexpected old state %d\n", __func__,
 old_state);
+}
+
+static void con_sock_state_connecting(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state,
 CON_SOCK_STATE_CONNECTING);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CLOSED))
+               printk("%s: unexpected old state %d\n", __func__,
 old_state);
+}
+
+static void con_sock_state_connected(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CONNECTED);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING))
+               printk("%s: unexpected old state %d\n", __func__,

old_state);

net/ceph/messenger.c +10 -0

+}
+
+static void con_sock_state_closing(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSING);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTING &&
+                       old_state != CON_SOCK_STATE_CONNECTED))
+               printk("%s: unexpected old state %d\n", __func__,

old_state);

net/ceph/messenger.c +10 -0

+}
+
+static void con_sock_state_closed(struct ceph_connection *con)
+{
+       int old_state;
+
+       old_state = atomic_xchg(&con->sock_state, CON_SOCK_STATE_CLOSED);
+       if (WARN_ON(old_state != CON_SOCK_STATE_CONNECTED &&
+                       old_state != CON_SOCK_STATE_CLOSING))
+               printk("%s: unexpected old state %d\n", __func__,

Quoted 3 lines

old_state);
+}

Hi Alex,