libceph fixes
Subsystems (2)
cover libceph fixes
These two patches from Sage fix two problems have arisen since the recent changes in the messenger code. I have already reviewed them both.
[PATCH 1/2] libceph: set peer name on con_open, not init [PATCH 2/2] libceph: close socket directly during ceph_con_close()
-Alex
PATCH 1/2 libceph: set peer name on con_open, not init
The peer name may change on each open attempt, even when the connection is reused.
Signed-off-by Sage Weil <[email protected]>
Reviewed-by Alex Elder <[email protected]>
--- fs/ceph/mds_client.c | 7 ++++--- include/linux/ceph/messenger.h | 4 ++-- net/ceph/messenger.c | 12 +++++++----- net/ceph/mon_client.c | 8 ++++---- net/ceph/osd_client.c | 10 ++++++---- 5 files changed, 23 insertions(+), 18 deletions(-)
fs/ceph/mds_client.c +4 -3
@@ -394,8 +394,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, s->s_seq =0 ;mutex_init (&s->s_mutex); -ceph_con_init (&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr, -CEPH_ENTITY_TYPE_MDS , mds); +ceph_con_init (&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr);spin_lock_init (&s->s_gen_ttl_lock); s->s_cap_gen =0 ;
@@ -437,7 +436,8 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, mdsc->sessions[mds] = s;atomic_inc (&s->s_ref);/* one ref to sessions[], one to caller */ -ceph_con_open (&s->s_con,ceph_mdsmap_get_addr (mdsc->mdsmap, mds)); +ceph_con_open (&s->s_con,CEPH_ENTITY_TYPE_MDS , mds, +ceph_mdsmap_get_addr (mdsc->mdsmap, mds));return s;
@@ -2529,6 +2529,7 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, session->s_seq =0 ;ceph_con_open (&session->s_con, +CEPH_ENTITY_TYPE_MDS , mds,ceph_mdsmap_get_addr (mdsc->mdsmap, mds));/* replay unsafe requests */
include/linux/ceph/messenger.h +2 -2
@@ -232,9 +232,9 @@ extern void ceph_messenger_init(struct ceph_messenger *msgr,extern void ceph_con_init (struct ceph_connection *con,void *private,const struct ceph_connection_operations *ops, -struct ceph_messenger *msgr,__u8 entity_type, -__u64 entity_num); +struct ceph_messenger *msgr);extern void ceph_con_open (struct ceph_connection *con, +__u8 entity_type,__u64 entity_num,struct ceph_entity_addr *addr);extern bool ceph_con_opened (struct ceph_connection *con);extern void ceph_con_close (struct ceph_connection *con);
net/ceph/messenger.c +7 -5
@@ -523,12 +523,17 @@ EXPORT_SYMBOL(ceph_con_close);/* * Reopen a closed connection, with a new peer address. */ -void ceph_con_open (struct ceph_connection *con,struct ceph_entity_addr *addr) +void ceph_con_open (struct ceph_connection *con, +__u8 entity_type,__u64 entity_num, +struct ceph_entity_addr *addr) {dout ("con_open %p %s\n" , con,ceph_pr_addr (&addr->in_addr));set_bit (OPENING , &con->state);WARN_ON (!test_and_clear_bit (CLOSED , &con->state)); + con->peer_name.type = (__u8 ) entity_type; + con->peer_name.num =cpu_to_le64 (entity_num); +memcpy (&con->peer_addr, addr,sizeof (*addr)); con->delay =0 ;/* reset backoff memory */ queue_con (con);
@@ -548,7 +553,7 @@ bool ceph_con_opened(struct ceph_connection *con)*/ void ceph_con_init (struct ceph_connection *con,void *private,const struct ceph_connection_operations *ops, -struct ceph_messenger *msgr,__u8 entity_type,__u64 entity_num) +struct ceph_messenger *msgr) {dout ("con_init %p\n" , con);memset (con,0 ,sizeof (*con));
@@ -558,9 +563,6 @@ void ceph_con_init(struct ceph_connection *con, void *private,con_sock_state_init (con); - con->peer_name.type = (__u8 ) entity_type; - con->peer_name.num =cpu_to_le64 (entity_num); -mutex_init (&con->mutex);INIT_LIST_HEAD (&con->out_queue);INIT_LIST_HEAD (&con->out_sent);
net/ceph/mon_client.c +4 -4
@@ -142,12 +142,9 @@ static int __open_session(struct ceph_mon_client *monc) monc->sub_renew_after = jiffies;/* i.e., expired */ monc->want_next_osdmap = !!monc->want_next_osdmap; -ceph_con_init (&monc->con, monc, &mon_con_ops, - &monc->client->msgr, -CEPH_ENTITY_TYPE_MON , monc->cur_mon); -dout ("open_session mon%d opening\n" , monc->cur_mon);ceph_con_open (&monc->con, +CEPH_ENTITY_TYPE_MON , monc->cur_mon, &monc->monmap->mon_inst[monc->cur_mon].addr);/* initiatiate authentication handshake */
@@ -798,6 +795,9 @@ int ceph_monc_init(struct ceph_mon_client *monc, struct ceph_client *cl)if (!monc->m_auth) goto out_auth_reply; +ceph_con_init (&monc->con, monc, &mon_con_ops, + &monc->client->msgr); + monc->cur_mon =-1 ; monc->hunting = true; monc->sub_renew_after = jiffies;
net/ceph/osd_client.c +6 -4
@@ -639,8 +639,7 @@ static struct ceph_osd *create_osd(struct ceph_osd_client *osdc, int onum)INIT_LIST_HEAD (&osd->o_osd_lru); osd->o_incarnation =1 ; -ceph_con_init (&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr, -CEPH_ENTITY_TYPE_OSD , onum); +ceph_con_init (&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr);INIT_LIST_HEAD (&osd->o_keepalive_item);return osd;
@@ -750,7 +749,8 @@ static int __reset_osd(struct ceph_osd_client *osdc, struct ceph_osd *osd) ret = -EAGAIN ; }else {ceph_con_close (&osd->o_con); -ceph_con_open (&osd->o_con, &osdc->osdmap->osd_addr[osd->o_osd]); +ceph_con_open (&osd->o_con,CEPH_ENTITY_TYPE_OSD , osd->o_osd, + &osdc->osdmap->osd_addr[osd->o_osd]); osd->o_incarnation++; }return ret;
@@ -1005,7 +1005,9 @@ static int __map_request(struct ceph_osd_client *osdc,dout ("map_request osd %p is osd%d\n" , req->r_osd, o);__insert_osd (osdc, req->r_osd); -ceph_con_open (&req->r_osd->o_con, &osdc->osdmap->osd_addr[o]); +ceph_con_open (&req->r_osd->o_con, +CEPH_ENTITY_TYPE_OSD , o, + &osdc->osdmap->osd_addr[o]); }if (req->r_osd) {
Signature · 1 line
1.7.9.5
PATCH 2/2 libceph: close socket directly during ceph_con_close()
In reply to Alex ElderWhen we close the connection, immediately shut down the socket instead of queuing work and letting the worker thread do it. This is simpler.
Signed-off-by Sage Weil <[email protected]>
Reviewed-by Alex Elder <[email protected]>
--- net/ceph/messenger.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-)
net/ceph/messenger.c +15 -14
@@ -47,18 +47,18 @@* | \ con_sock_state_connecting() * | ---------------------- * | \ -* + con_sock_state_closed() \ -* |\ \ -* | \ \ -* | ----------- \ -* | | CLOSING | socket event; \ -* | ----------- await close \ -* | ^ | -* | | | -* | + con_sock_state_closing() | -* | / \ | -* | / --------------- | -* | / \ v +* +--------------------------- \ +* |\ con_sock_state_closed() \ \ +* | \ \ \ +* | ----------- \ \ +* | | CLOSING | socket event; \ \ +* | ----------- await close \ \ +* | ^ \ | +* | | \ | +* | + con_sock_state_closing() \ | +* | / \ | | +* | / --------------- | | +* | / \ | v * | / -------------- * | / -----------------| CONNECTING | socket created, TCP * | | / -------------- connect initiated
@@ -241,7 +241,8 @@ static void con_sock_state_closed(struct ceph_connection *con) old_state =atomic_xchg (&con->sock_state,CON_SOCK_STATE_CLOSED );if (WARN_ON (old_state !=CON_SOCK_STATE_CONNECTED && - old_state !=CON_SOCK_STATE_CLOSING )) + old_state !=CON_SOCK_STATE_CLOSING && + old_state !=CON_SOCK_STATE_CONNECTING ))printk ("%s: unexpected old state %d\n" , __func__, old_state); }
@@ -514,9 +515,9 @@ void ceph_con_close(struct ceph_connection *con)mutex_lock (&con->mutex);reset_connection (con); con->peer_global_seq =0 ; +con_close_socket (con);cancel_delayed_work (&con->work);mutex_unlock (&con->mutex); -queue_con (con); }EXPORT_SYMBOL (ceph_con_close);
Signature · 1 line
1.7.9.5