net/smc: patches 2019-02-07
Subsystems (5)
cover net/smc: patches 2019-02-07
Dave,
here are patches for SMC:
* patches 1, 3, and 6 are cleanups without functional change
* patch 2 postpones closing of internal clcsock
* patches 4 and 5 improve link group creation locking
* patch 7 restores AF_SMC as diag_family field
Thanks, Ursula
Hans Wippel (2):
net/smc: unlock LGR pending lock earlier for SMC-D net/smc: use client and server LGR pending locks for SMC-R
Karsten Graul (2):
net/smc: move code to clear the conn->lgr field net/smc: original socket family in inet_sock_diag
Ursula Braun (3):
s390/net: move pnet constants
net/smc: postpone release of clcsock
net/smc: use smc_curs_copy() for SMC-D
arch/s390/include/asm/pnet.h | 8 ----- arch/s390/net/pnet.c | 8 +++++ net/smc/af_smc.c | 85 ++++++++++++++++++++++++++------------------ net/smc/smc_cdc.h | 9 ++--- net/smc/smc_close.c | 7 +++- net/smc/smc_core.c | 5 +-- net/smc/smc_diag.c | 3 +- 7 files changed, 74 insertions(+), 51 deletions(-)
Signature · 1 line
2.16.4
PATCH 1/7 s390/net: move pnet constants
There is no need to define these PNETID related constants in the pnet.h file, since they are just used locally within pnet.c. Just code cleanup, no functional change.
--- arch/s390/include/asm/pnet.h | 8 -------- arch/s390/net/pnet.c | 8 ++++++++ 2 files changed, 8 insertions(+), 8 deletions(-)
arch/s390/include/asm/pnet.h +0 -8
@@ -11,13 +11,5 @@#include <linux/device.h> #include <linux/types.h> -#define PNETIDS_LEN 64/* Total utility string length in bytes -* to cover up to 4 PNETIDs of 16 bytes -* for up to 4 device ports -*/ -#define MAX_PNETID_LEN 16/* Max.length of a single port PNETID */ -#define MAX_PNETID_PORTS (PNETIDS_LEN / MAX_PNETID_LEN) -/* Max. # of ports with a PNETID */ -int pnet_id_by_dev_port (struct device *dev,unsigned short port,u8 *pnetid);#endif /* _ASM_S390_PNET_H */
arch/s390/net/pnet.c +8 -0
@@ -13,6 +13,14 @@#include <asm/ccwdev.h> #include <asm/pnet.h> +#define PNETIDS_LEN 64/* Total utility string length in bytes +* to cover up to 4 PNETIDs of 16 bytes +* for up to 4 device ports +*/ +#define MAX_PNETID_LEN 16/* Max.length of a single port PNETID */ +#define MAX_PNETID_PORTS (PNETIDS_LEN / MAX_PNETID_LEN) +/* Max. # of ports with a PNETID */ +/* * Get the PNETIDs from a device. * s390 hardware supports the definition of a so-called Physical Network
Signature · 1 line
2.16.4
PATCH 2/7 net/smc: postpone release of clcsock
In reply to Ursula BraunAccording to RFC7609 (http://www.rfc-editor.org/info/rfc7609)
first the SMC-R connection is shut down and then the normal TCP connection FIN processing drives cleanup of the internal TCP connection. The unconditional release of the clcsock during active socket closing has to be postponed if the peer has not yet signalled socket closing.
--- net/smc/af_smc.c | 33 +++++++++++++++++---------------- net/smc/smc_close.c | 7 ++++++- 2 files changed, 23 insertions(+), 17 deletions(-)
net/smc/af_smc.c +17 -16
@@ -145,32 +145,33 @@ static int smc_release(struct socket *sock) rc =smc_close_active (smc);sock_set_flag (sk,SOCK_DEAD ); sk->sk_shutdown |=SHUTDOWN_MASK ; - } - - sk->sk_prot->unhash (sk); - -if (smc->clcsock) { -if (smc->use_fallback && sk->sk_state ==SMC_LISTEN ) { + }else { +if (sk->sk_state !=SMC_LISTEN && sk->sk_state !=SMC_INIT ) +sock_put (sk);/* passive closing */ +if (sk->sk_state ==SMC_LISTEN ) {/* wake up clcsock accept */ rc =kernel_sock_shutdown (smc->clcsock,SHUT_RDWR ); } -mutex_lock (&smc->clcsock_release_lock); -sock_release (smc->clcsock); - smc->clcsock =NULL ; -mutex_unlock (&smc->clcsock_release_lock); - } -if (smc->use_fallback) { -if (sk->sk_state !=SMC_LISTEN && sk->sk_state !=SMC_INIT ) -sock_put (sk);/* passive closing */ sk->sk_state =SMC_CLOSED ; sk->sk_state_change (sk); } + sk->sk_prot->unhash (sk); + +if (sk->sk_state ==SMC_CLOSED ) { +if (smc->clcsock) { +mutex_lock (&smc->clcsock_release_lock); +sock_release (smc->clcsock); + smc->clcsock =NULL ; +mutex_unlock (&smc->clcsock_release_lock); + } +if (!smc->use_fallback) +smc_conn_free (&smc->conn); + } +/* detach socket */ sock_orphan (sk); sock->sk =NULL ; -if (!smc->use_fallback && sk->sk_state ==SMC_CLOSED ) -smc_conn_free (&smc->conn);release_sock (sk);sock_put (sk);/* final sock_put */
net/smc/smc_close.c +6 -1
@@ -405,8 +405,13 @@ static void smc_close_passive_work(struct work_struct *work)if (old_state != sk->sk_state) { sk->sk_state_change (sk);if ((sk->sk_state ==SMC_CLOSED ) && - (sock_flag (sk,SOCK_DEAD ) || !sk->sk_socket)) + (sock_flag (sk,SOCK_DEAD ) || !sk->sk_socket)) {smc_conn_free (conn); +if (smc->clcsock) { +sock_release (smc->clcsock); + smc->clcsock =NULL ; + } + } }release_sock (sk);sock_put (sk);/* sock_hold done by schedulers of close_work */
Signature · 1 line
2.16.4
PATCH 3/7 net/smc: use smc_curs_copy() for SMC-D
In reply to Ursula BraunSMC already provides a wrapper for atomic64 calls to be architecture independent. Use this wrapper for SMC-D as well.
--- net/smc/smc_cdc.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
net/smc/smc_cdc.h +5 -4
@@ -245,17 +245,18 @@ static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local, }static inline void smcd_cdc_msg_to_host (struct smc_host_cdc_msg *local, -struct smcd_cdc_msg *peer) +struct smcd_cdc_msg *peer, +struct smc_connection *conn) {union smc_host_cursor temp; temp.wrap = peer->prod.wrap; temp.count = peer->prod.count; -atomic64_set (&local->prod.acurs,atomic64_read (&temp.acurs)); +smc_curs_copy (&local->prod, &temp, conn); temp.wrap = peer->cons.wrap; temp.count = peer->cons.count; -atomic64_set (&local->cons.acurs,atomic64_read (&temp.acurs)); +smc_curs_copy (&local->cons, &temp, conn); local->prod_flags = peer->cons.prod_flags; local->conn_state_flags = peer->cons.conn_state_flags; }
@@ -265,7 +266,7 @@ static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local, struct smc_connection *conn) {if (conn->lgr->is_smcd) -smcd_cdc_msg_to_host (local, (struct smcd_cdc_msg *)peer); +smcd_cdc_msg_to_host (local, (struct smcd_cdc_msg *)peer, conn);else smcr_cdc_msg_to_host (local, peer, conn); }
Signature · 1 line
2.16.4
PATCH 4/7 net/smc: unlock LGR pending lock earlier for SMC-D
In reply to Ursula BraunFrom: Hans Wippel <[email protected]>
If SMC client and server connections are both established at the same time, smc_connect_ism() cannot send a CLC confirm message while smc_listen_work() is waiting for one due to lock contention. This can result in timeouts in smc_clc_wait_msg() and failed SMC connections.
In case of SMC-D, the LGR pending lock is not needed while smc_listen_work() is waiting for the CLC confirm message. So, this patch releases the lock earlier for SMC-D to avoid the locking issue.
--- net/smc/af_smc.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-)
net/smc/af_smc.c +9 -5
@@ -1287,24 +1287,28 @@ static void smc_listen_work(struct work_struct *work)return ; } +/* SMC-D does not need this lock any more */ +if (ism_supported) +mutex_unlock (&smc_create_lgr_pending); +/* receive SMC Confirm CLC message */ reason_code =smc_clc_wait_msg (new_smc, &cclc,sizeof (cclc),SMC_CLC_CONFIRM ,CLC_WAIT_TIME );if (reason_code) { -mutex_unlock (&smc_create_lgr_pending); +if (!ism_supported) +mutex_unlock (&smc_create_lgr_pending);smc_listen_decline (new_smc, reason_code, local_contact);return ; }/* finish worker */ if (!ism_supported) { -if (smc_listen_rdma_finish (new_smc, &cclc, local_contact)) { -mutex_unlock (&smc_create_lgr_pending); + rc =smc_listen_rdma_finish (new_smc, &cclc, local_contact); +mutex_unlock (&smc_create_lgr_pending); +if (rc)return ; - } }smc_conn_save_peer_info (new_smc, &cclc); -mutex_unlock (&smc_create_lgr_pending);smc_listen_out_connected (new_smc); }
Signature · 1 line
2.16.4
PATCH 5/7 net/smc: use client and server LGR pending locks for SMC-R
In reply to Ursula BraunFrom: Hans Wippel <[email protected]>
If SMC client and server connections are both established at the same time, smc_connect_rdma() cannot send a CLC confirm message while smc_listen_work() is waiting for one due to lock contention. This can result in timeouts in smc_clc_wait_msg() and failed SMC connections.
In case of SMC-R, there are two types of LGRs (client and server LGRs) which can be protected by separate locks. So, this patch splits the LGR pending lock into two separate locks for client and server to avoid the locking issue for SMC-R.
--- net/smc/af_smc.c | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-)
net/smc/af_smc.c +28 -16
@@ -42,8 +42,11 @@#include "smc_rx.h" #include "smc_close.h" -static DEFINE_MUTEX (smc_create_lgr_pending);/* serialize link group -* creation +static DEFINE_MUTEX (smc_server_lgr_pending);/* serialize link group +* creation on server +*/ +static DEFINE_MUTEX (smc_client_lgr_pending);/* serialize link group +* creation on client */ static void smc_tcp_listen_work (struct work_struct *);
@@ -477,7 +480,12 @@ static int smc_connect_abort(struct smc_sock *smc, int reason_code, {if (local_contact ==SMC_FIRST_CONTACT )smc_lgr_forget (smc->conn.lgr); -mutex_unlock (&smc_create_lgr_pending); +if (smc->conn.lgr->is_smcd) +/* there is only one lgr role for SMC-D; use server lock */ +mutex_unlock (&smc_server_lgr_pending); +else +mutex_unlock (&smc_client_lgr_pending); +smc_conn_free (&smc->conn);return reason_code; }
@@ -562,7 +570,7 @@ static int smc_connect_rdma(struct smc_sock *smc,struct smc_link *link;int reason_code =0 ; -mutex_lock (&smc_create_lgr_pending); +mutex_lock (&smc_client_lgr_pending); local_contact =smc_conn_create (smc, false, aclc->hdr.flag, ibdev, ibport,ntoh24 (aclc->qpn), &aclc->lcl,NULL ,0 );
@@ -573,7 +581,8 @@ static int smc_connect_rdma(struct smc_sock *smc, reason_code =SMC_CLC_DECL_SYNCERR ;/* synchr. error */ else reason_code =SMC_CLC_DECL_INTERR ;/* other error */ -return smc_connect_abort (smc, reason_code,0 ); +mutex_unlock (&smc_client_lgr_pending); +return reason_code; } link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK ];
@@ -617,7 +626,7 @@ static int smc_connect_rdma(struct smc_sock *smc,return smc_connect_abort (smc, reason_code, local_contact); } -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_client_lgr_pending);smc_copy_sock_settings_to_clc (smc);if (smc->sk.sk_state ==SMC_INIT )
@@ -634,11 +643,14 @@ static int smc_connect_ism(struct smc_sock *smc,int local_contact =SMC_FIRST_CONTACT ;int rc =0 ; -mutex_lock (&smc_create_lgr_pending); +/* there is only one lgr role for SMC-D; use server lock */ +mutex_lock (&smc_server_lgr_pending); local_contact =smc_conn_create (smc, true, aclc->hdr.flag,NULL ,0 ,0 ,NULL , ismdev, aclc->gid); -if (local_contact <0 ) -return smc_connect_abort (smc,SMC_CLC_DECL_MEM ,0 ); +if (local_contact <0 ) { +mutex_unlock (&smc_server_lgr_pending); +return SMC_CLC_DECL_MEM ; + }/* Create send and receive buffers */ if (smc_buf_create (smc, true))
@@ -652,7 +664,7 @@ static int smc_connect_ism(struct smc_sock *smc, rc =smc_clc_send_confirm (smc);if (rc)return smc_connect_abort (smc, rc, local_contact); -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_server_lgr_pending);smc_copy_sock_settings_to_clc (smc);if (smc->sk.sk_state ==SMC_INIT )
@@ -1251,7 +1263,7 @@ static void smc_listen_work(struct work_struct *work)return ; } -mutex_lock (&smc_create_lgr_pending); +mutex_lock (&smc_server_lgr_pending);smc_close_init (new_smc);smc_rx_init (new_smc);smc_tx_init (new_smc);
@@ -1273,7 +1285,7 @@ static void smc_listen_work(struct work_struct *work) &local_contact) || smc_listen_rdma_reg(new_smc, local_contact))) {/* SMC not supported, decline */ -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_server_lgr_pending);smc_listen_decline (new_smc,SMC_CLC_DECL_MODEUNSUPP , local_contact);return ;
@@ -1282,21 +1294,21 @@ static void smc_listen_work(struct work_struct *work)/* send SMC Accept CLC message */ rc =smc_clc_send_accept (new_smc, local_contact);if (rc) { -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_server_lgr_pending);smc_listen_decline (new_smc, rc, local_contact);return ; }/* SMC-D does not need this lock any more */ if (ism_supported) -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_server_lgr_pending);/* receive SMC Confirm CLC message */ reason_code =smc_clc_wait_msg (new_smc, &cclc,sizeof (cclc),SMC_CLC_CONFIRM ,CLC_WAIT_TIME );if (reason_code) {if (!ism_supported) -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_server_lgr_pending);smc_listen_decline (new_smc, reason_code, local_contact);return ; }
@@ -1304,7 +1316,7 @@ static void smc_listen_work(struct work_struct *work)/* finish worker */ if (!ism_supported) { rc =smc_listen_rdma_finish (new_smc, &cclc, local_contact); -mutex_unlock (&smc_create_lgr_pending); +mutex_unlock (&smc_server_lgr_pending);if (rc)return ; }
Signature · 1 line
2.16.4
PATCH 6/7 net/smc: move code to clear the conn->lgr field
In reply to Ursula BraunFrom: Karsten Graul <[email protected]>
The lgr field of an smc_connection is set in smc_conn_create() and should be cleared in smc_conn_free() for consistency reasons, so move the responsible code.
--- net/smc/smc_core.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
net/smc/smc_core.c +3 -2
@@ -118,7 +118,6 @@ static void __smc_lgr_unregister_conn(struct smc_connection *conn)rb_erase (&conn->alert_node, &lgr->conns_all); lgr->conns_num--; conn->alert_token_local =0 ; - conn->lgr =NULL ;sock_put (&smc->sk);/* sock_hold in smc_lgr_register_conn() */ }
@@ -331,8 +330,9 @@ void smc_conn_free(struct smc_connection *conn) }else {smc_cdc_tx_dismiss_slots (conn); } -smc_lgr_unregister_conn (conn);/* unsets conn->lgr */ +smc_lgr_unregister_conn (conn);smc_buf_unuse (conn, lgr);/* allow buffer reuse */ + conn->lgr =NULL ;if (!lgr->conns_num)smc_lgr_schedule_free_work (lgr);
@@ -462,6 +462,7 @@ static void __smc_lgr_terminate(struct smc_link_group *lgr)sock_hold (&smc->sk);/* sock_put in close work */ conn->local_tx_ctrl.conn_state_flags.peer_conn_abort =1 ;__smc_lgr_unregister_conn (conn); + conn->lgr =NULL ;write_unlock_bh (&lgr->conns_lock);if (!schedule_work (&conn->close_work))sock_put (&smc->sk);
Signature · 1 line
2.16.4
PATCH 7/7 net/smc: original socket family in inet_sock_diag
In reply to Ursula BraunFrom: Karsten Graul <[email protected]>
Commit ed75986f4aae ("net/smc: ipv6 support for smc_diag.c") changed the value of the diag_family field. The idea was to indicate the family of the IP address in the inet_diag_sockid field. But the change makes it impossible to distinguish an inet_sock_diag response message from SMC sock_diag response. This patch restores the original behaviour and sends AF_SMC as value of the diag_family field.
--- net/smc/smc_diag.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
net/smc/smc_diag.c +1 -2
@@ -38,6 +38,7 @@ static void smc_diag_msg_common_fill(struct smc_diag_msg *r, struct sock *sk) {struct smc_sock *smc =smc_sk (sk); + r->diag_family = sk->sk_family;if (!smc->clcsock)return ; r->id.idiag_sport =htons (smc->clcsock->sk->sk_num);
@@ -45,14 +46,12 @@ static void smc_diag_msg_common_fill(struct smc_diag_msg *r, struct sock *sk) r->id.idiag_if = smc->clcsock->sk->sk_bound_dev_if;sock_diag_save_cookie (sk, r->id.idiag_cookie);if (sk->sk_protocol ==SMCPROTO_SMC ) { - r->diag_family =PF_INET ;memset (&r->id.idiag_src,0 ,sizeof (r->id.idiag_src));memset (&r->id.idiag_dst,0 ,sizeof (r->id.idiag_dst)); r->id.idiag_src[0 ] = smc->clcsock->sk->sk_rcv_saddr; r->id.idiag_dst[0 ] = smc->clcsock->sk->sk_daddr;#if IS_ENABLED (CONFIG_IPV6 ) } else if (sk->sk_protocol == SMCPROTO_SMC6) { - r->diag_family =PF_INET6 ;memcpy (&r->id.idiag_src, &smc->clcsock->sk->sk_v6_rcv_saddr,sizeof (smc->clcsock->sk->sk_v6_rcv_saddr)); memcpy(&r->id.idiag_dst, &smc->clcsock->sk->sk_v6_daddr,
Signature · 1 line
2.16.4
David Miller
maintainerFrom: Ursula Braun <[email protected]>
Date: Thu, 7 Feb 2019 15:56:13 +0100
Quoted 5 lines
here are patches for SMC:
* patches 1, 3, and 6 are cleanups without functional change
* patch 2 postpones closing of internal clcsock
* patches 4 and 5 improve link group creation locking
* patch 7 restoresAF_SMCasdiag_familyfield
Series applied, thank you.