RDMA/rxe: Cleanup rxe_init_packet()
Subsystems (2)
PATCH 1/1 RDMA/rxe: Cleanup rxe_init_packet()
Currently rxe_init_packet() recomputes ndev from the address vector but struct rxe_dev already holds a reference to ndev.
Cleanup rxe_init_packet to use the value of ndev in rxe and drop an unneeded parameter paylen since it is already carried in the packet info struct.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 +- drivers/infiniband/sw/rxe/rxe_net.c | 49 +++++++++++----------------- drivers/infiniband/sw/rxe/rxe_req.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 3 +- 4 files changed, 23 insertions(+), 33 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +1 -1
@@ -93,7 +93,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem);/* rxe_net.c */ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt); +struct rxe_pkt_info *pkt);int rxe_prepare (struct rxe_av *av,struct rxe_pkt_info *pkt,struct sk_buff *skb); int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
drivers/infiniband/sw/rxe/rxe_net.c +19 -30
74 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_req.c +1 -1
@@ -397,7 +397,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, pkt->paylen = paylen;/* init skb */ - skb =rxe_init_packet (rxe, av, paylen, pkt); + skb =rxe_init_packet (rxe, av, pkt);if (unlikely (!skb))return NULL ;
drivers/infiniband/sw/rxe/rxe_resp.c +2 -1
@@ -670,8 +670,9 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,*/ pad = (-payload) &0x3 ; paylen = rxe_opcode[opcode].length + payload + pad +RXE_ICRC_SIZE ; + ack->paylen = paylen; - skb =rxe_init_packet (rxe, &qp->pri_av, paylen, ack); + skb =rxe_init_packet (rxe, &qp->pri_av, ack);if (!skb)return NULL ;
Signature · 1 line
2.34.1
Quoted 64 lines · Bob Pearson
Currently
rxe_init_packet()recomputes ndev from the address vector but structrxe_devalready holds a reference to ndev.Cleanup
rxe_init_packetto use the value of ndev in rxe and drop an unneeded parameter paylen since it is already carried in the packet info struct.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 +- drivers/infiniband/sw/rxe/rxe_net.c | 49 +++++++++++----------------- drivers/infiniband/sw/rxe/rxe_req.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 3 +- 4 files changed, 23 insertions(+), 33 deletions(-)drivers/infiniband/sw/rxe/rxe_loc.h +1
-1@@ -93,7 +93,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem);/* rxe_net.c */ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt); +struct rxe_pkt_info *pkt);int rxe_prepare (struct rxe_av *av,struct rxe_pkt_info *pkt,struct sk_buff *skb); int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,drivers/infiniband/sw/rxe/rxe_net.c +11
-7@@ -443,18 +443,22 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,return err; } +/** +* rxe_init_packet - allocate and initialize a new skb +* @rxe: rxe device +* @av: remote address vector +* @pkt: packet info +* +* Returns: an skb on success else NULL +*/ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt) +struct rxe_pkt_info *pkt) {unsigned int hdr_len;struct sk_buff *skb =NULL ; -struct net_device *ndev; -const struct ib_gid_attr *attr; +struct net_device *ndev = rxe->ndev;const int port_num =1 ; - - attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index); -if (IS_ERR (attr)) -return NULL ; +int skb_size;if (av->network_type ==RXE_NETWORK_TYPE_IPV4 ) hdr_len = ETH_HLEN + sizeof(struct udphdr) +
drivers/infiniband/sw/rxe/rxe_net.c struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, · line 466
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_net.c +0
-1@@ -463,26 +467,13 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, hdr_len =ETH_HLEN +sizeof (struct udphdr ) +sizeof (struct ipv6hdr ); -rcu_read_lock ();
Was removing this rcu lock intentional? If so, do we need a mention of this in the commit message why its not needed anymore?
Quoted 60 lines
drivers/infiniband/sw/rxe/rxe_net.c +8
-22- ndev =rdma_read_gid_attr_ndev_rcu (attr); -if (IS_ERR (ndev)) { -rcu_read_unlock (); - goto out; - } - skb =alloc_skb (paylen + hdr_len +LL_RESERVED_SPACE (ndev), -GFP_ATOMIC ); - -if (unlikely (!skb)) { -rcu_read_unlock (); - goto out; - } - -skb_reserve (skb, hdr_len +LL_RESERVED_SPACE (ndev)); - -/* FIXME: hold reference to this netdev until life of this skb. */ - skb->dev = ndev; -rcu_read_unlock (); + skb_size =LL_RESERVED_SPACE (ndev) + hdr_len + pkt->paylen; + skb =alloc_skb (skb_size,GFP_ATOMIC ); +if (unlikely (!skb)) +return NULL ; +skb_reserve (skb,LL_RESERVED_SPACE (ndev) + hdr_len); + skb->dev = ndev;if (av->network_type ==RXE_NETWORK_TYPE_IPV4 ) skb->protocol =htons (ETH_P_IP );else @@ -490,11 +481,9 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, pkt->rxe = rxe; pkt->port_num = port_num; - pkt->hdr =skb_put (skb, paylen); - pkt->mask |=RXE_GRH_MASK ; + pkt->hdr =skb_put (skb, pkt->paylen); + pkt->mask |=RXE_GRH_MASK ; -out: -rdma_put_gid_attr (attr);return skb; }drivers/infiniband/sw/rxe/rxe_req.c +1
-1@@ -397,7 +397,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, pkt->paylen = paylen;/* init skb */ - skb =rxe_init_packet (rxe, av, paylen, pkt); + skb =rxe_init_packet (rxe, av, pkt);if (unlikely (!skb))return NULL ;drivers/infiniband/sw/rxe/rxe_resp.c +0
-0
drivers/infiniband/sw/rxe/rxe_resp.c static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, · line 673
Quoted 4 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-0@@ -670,8 +670,9 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,*/ pad = (-payload) &0x3 ; paylen = rxe_opcode[opcode].length + payload + pad +RXE_ICRC_SIZE ; + ack->paylen = paylen;
Maybe remove the old ack->paylen assignment which is done later in this function?
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-2- skb =rxe_init_packet (rxe, &qp->pri_av, paylen, ack); + skb =rxe_init_packet (rxe, &qp->pri_av, ack);if (!skb)return NULL ; --2.34.1
Quoted 68 lines · Haris Iqbal
Quoted 67 lines · Bob Pearson
Currently
rxe_init_packet()recomputes ndev from the address vector but structrxe_devalready holds a reference to ndev.Cleanup
rxe_init_packetto use the value of ndev in rxe and drop an unneeded parameter paylen since it is already carried in the packet info struct.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 +- drivers/infiniband/sw/rxe/rxe_net.c | 49 +++++++++++----------------- drivers/infiniband/sw/rxe/rxe_req.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 3 +- 4 files changed, 23 insertions(+), 33 deletions(-)drivers/infiniband/sw/rxe/rxe_loc.h +1
-1@@ -93,7 +93,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem);/* rxe_net.c */ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt); +struct rxe_pkt_info *pkt);int rxe_prepare (struct rxe_av *av,struct rxe_pkt_info *pkt,struct sk_buff *skb); int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,drivers/infiniband/sw/rxe/rxe_net.c +11
-7@@ -443,18 +443,22 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,return err; } +/** +* rxe_init_packet - allocate and initialize a new skb +* @rxe: rxe device +* @av: remote address vector +* @pkt: packet info +* +* Returns: an skb on success else NULL +*/ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt) +struct rxe_pkt_info *pkt) {unsigned int hdr_len;struct sk_buff *skb =NULL ; -struct net_device *ndev; -const struct ib_gid_attr *attr; +struct net_device *ndev = rxe->ndev;const int port_num =1 ; - - attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index); -if (IS_ERR (attr)) -return NULL ; +int skb_size;if (av->network_type ==RXE_NETWORK_TYPE_IPV4 ) hdr_len = ETH_HLEN + sizeof(struct udphdr) +@@ -463,26 +467,13 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, hdr_len =ETH_HLEN +sizeof (struct udphdr ) +sizeof (struct ipv6hdr );
Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_net.c struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, · line 466
Quoted 1 line
drivers/infiniband/sw/rxe/rxe_net.c +0
-1-rcu_read_lock ();Was removing this rcu lock intentional? If so, do we need a mention of this in the commit message why its not needed anymore?
The rcu lock was required to use the rdma_read_gid_attr_ndev_rcu() API. For the rxe driver there is no way that the ndev is changing once the rxe device is set up and ndev was passed in to rxe_newlink() and saved in the rxe_dev struct. Not any good reason to do all this work to get ndev when we already know it.
Quoted 55 lines
Quoted 55 lines
drivers/infiniband/sw/rxe/rxe_net.c +8
-22- ndev =rdma_read_gid_attr_ndev_rcu (attr); -if (IS_ERR (ndev)) { -rcu_read_unlock (); - goto out; - } - skb =alloc_skb (paylen + hdr_len +LL_RESERVED_SPACE (ndev), -GFP_ATOMIC ); - -if (unlikely (!skb)) { -rcu_read_unlock (); - goto out; - } - -skb_reserve (skb, hdr_len +LL_RESERVED_SPACE (ndev)); - -/* FIXME: hold reference to this netdev until life of this skb. */ - skb->dev = ndev; -rcu_read_unlock (); + skb_size =LL_RESERVED_SPACE (ndev) + hdr_len + pkt->paylen; + skb =alloc_skb (skb_size,GFP_ATOMIC ); +if (unlikely (!skb)) +return NULL ; +skb_reserve (skb,LL_RESERVED_SPACE (ndev) + hdr_len); + skb->dev = ndev;if (av->network_type ==RXE_NETWORK_TYPE_IPV4 ) skb->protocol =htons (ETH_P_IP );else @@ -490,11 +481,9 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, pkt->rxe = rxe; pkt->port_num = port_num; - pkt->hdr =skb_put (skb, paylen); - pkt->mask |=RXE_GRH_MASK ; + pkt->hdr =skb_put (skb, pkt->paylen); + pkt->mask |=RXE_GRH_MASK ; -out: -rdma_put_gid_attr (attr);return skb; }drivers/infiniband/sw/rxe/rxe_req.c +1
-1@@ -397,7 +397,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, pkt->paylen = paylen;/* init skb */ - skb =rxe_init_packet (rxe, av, paylen, pkt); + skb =rxe_init_packet (rxe, av, pkt);if (unlikely (!skb))return NULL ;
Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_resp.c static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, · line 673
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-0@@ -670,8 +670,9 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,*/ pad = (-payload) &0x3 ; paylen = rxe_opcode[opcode].length + payload + pad +RXE_ICRC_SIZE ; + ack->paylen = paylen;Maybe remove the old
ack->paylenassignment which is done later in this function?
Agreed. Will send a v2 with this change.
Quoted 9 lines
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-2- skb =rxe_init_packet (rxe, &qp->pri_av, paylen, ack); + skb =rxe_init_packet (rxe, &qp->pri_av, ack);if (!skb)return NULL ; --2.34.1
Bob
Quoted 75 lines · Bob Pearson
Quoted 68 lines · Haris Iqbal
Quoted 67 lines · Bob Pearson
Currently
rxe_init_packet()recomputes ndev from the address vector but structrxe_devalready holds a reference to ndev.Cleanup
rxe_init_packetto use the value of ndev in rxe and drop an unneeded parameter paylen since it is already carried in the packet info struct.Signed-off-by Bob Pearson <[email protected]>--- drivers/infiniband/sw/rxe/rxe_loc.h | 2 +- drivers/infiniband/sw/rxe/rxe_net.c | 49 +++++++++++----------------- drivers/infiniband/sw/rxe/rxe_req.c | 2 +- drivers/infiniband/sw/rxe/rxe_resp.c | 3 +- 4 files changed, 23 insertions(+), 33 deletions(-)drivers/infiniband/sw/rxe/rxe_loc.h +1
-1@@ -93,7 +93,7 @@ void rxe_mw_cleanup(struct rxe_pool_elem *elem);/* rxe_net.c */ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt); +struct rxe_pkt_info *pkt);int rxe_prepare (struct rxe_av *av,struct rxe_pkt_info *pkt,struct sk_buff *skb); int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,drivers/infiniband/sw/rxe/rxe_net.c +11
-7@@ -443,18 +443,22 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,return err; } +/** +* rxe_init_packet - allocate and initialize a new skb +* @rxe: rxe device +* @av: remote address vector +* @pkt: packet info +* +* Returns: an skb on success else NULL +*/ struct sk_buff *rxe_init_packet (struct rxe_dev *rxe,struct rxe_av *av, -int paylen,struct rxe_pkt_info *pkt) +struct rxe_pkt_info *pkt) {unsigned int hdr_len;struct sk_buff *skb =NULL ; -struct net_device *ndev; -const struct ib_gid_attr *attr; +struct net_device *ndev = rxe->ndev;const int port_num =1 ; - - attr =rdma_get_gid_attr (&rxe->ib_dev, port_num, av->grh.sgid_index); -if (IS_ERR (attr)) -return NULL ; +int skb_size;if (av->network_type ==RXE_NETWORK_TYPE_IPV4 ) hdr_len = ETH_HLEN + sizeof(struct udphdr) +@@ -463,26 +467,13 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, hdr_len =ETH_HLEN +sizeof (struct udphdr ) +sizeof (struct ipv6hdr );Quoted 5 lines
drivers/infiniband/sw/rxe/rxe_net.c struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, · line 466
Quoted 1 line
drivers/infiniband/sw/rxe/rxe_net.c +0
-1-rcu_read_lock ();Was removing this rcu lock intentional? If so, do we need a mention of this in the commit message why its not needed anymore?
Quoted 4 lines
The rcu lock was required to use the
rdma_read_gid_attr_ndev_rcu()API. For the rxe driver there is no way that the ndev is changing once the rxe device is set up and ndev was passed in torxe_newlink()and saved in therxe_devstruct. Not any good reason to do all this work to get ndev when we already know it.
I see. Thanks for the explanation.
One more question for my understanding.
There was a FIXME line (removed with this commit) mentioning that we should hold a reference to the netdev. I assume it was talking about dev_hold.
Now, while initing the rxe we call ib_device_set_netdev, which does dev_hold. But I could not find rxe doing dev_hold anywhere directly. I am assuming this is enough, or should rxe be also holding a reference for this net device since technically it does use it directly at some places?
Quoted 80 lines
Quoted 55 lines
Quoted 55 lines
drivers/infiniband/sw/rxe/rxe_net.c +8
-22- ndev =rdma_read_gid_attr_ndev_rcu (attr); -if (IS_ERR (ndev)) { -rcu_read_unlock (); - goto out; - } - skb =alloc_skb (paylen + hdr_len +LL_RESERVED_SPACE (ndev), -GFP_ATOMIC ); - -if (unlikely (!skb)) { -rcu_read_unlock (); - goto out; - } - -skb_reserve (skb, hdr_len +LL_RESERVED_SPACE (ndev)); - -/* FIXME: hold reference to this netdev until life of this skb. */ - skb->dev = ndev; -rcu_read_unlock (); + skb_size =LL_RESERVED_SPACE (ndev) + hdr_len + pkt->paylen; + skb =alloc_skb (skb_size,GFP_ATOMIC ); +if (unlikely (!skb)) +return NULL ; +skb_reserve (skb,LL_RESERVED_SPACE (ndev) + hdr_len); + skb->dev = ndev;if (av->network_type ==RXE_NETWORK_TYPE_IPV4 ) skb->protocol =htons (ETH_P_IP );else @@ -490,11 +481,9 @@ struct sk_buff *rxe_init_packet(struct rxe_dev *rxe, struct rxe_av *av, pkt->rxe = rxe; pkt->port_num = port_num; - pkt->hdr =skb_put (skb, paylen); - pkt->mask |=RXE_GRH_MASK ; + pkt->hdr =skb_put (skb, pkt->paylen); + pkt->mask |=RXE_GRH_MASK ; -out: -rdma_put_gid_attr (attr);return skb; }drivers/infiniband/sw/rxe/rxe_req.c +1
-1@@ -397,7 +397,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp, pkt->paylen = paylen;/* init skb */ - skb =rxe_init_packet (rxe, av, paylen, pkt); + skb =rxe_init_packet (rxe, av, pkt);if (unlikely (!skb))return NULL ;Quoted 13 lines
drivers/infiniband/sw/rxe/rxe_resp.c static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp, · line 673
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-0@@ -670,8 +670,9 @@ static struct sk_buff *prepare_ack_packet(struct rxe_qp *qp,*/ pad = (-payload) &0x3 ; paylen = rxe_opcode[opcode].length + payload + pad +RXE_ICRC_SIZE ; + ack->paylen = paylen;Maybe remove the old
ack->paylenassignment which is done later in this function?Agreed. Will send a v2 with this change.
Quoted 9 lines
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_resp.c +1
-2- skb =rxe_init_packet (rxe, &qp->pri_av, paylen, ack); + skb =rxe_init_packet (rxe, &qp->pri_av, ack);if (!skb)return NULL ; --2.34.1
Bob