Generate UDP src port with flow label or lqpn/rqpn
Subsystems (4)
cover Generate UDP src port with flow label or lqpn/rqpn
From: Zhu Yanjun <[email protected]>
Follow the advice from Leon Romanovsky, rdma_get_udp_sport is moved to ib_verbs.h. several drivers generate udp source port with this function.
Zhu Yanjun (4):
RDMA/core: Calculate UDP source port based on flow label or lqpn/rqpn RDMA/hns: Replace get_udp_sport with rdma_get_udp_sport RDMA/irdma: Make the source udp port vary
RDMA/rxe: Use the standard method to produce udp source port
drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 12 ++---------- drivers/infiniband/hw/irdma/verbs.c | 5 +++++ drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++++++++ include/rdma/ib_verbs.h | 17 +++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-)
Signature · 1 line
2.27.0
PATCH 1/4 RDMA/core: Calculate UDP source port based on flow label or lqpn/rqpn
From: Zhu Yanjun <[email protected]>
Calculate and set UDP source port based on the flow label. If flow label is not defined in GRH then calculate it based on lqpn/rqpn.
--- include/rdma/ib_verbs.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)
include/rdma/ib_verbs.h +17 -0
@@ -4749,6 +4749,23 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) return (u32)(v & IB_GRH_FLOWLABEL_MASK); } +/** +* rdma_get_udp_sport - Calculate and set UDP source port based on the flow +* label. If flow label is not defined in GRH then +* calculate it based on lqpn/rqpn. +* +* @fl: flow label from GRH +* @lqpn: local qp number +* @rqpn: remote qp number +*/ +static inline u16 rdma_get_udp_sport (u32 fl,u32 lqpn,u32 rqpn) +{ +if (!fl) + fl =rdma_calc_flow_label (lqpn, rqpn); + +return rdma_flow_label_to_udp_sport (fl); +} +const struct ib_port_immutable *ib_port_immutable_read (struct ib_device *dev,unsigned int port);#endif /* IB_VERBS_H */
Signature · 1 line
2.27.0
PATCH 2/4 RDMA/hns: Replace get_udp_sport with rdma_get_udp_sport
In reply to [email protected]From: Zhu Yanjun <[email protected]>
Several drivers have the same function xxx_get_udp_sport. So this function is moved to ib_verbs.h.
--- drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-)
drivers/infiniband/hw/hns/hns_roce_hw_v2.c +2 -10
@@ -4488,14 +4488,6 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,return 0 ; } -static inline u16 get_udp_sport (u32 fl,u32 lqpn,u32 rqpn) -{ -if (!fl) - fl =rdma_calc_flow_label (lqpn, rqpn); - -return rdma_flow_label_to_udp_sport (fl); -} -static int get_dip_ctx_idx (struct ib_qp *ibqp,const struct ib_qp_attr *attr,u32 *dip_idx) {
@@ -4712,8 +4704,8 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp, }hr_reg_write (context,QPC_UDPSPN , - is_udp ?get_udp_sport (grh->flow_label, ibqp->qp_num, - attr->dest_qp_num) :0 ); + is_udp ?rdma_get_udp_sport (grh->flow_label, ibqp->qp_num, + attr->dest_qp_num) :0 );hr_reg_clear (qpc_mask,QPC_UDPSPN );
Signature · 1 line
2.27.0
PATCH 3/4 RDMA/irdma: Make the source udp port vary
In reply to [email protected]From: Zhu Yanjun <[email protected]>
Get the source udp port number for a QP based on the grh.flow_label or lqpn/rqrpn. This provides a better spread of traffic across NIC RX queues.
--- drivers/infiniband/hw/irdma/verbs.c | 5 +++++ 1 file changed, 5 insertions(+)
drivers/infiniband/hw/irdma/verbs.c +5 -0
@@ -1167,6 +1167,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,memset (&iwqp->roce_ah,0 ,sizeof (iwqp->roce_ah));if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = ibqp->qp_num; +u32 rqp = roce_info->dest_qp; + + udp_info->src_port =rdma_get_udp_sport (fl, lqp, rqp); udp_info->ttl = attr->ah_attr.grh.hop_limit; udp_info->flow_label = attr->ah_attr.grh.flow_label; udp_info->tos = attr->ah_attr.grh.traffic_class;
Signature · 1 line
2.27.0
PATCH 4/4 RDMA/rxe: Use the standard method to produce udp source port
In reply to [email protected]From: Zhu Yanjun <[email protected]>
Use the standard method to produce udp source port.
--- drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
drivers/infiniband/sw/rxe/rxe_verbs.c +10 -0
@@ -469,6 +469,16 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,if (err) goto err1; +if (mask &IB_QP_AV ) { +if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = qp->ibqp.qp_num; +u32 rqp = qp->attr.dest_qp_num; + + qp->src_port =rdma_get_udp_sport (fl, lqp, rqp); + } + } +return 0 ; err1:
Signature · 1 line
2.27.0
RE: [PATCH 4/4] RDMA/rxe: Use the standard method to produce udp source port
Quoted 18 lines
Subject: [PATCH 4/4] RDMA/rxe: Use the standard method to produce udp source port
From: Zhu Yanjun <[email protected]>
Use the standard method to produce udp source port.
Signed-off-by Zhu Yanjun <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++++++++ 1 file changed, 10 insertions(+)drivers/infiniband/sw/rxe/rxe_verbs.c +0
-0
drivers/infiniband/sw/rxe/rxe_verbs.c static int rxe_modify_qp(struct ib_qp *ibqp, struct · line 478
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +6
-0@@ -469,6 +469,16 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (err) goto err1; +if (mask &IB_QP_AV ) { +if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = qp->ibqp.qp_num; +u32 rqp = qp->attr.dest_qp_num; +
Isn't the randomization for src_port done in rxe_qp_init_req redundant then?
https://elixir.bootlin.com/linux/v5.16-rc8/source/drivers/infiniband/sw/rxe/rxe_qp.c#L220
Can we remove it?
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +4
-0+ qp->src_port =rdma_get_udp_sport (fl, lqp, rqp); + } + } +return 0 ;err1:
--
2.27.0
RE: [PATCH 3/4] RDMA/irdma: Make the source udp port vary
In reply to [email protected]Quoted 17 lines
Subject: [PATCH 3/4] RDMA/irdma: Make the source udp port vary
From: Zhu Yanjun <[email protected]>
Get the source udp port number for a QP based on the grh.flow_label or lqpn/rqrpn. This provides a better spread of traffic across NIC RX queues.
Signed-off-by Zhu Yanjun <[email protected]>--- drivers/infiniband/hw/irdma/verbs.c | 5 +++++ 1 file changed, 5 insertions(+)drivers/infiniband/hw/irdma/verbs.c +0
-0
drivers/infiniband/hw/irdma/verbs.c int irdma_modify_qp_roce(struct ib_qp *ibqp, struct · line 1175
Quoted 9 lines
drivers/infiniband/hw/irdma/verbs.c +5
-0@@ -1167,6 +1167,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,memset (&iwqp->roce_ah,0 ,sizeof (iwqp->roce_ah));if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = ibqp->qp_num; +u32 rqp = roce_info->dest_qp; + +
Do you really need these locals?
udp_info->src_port = rdma_get_udp_sport(fl, lqp, rqp);
Quoted 5 lines
drivers/infiniband/hw/irdma/verbs.c +0
-0udp_info->ttl = attr->ah_attr.grh.hop_limit; udp_info->flow_label = attr->ah_attr.grh.flow_label;
udp_info->tos=attr->ah_attr.grh.traffic_class;
--
2.27.0
Re: [PATCH 3/4] RDMA/irdma: Make the source udp port vary
Quoted 19 lines · Shiraz
Quoted 19 lines
Subject: [PATCH 3/4] RDMA/irdma: Make the source udp port vary
From: Zhu Yanjun <[email protected]>
Get the source udp port number for a QP based on the grh.flow_label or lqpn/rqrpn. This provides a better spread of traffic across NIC RX queues.
Signed-off-by Zhu Yanjun <[email protected]>--- drivers/infiniband/hw/irdma/verbs.c | 5 +++++ 1 file changed, 5 insertions(+)drivers/infiniband/hw/irdma/verbs.c +0
-0@@ -1167,6 +1167,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
Quoted 8 lines
drivers/infiniband/hw/irdma/verbs.c int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
Quoted 7 lines
drivers/infiniband/hw/irdma/verbs.c +5
-0memset (&iwqp->roce_ah,0 ,sizeof (iwqp->roce_ah));if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = ibqp->qp_num; +u32 rqp = roce_info->dest_qp; + +Do you really need these locals?
I asked same question in previous revision.
Zhu, please remove them.
Thanks
Re: [PATCH 4/4] RDMA/rxe: Use the standard method to produce udp source port
In reply to Saleem, ShirazQuoted 30 lines · Shiraz
Quoted 18 lines
Subject: [PATCH 4/4] RDMA/rxe: Use the standard method to produce udp source port
From: Zhu Yanjun <[email protected]>
Use the standard method to produce udp source port.
Signed-off-by Zhu Yanjun <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++++++++++ 1 file changed, 10 insertions(+)drivers/infiniband/sw/rxe/rxe_verbs.c +0
-0drivers/infiniband/sw/rxe/rxe_verbs.c static int rxe_modify_qp(struct ib_qp *ibqp, struct · line 478
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +6
-0@@ -469,6 +469,16 @@ static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (err) goto err1; +if (mask &IB_QP_AV ) { +if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = qp->ibqp.qp_num; +u32 rqp = qp->attr.dest_qp_num; +Isn't the randomization for
src_portdone inrxe_qp_init_reqredundant then?
Quoted 3 lines
https://elixir.bootlin.com/linux/v5.16-rc8/source/drivers/infiniband/sw/rxe/rxe_qp.c#L220
Can we remove it?
Yes. We can remove it.
Because this "randomization for src_port done in rxe_qp_init_req" is replaced by rdma_get_udp_sport in rxe_modify_qp, I do not remove it.
I will remove it in the latest commits soon.
Zhu Yanjun
Quoted 11 lines
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +4
-0+ qp->src_port =rdma_get_udp_sport (fl, lqp, rqp); + } + } +return 0 ;err1:
--
2.27.0
Re: [PATCH 3/4] RDMA/irdma: Make the source udp port vary
In reply to Leon RomanovskyQuoted 29 lines · Leon Romanovsky
Quoted 19 lines · Shiraz
Quoted 19 lines
Subject: [PATCH 3/4] RDMA/irdma: Make the source udp port vary
From: Zhu Yanjun <[email protected]>
Get the source udp port number for a QP based on the grh.flow_label or lqpn/rqrpn. This provides a better spread of traffic across NIC RX queues.
Signed-off-by Zhu Yanjun <[email protected]>--- drivers/infiniband/hw/irdma/verbs.c | 5 +++++ 1 file changed, 5 insertions(+)drivers/infiniband/hw/irdma/verbs.c +0
-0@@ -1167,6 +1167,11 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,Quoted 8 lines
drivers/infiniband/hw/irdma/verbs.c int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
Quoted 7 lines
drivers/infiniband/hw/irdma/verbs.c +5
-0memset (&iwqp->roce_ah,0 ,sizeof (iwqp->roce_ah));if (attr->ah_attr.ah_flags &IB_AH_GRH ) { +u32 fl = attr->ah_attr.grh.flow_label; +u32 lqp = ibqp->qp_num; +u32 rqp = roce_info->dest_qp; + +Do you really need these locals?
Quoted 3 lines
I asked same question in previous revision.
Zhu, please remove them.
Hi, Leon Romanovsky && Saleem, Shiraz
Without these local variables, the line "udp_info->src_port = rdma_get_udp_sport(fl, lqp, rqp);" will exceed 80. This will cause warning when this commit is checked by script/checkpatch.pl.
Let me have a try to find a better way to avoid these local variables.
Zhu Yanjun
Quoted 2 lines
Thanks
Re: [PATCH 4/4] RDMA/rxe: Use the standard method to produce udp source port
In reply to Yanjun ZhuQuoted 32 lines · Yanjun Zhu
Quoted 31 lines · Shiraz
Quoted 23 lines
Subject: [PATCH 4/4] RDMA/rxe: Use the standard method to produce udp source port
From: Zhu Yanjun <[email protected]>
Use the standard method to produce udp source port.
Signed-off-by Zhu Yanjun <[email protected]>---
drivers/infiniband/sw/rxe/rxe_verbs.c| 10 ++++++++++1 file changed, 10 insertions(+)drivers/infiniband/sw/rxe/rxe_verbs.c +0
-0@@ -469,6 +469,16 @@ static int rxe_modify_qp(struct ib_qp *ibqp,struct
ib_qp_attr*attr,
if (err)
goto err1;Quoted 6 lines
+ if (mask &
IB_QP_AV) {
+ if (attr->ah_attr.ah_flags&IB_AH_GRH) { + u32 fl =attr->ah_attr.grh.flow_label; + u32 lqp =qp->ibqp.qp_num;
+ u32 rqp =qp->attr.dest_qp_num; +Isn't the randomization for
src_portdone inrxe_qp_init_reqredundant then?
Quoted 4 lines
Quoted 4 lines
https://elixir.bootlin.com/linux/v5.16-rc8/source/drivers/infiniband/sw/rxe/rxe_qp.c#L220
Can we remove it?
Based on the discussion with Leon Romanovsky, when in-subnet communications, GRH is optional.
So It is possible that GRH is not set. Without the randomization for src_port done in rxe_qp_init_req, udp source
port will be 0xC000 in that case.
So it is better to keep this randomization for src_port done in rxe_qp_init_req.
Zhu Yanjun
Quoted 20 lines
Yes. We can remove it.
Because this "randomization forsrc_portdone inrxe_qp_init_req" is replaced byrdma_get_udp_sportinrxe_modify_qp, I do not remove it.I will remove it in the latest commits soon.
Zhu Yanjun
Quoted 11 lines
Quoted 10 lines
+
qp->src_port=rdma_get_udp_sport(fl, lqp, rqp); + }
+ }
+
return 0;err1:
Signature · 1 line
2.27.0
Re: [PATCH 1/4] RDMA/core: Calculate UDP source port based on flow label or lqpn/rqpn
In reply to [email protected]Quoted 15 lines · On 1/5/2022 4:07 PM, [email protected] wrote:
From: Zhu Yanjun <[email protected]>
Calculate and set UDP source port based on the flow label. If flow label is not defined in GRH then calculate it based on lqpn/rqpn.
Signed-off-by Zhu Yanjun <[email protected]>--- include/rdma/ib_verbs.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+)include/rdma/ib_verbs.h +0
-0
include/rdma/ib_verbs.h static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) · line 4757
Quoted 9 lines
include/rdma/ib_verbs.h +6
-0@@ -4749,6 +4749,23 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn)return (u32)(v &IB_GRH_FLOWLABEL_MASK ); } +/** +* rdma_get_udp_sport - Calculate and set UDP source port based on the flow +* label. If flow label is not defined in GRH then +* calculate it based on lqpn/rqpn. +* +* @fl: flow label from GRH
Indent:
+ * @fl: flow label from GRH
include/rdma/ib_verbs.h static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) · line 4771
Quoted 13 lines
include/rdma/ib_verbs.h +11
-0+* @lqpn: local qp number +* @rqpn: remote qp number +*/ +static inline u16 rdma_get_udp_sport (u32 fl,u32 lqpn,u32 rqpn) +{ +if (!fl) + fl =rdma_calc_flow_label (lqpn, rqpn); + +return rdma_flow_label_to_udp_sport (fl); +} + > const struct ib_port_immutable*ib_port_immutable_read (struct ib_device *dev,unsigned int port);#endif /* IB_VERBS_H */
Maybe this and next patch can be squashed into one?
Re: [PATCH 1/4] RDMA/core: Calculate UDP source port based on flow label or lqpn/rqpn
Quoted 21 lines · Mark Zhang
Quoted 20 lines · On 1/5/2022 4:07 PM, [email protected] wrote:
From: Zhu Yanjun <[email protected]>
Calculate and set UDP source port based on the flow label. If flow label is not defined in GRH then calculate it based on lqpn/rqpn.
Signed-off-by Zhu Yanjun <[email protected]>---
include/rdma/ib_verbs.h| 17 +++++++++++++++++1 file changed, 17 insertions(+)include/rdma/ib_verbs.h +0
-0@@ -4749,6 +4749,23 @@ static inline u32 rdma_calc_flow_label(u32 lqpn, u32 rqpn) return (u32)(v & IB_GRH_FLOWLABEL_MASK); }+/**
+ *rdma_get_udp_sport- Calculate and set UDP source port based on
Quoted 8 lines
Quoted 5 lines
the flow
+ * label. If flow label is not defined in GRH then
+ * calculate it based on lqpn/rqpn.
+ *
+ * @fl: flow label from GRHIndent:
+ * @fl: flow label from GRH
Thanks. In V2, this problem is fixed.
Quoted 16 lines
Quoted 12 lines
+ * @lqpn: local qp number
+ * @rqpn: remote qp number
+ */include/rdma/ib_verbs.h +8
-0+static inline u16 rdma_get_udp_sport (u32 fl,u32 lqpn,u32 rqpn) +{ + if (!fl) + fl = rdma_calc_flow_label(lqpn, rqpn); + + return rdma_flow_label_to_udp_sport(fl); +} + > const struct ib_port_immutable*Quoted 2 lines
ib_port_immutable_read(struct ib_device *dev, unsigned int port); #endif /*IB_VERBS_H*/Maybe this and next patch can be squashed into one?
Why do you think this commit and next commit should be squashed into one?
Best Regards,
Zhu Yanjun