RDMA/rxe: Avoid -Wflex-array-member-not-at-end warnings
Subsystems (2)
PATCH 1/1 RDMA/rxe: Avoid -Wflex-array-member-not-at-end warnings
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new TRAILING_OVERLAP() helper to fix the following warning:
21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]
This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.
The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.
Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.
--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +11 -7
@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe; }; +static_assert(offsetof(struct rxe_resp_info, srq_wqe.wqe.dma.sge) == + offsetof(struct rxe_resp_info, srq_wqe.sge));struct rxe_qp {struct ib_qp ibqp;
@@ -269,7 +271,6 @@ struct rxe_qp {struct rxe_req_info req;struct rxe_comp_info comp; -struct rxe_resp_info resp;atomic_t ssn;atomic_t skb_out;
@@ -289,6 +290,9 @@ struct rxe_qp {spinlock_t state_lock;/* guard requester and completer */ struct execute_work cleanup_work; + +/* Must be last as it ends in a flexible-array member. */ +struct rxe_resp_info resp; };enum {
Signature · 1 line
2.43.0
Quoted 44 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;
Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -6
@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };struct rxe_qp {
~
Quoted 75 lines · Leon Romanovsky
Quoted 44 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };
The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge {
u64 addr;
u32 length;
u32 lkey;
};and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge {
__aligned_u64 addr;
__u32 length;
__u32 lkey;
};Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
Thanks
-Gustavo
Quoted 105 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge { u64 addr; u32 length; u32 lkey; };and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge { __aligned_u64 addr; __u32 length; __u32 lkey; };
Quoted 7 lines
Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".
My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.
Thanks
Quoted 8 lines
Thanks
-Gustavo
Quoted 108 lines · Leon Romanovsky
Quoted 107 lines · Gustavo A. R. Silva
Quoted 72 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge { u64 addr; u32 length; u32 lkey; };and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge { __aligned_u64 addr; __u32 length; __u32 lkey; };Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Quoted 7 lines
Quoted 2 lines
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".
My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.
Thanks, I agree with this approach. The macro is rather complicated, and this solution fixes the problem in a straightforward way.
Yanjun.Zhu
Quoted 10 lines
Thanks
Quoted 7 lines
Thanks
-Gustavo
Signature · 2 lines
Best Regards, Yanjun.Zhu
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge { u64 addr; u32 length; u32 lkey; };and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge { __aligned_u64 addr; __u32 length; __u32 lkey; };Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Quoted 8 lines
Quoted 2 lines
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".
My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.
Yep, but as I commented above, that doesn't seem to be the right change.
Look at the following couple of lines:
drivers/infiniband/sw/rxe/rxe_resp.c-286- size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);drivers/infiniband/sw/rxe/rxe_resp.c-287- memcpy(&qp->resp.srq_wqe, wqe, size);
Notice that line 286 is the open-coded arithmetic (struct_size(wqe, dma.sge, wqe->dma.num_sge) is preferred) to get the number of bytes to allocate for a flexible structure, in this case struct rxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].
So, size bytes are written in qp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation of RXE_MAX_SGE number of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:
struct {
struct rxe_recv_wqe wqe;
struct ib_sge sge[RXE_MAX_SGE];
} srq_wqe;So, unless I'm missing something, struct ib_sge sge[RXE_MAX_SGE]; should be aligned with struct rxe_recv_wqe wqe::dma.sge[].
The TRAILING_OVERLAP() macro is also designed to ensure alignment in these cases (and the static_assert() to preserve it). See this thread:
https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/
Thanks
-Gustavo
Quoted 135 lines · Gustavo A. R. Silva
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge { u64 addr; u32 length; u32 lkey; };and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge { __aligned_u64 addr; __u32 length; __u32 lkey; };Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Quoted 8 lines
Quoted 2 lines
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".
My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.
Yep, but as I commented above, that doesn't seem to be the right change.
Look at the following couple of lines:
drivers/infiniband/sw/rxe/rxe_resp.c-286-size =sizeof(*wqe)+ wqe->dma.num_sge*sizeof(structrxe_sge);drivers/infiniband/sw/rxe/rxe_resp.c-287-memcpy(&qp->resp.srq_wqe, wqe, size);Notice that line 286 is the open-coded arithmetic (
struct_size(wqe, dma.sge, wqe->dma.num_sge)is preferred) to get the number of bytes to allocate for a flexible structure, in this case structrxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].
Quoted 8 lines
So,
sizebytes are written inqp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation ofRXE_MAX_SGEnumber of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:struct { struct rxe_recv_wqe wqe; struct ib_sge sge[RXE_MAX_SGE]; } srq_wqe;
So you are saying that it works because it is written properly, so what is the problem? Why do we need to fix properly working and written code to be less readable?
Quoted 3 lines
So, unless I'm missing something, struct
ib_sgesge[RXE_MAX_SGE]; should be aligned with structrxe_recv_wqewqe::dma.sge[].
It is and moving to the end of struct will continue to keep it aligned.
Quoted 12 lines
The
TRAILING_OVERLAP()macro is also designed to ensure alignment in these cases (and thestatic_assert()to preserve it). See this thread:https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/
Thanks
-Gustavo
Quoted 140 lines · Leon Romanovsky
Quoted 139 lines · Gustavo A. R. Silva
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge { u64 addr; u32 length; u32 lkey; };and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge { __aligned_u64 addr; __u32 length; __u32 lkey; };Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Quoted 8 lines
Quoted 2 lines
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".
My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.
Yep, but as I commented above, that doesn't seem to be the right change.
Look at the following couple of lines:
drivers/infiniband/sw/rxe/rxe_resp.c-286-size =sizeof(*wqe)+ wqe->dma.num_sge*sizeof(structrxe_sge);drivers/infiniband/sw/rxe/rxe_resp.c-287-memcpy(&qp->resp.srq_wqe, wqe, size);Notice that line 286 is the open-coded arithmetic (
struct_size(wqe, dma.sge, wqe->dma.num_sge)is preferred) to get the number of bytes to allocate for a flexible structure, in this case structrxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].So,
sizebytes are written inqp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation ofRXE_MAX_SGEnumber of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:
Quoted 8 lines
Quoted 4 lines
struct { struct rxe_recv_wqe wqe; struct ib_sge sge[RXE_MAX_SGE]; } srq_wqe;So you are saying that it works because it is written properly, so what is the problem? Why do we need to fix properly working and written code to be less readable?
No one said the original code is not working as expected. The issue here is that the FAM is not at the end, and this causes a -Wflex-array-member-not-at-end warning. The change I propose places the FAM at the end, and the functionality remains exactly the same.
You're probably not aware of the work we've been doing to enable -Wflex-array-member-not-at-end in mainline. If you're interested, below you can take a look at other similar changes I (and others) have been doing to complete this work:
Quoted 6 lines
Quoted 4 lines
So, unless I'm missing something, struct
ib_sgesge[RXE_MAX_SGE]; should be aligned with structrxe_recv_wqewqe::dma.sge[].It is and moving to the end of struct will continue to keep it aligned.
I think there is something you are missing here. The following pieces of code are no equivalent:
struct {
struct rxe_recv_wqe wqe; struct ib_sge sge[RXE_MAX_SGE];
} srq_wqe;
struct {
struct ib_sge sge[RXE_MAX_SGE]; struct rxe_recv_wqe wqe; } srq_wqe;
What I'm understanding from your last couple of responses is that you think the above are equivalent. My previous response tried to explain why that is not the case.
Quoted 7 lines
Quoted 7 lines
The
TRAILING_OVERLAP()macro is also designed to ensure alignment in these cases (and thestatic_assert()to preserve it). See this thread:https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/
Thanks
-Gustavo
Quoted 157 lines · Gustavo A. R. Silva
Quoted 140 lines · Leon Romanovsky
Quoted 139 lines · Gustavo A. R. Silva
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva
-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.
Use the new
TRAILING_OVERLAP()helper to fix the following warning:21
drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.
This overlays the trailing MEMBER struct
ib_sgesge[RXE_MAX_SGE]; onto the FAM structrxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.The
static_assert()ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.Lastly, move the conflicting declaration struct
rxe_resp_inforesp; to the end of the corresponding structure.Signed-off-by Gustavo A. R. Silva <[email protected]>--- drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-)drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234
Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,7 +226,15 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +/* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge, + struct ib_sge sge[RXE_MAX_SGE]; + ) srq_wqe;Will this change be enough?
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235
Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6
-0@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only */ +struct { +struct ib_sge sge[RXE_MAX_SGE ]; +struct rxe_recv_wqe wqe; + } srq_wqe; };The question is if this is really what you want?
sge[RXE_MAX_SGE] is of the following type:
struct ib_sge { u64 addr; u32 length; u32 lkey; };and struct rxe_recv_wqe::dma.sge[] is of type:
struct rxe_sge { __aligned_u64 addr; __u32 length; __u32 lkey; };Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.
Quoted 8 lines
Quoted 2 lines
Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?
You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".
My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.
Yep, but as I commented above, that doesn't seem to be the right change.
Look at the following couple of lines:
drivers/infiniband/sw/rxe/rxe_resp.c-286-size =sizeof(*wqe)+ wqe->dma.num_sge*sizeof(structrxe_sge);drivers/infiniband/sw/rxe/rxe_resp.c-287-memcpy(&qp->resp.srq_wqe, wqe, size);Notice that line 286 is the open-coded arithmetic (
struct_size(wqe, dma.sge, wqe->dma.num_sge)is preferred) to get the number of bytes to allocate for a flexible structure, in this case structrxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].So,
sizebytes are written inqp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation ofRXE_MAX_SGEnumber of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:Quoted 8 lines
Quoted 4 lines
struct { struct rxe_recv_wqe wqe; struct ib_sge sge[RXE_MAX_SGE]; } srq_wqe;So you are saying that it works because it is written properly, so what is the problem? Why do we need to fix properly working and written code to be less readable?
No one said the original code is not working as expected. The issue here is that the FAM is not at the end, and this causes a -Wflex-array-member-not-at-end warning. The change I propose places the FAM at the end, and the functionality remains exactly the same.
Quoted 6 lines
You're probably not aware of the work we've been doing to enable -Wflex-array-member-not-at-end in mainline. If you're interested, below you can take a look at other similar changes I (and others) have been doing to complete this work:
I'm aware of that work, just trying to find a simplest possible solution for this specific driver. RXE is a development driver which won't benefit from this hardening work at all.
Quoted 16 lines
Quoted 7 lines
Quoted 5 lines
So, unless I'm missing something, struct
ib_sgesge[RXE_MAX_SGE]; should be aligned with structrxe_recv_wqewqe::dma.sge[].It is and moving to the end of struct will continue to keep it aligned.
I think there is something you are missing here. The following pieces of code are no equivalent:
struct { struct rxe_recv_wqe wqe;struct
ib_sgesge[RXE_MAX_SGE];
}srq_wqe;
Quoted 8 lines
struct {
struct ib_sge sge[RXE_MAX_SGE]; struct rxe_recv_wqe wqe; } srq_wqe;What I'm understanding from your last couple of responses is that you think the above are equivalent. My previous response tried to explain why that is not the case.
Yes, I already forgot about change in srq_wqe.
Thanks
Quoted 11 lines
Quoted 8 lines
Quoted 8 lines
The
TRAILING_OVERLAP()macro is also designed to ensure alignment in these cases (and thestatic_assert()to preserve it). See this thread:https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/
Thanks
-Gustavo
Jason Gunthorpe
maintainerdrivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 226
Quoted 13 lines · Gustavo A. R. Silva
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-5@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe;
I think this existing is just a mess, can we fix it properly?
What this code wants to do is to have rxe_resp_info allocate a rxe_recv_wqe and have a maximum size of its flex array pre-allocated.
Probably like this, though maybe we need a FLEX version of the INIT_RDMA_OBJ_SIZE macro to make it safer.
Zhu, this seems like a good thing for you to look at?
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, rxe_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, rxe_cq, ibcq),INIT_RDMA_OBJ_SIZE (ib_pd, rxe_pd, ibpd), -INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp), +/* For resp.srq_wqe.dma.sge */ +INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp) +RXE_MAX_SGE *sizeof (struct ib_sge ),INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc), INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
drivers/infiniband/sw/rxe/rxe_verbs.h +4 -7
@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
@@ -232,6 +226,9 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only. srq_wqe.dma.sge is a flex array */ +struct rxe_recv_wqe srq_wqe; };struct rxe_qp {
@@ -269,7 +266,6 @@ struct rxe_qp {struct rxe_req_info req;struct rxe_comp_info comp; -struct rxe_resp_info resp;atomic_t ssn;atomic_t skb_out;
@@ -289,6 +285,7 @@ struct rxe_qp {spinlock_t state_lock;/* guard requester and completer */ struct execute_work cleanup_work; +struct rxe_resp_info resp; };enum {
Quoted 16 lines · Jason Gunthorpe
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 226
Quoted 13 lines · Gustavo A. R. Silva
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-5@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe;I think this existing is just a mess, can we fix it properly?
Quoted 7 lines
What this code wants to do is to have
rxe_resp_infoallocate arxe_recv_wqeand have a maximum size of its flex array pre-allocated.Probably like this, though maybe we need a FLEX version of the
INIT_RDMA_OBJ_SIZEmacro to make it safer.Zhu, this seems like a good thing for you to look at?
Thanks. I’ll take a closer look at this.
I’ll provide my comments over the weekend.
Zhu Yanjun
Quoted 58 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +2
-1@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, rxe_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, rxe_cq, ibcq),INIT_RDMA_OBJ_SIZE (ib_pd, rxe_pd, ibpd), -INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp), +/* For resp.srq_wqe.dma.sge */ +INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp) +RXE_MAX_SGE *sizeof (struct ib_sge ),INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc), INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),drivers/infiniband/sw/rxe/rxe_verbs.h +4
-7@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */ @@ -232,6 +226,9 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only. srq_wqe.dma.sge is a flex array */ +struct rxe_recv_wqe srq_wqe; };struct rxe_qp {@@ -269,7 +266,6 @@ struct rxe_qp {struct rxe_req_info req;struct rxe_comp_info comp; -struct rxe_resp_info resp;atomic_t ssn;atomic_t skb_out;@@ -289,6 +285,7 @@ struct rxe_qp {spinlock_t state_lock;/* guard requester and completer */ struct execute_work cleanup_work; +struct rxe_resp_info resp; };enum {
Quoted 57 lines · Jason Gunthorpe
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 226
Quoted 13 lines · Gustavo A. R. Silva
drivers/infiniband/sw/rxe/rxe_verbs.h +0
-5@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe;I think this existing is just a mess, can we fix it properly?
What this code wants to do is to have
rxe_resp_infoallocate arxe_recv_wqeand have a maximum size of its flex array pre-allocated.Probably like this, though maybe we need a FLEX version of the
INIT_RDMA_OBJ_SIZEmacro to make it safer.Zhu, this seems like a good thing for you to look at?
drivers/infiniband/sw/rxe/rxe_verbs.c +2
-1@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, rxe_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, rxe_cq, ibcq),INIT_RDMA_OBJ_SIZE (ib_pd, rxe_pd, ibpd), -INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp), +/* For resp.srq_wqe.dma.sge */ +INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp) +RXE_MAX_SGE *sizeof (struct ib_sge ),INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc), INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),drivers/infiniband/sw/rxe/rxe_verbs.h +0
-6@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 231
Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +3
-0@@ -232,6 +226,9 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only. srq_wqe.dma.sge is a flex array */ +struct rxe_recv_wqe srq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has no member named wqe
289 | qp->resp.wqe = &qp->resp.srq_wqe.wqe;
| ^
struct {
struct rxe_recv_wqe wqe;
struct ib_sge sge[RXE_MAX_SGE];
} srq_wqe;IMO, it is better to move this struct to the end of the struct rxe_resp_info.
Yanjun.Zhu
Quoted 19 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +1
-1};struct rxe_qp {@@ -269,7 +266,6 @@ struct rxe_qp {struct rxe_req_info req;struct rxe_comp_info comp; -struct rxe_resp_info resp;atomic_t ssn;atomic_t skb_out;@@ -289,6 +285,7 @@ struct rxe_qp {spinlock_t state_lock;/* guard requester and completer */ struct execute_work cleanup_work; +struct rxe_resp_info resp; };enum {
Jason Gunthorpe
maintainerQuoted 4 lines · Zhu Yanjun
Quoted 4 lines
unsigned int
res_head;
unsigned intres_tail;
structresp_res*res;
+
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 231
Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2
-0+/* SRQ only. srq_wqe.dma.sge is a flex array */ +struct rxe_recv_wqe srq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe;
| ^
I didn't try to fix all the typos, you will need to do that.
Jason
Quoted 14 lines · Jason Gunthorpe
Quoted 4 lines · Zhu Yanjun
Quoted 4 lines
unsigned int res_head; unsigned int res_tail; struct resp_res *res;+
Quoted 8 lines
drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 231
Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2
-0+/* SRQ only. srq_wqe.dma.sge is a flex array */ +struct rxe_recv_wqe srq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe;
| ^
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Yanjun.Zhu
Quoted 2 lines
Jason
Quoted 15 lines · yanjun.zhu
Quoted 14 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head; unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe;
| ^
Quoted 3 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
I am not sure if this commit is good or not.
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {INIT_RDMA_OBJ_SIZE (ib_ah, rxe_ah, ibah),INIT_RDMA_OBJ_SIZE (ib_cq, rxe_cq, ibcq),INIT_RDMA_OBJ_SIZE (ib_pd, rxe_pd, ibpd), -INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp), +/* For resp.srq_wqe.dma.sge */ +INIT_RDMA_OBJ_SIZE (ib_qp, rxe_qp, ibqp) +RXE_MAX_SGE *sizeof (struct ib_sge ),INIT_RDMA_OBJ_SIZE (ib_srq, rxe_srq, ibsrq),INIT_RDMA_OBJ_SIZE (ib_ucontext, rxe_ucontext, ibuc), INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -7
@@ -219,12 +219,6 @@ struct rxe_resp_info {u32 rkey;u32 length; -/* SRQ only */ -struct { -struct rxe_recv_wqe wqe; -struct ib_sge sge[RXE_MAX_SGE ]; - } srq_wqe; -/* Responder resources. It's a circular list where the oldest * resource is dropped first. */
@@ -232,6 +226,12 @@ struct rxe_resp_info {unsigned int res_head;unsigned int res_tail;struct resp_res *res; + +/* SRQ only. srq_wqe.dma.sge is a flex array */ +struct { +struct rxe_recv_wqe wqe; +struct ib_sge sge[RXE_MAX_SGE ]; + } srq_wqe; };struct rxe_qp {
@@ -269,7 +269,6 @@ struct rxe_qp {struct rxe_req_info req;struct rxe_comp_info comp; -struct rxe_resp_info resp;atomic_t ssn;atomic_t skb_out;
@@ -289,6 +288,7 @@ struct rxe_qp {spinlock_t state_lock;/* guard requester and
completer */
struct execute_work cleanup_work;
+ struct rxe_resp_info resp;
};
enum {
Yanjun.Zhu
Quoted 6 lines
Yanjun.Zhu
Quoted 2 lines
Jason
Signature · 2 lines
Best Regards, Yanjun.Zhu
Quoted 18 lines · Zhu Yanjun
Quoted 15 lines · yanjun.zhu
Quoted 14 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head; unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe;
| ^
Quoted 8 lines
Quoted 3 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
I am not sure if this commit is good or not.
Hi, Jason && Leon
Any update? If this looks good to you, I will send out an official commit based on the following commit.
Thanks,
Yanjun.Zhu
Quoted 74 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +2
-1@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = { INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah), INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd), - INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp), + /* For resp.srq_wqe.dma.sge */ + INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp) +RXE_MAX_SGE *sizeof (struct ib_sge ), INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),drivers/infiniband/sw/rxe/rxe_verbs.h +6
-7@@ -219,12 +219,6 @@ struct rxe_resp_info { u32 rkey; u32 length; - /* SRQ only */ - struct { - struct rxe_recv_wqe wqe; - struct ib_sge sge[RXE_MAX_SGE]; - } srq_wqe; - /* Responder resources. It's a circular list where the oldest* resource is dropped first. */ @@ -232,6 +226,12 @@ struct rxe_resp_info { unsigned int res_head; unsigned int res_tail; struct resp_res *res; + + /* SRQ only. srq_wqe.dma.sge is a flex array */ + struct { + struct rxe_recv_wqe wqe; + struct ib_sge sge[RXE_MAX_SGE]; + } srq_wqe; }; struct rxe_qp {@@ -269,7 +269,6 @@ struct rxe_qp { struct rxe_req_info req; struct rxe_comp_info comp; - struct rxe_resp_info resp; atomic_t ssn; atomic_t skb_out;@@ -289,6 +288,7 @@ struct rxe_qp { spinlock_t state_lock; /* guard requester andcompleter */
struct execute_work
cleanup_work;
+ struct rxe_resp_info resp;
};enum {
Yanjun.Zhu
Quoted 6 lines
Yanjun.Zhu
Quoted 2 lines
Jason
Signature · 2 lines
Best Regards, Yanjun.Zhu
Quoted 31 lines · Zhu Yanjun
Quoted 28 lines · Zhu Yanjun
Quoted 16 lines · yanjun.zhu
Quoted 15 lines · Jason Gunthorpe
Quoted 13 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head; unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas
no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe; | ^Quoted 4 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
Quoted 6 lines
Quoted 1 line
I am not sure if this commit is good or not.
Hi, Jason && Leon
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Thanks
Quoted 27 lines · Leon Romanovsky
Quoted 26 lines · Zhu Yanjun
Quoted 24 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head; unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas
no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe; | ^Quoted 3 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
Quoted 6 lines
Quoted 5 lines
Quoted 1 line
I am not sure if this commit is good or not.
Hi, Jason && Leon
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Will do. I will send it out very soon.
Yanjun.Zhu
Quoted 2 lines
Thanks
Quoted 29 lines · Yanjun.Zhu
Quoted 27 lines · Leon Romanovsky
Quoted 26 lines · Zhu Yanjun
Quoted 24 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head; unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas
no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe; | ^Quoted 3 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
Quoted 8 lines
Quoted 6 lines
Quoted 5 lines
Quoted 1 line
I am not sure if this commit is good or not.
Hi, Jason && Leon
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Will do. I will send it out very soon.
I don't see how this addresses the flex-array in the middle issue that originated this discussion.
-Gustavo
Quoted 35 lines · Gustavo A. R. Silva
Quoted 32 lines · Yanjun.Zhu
Quoted 30 lines · Leon Romanovsky
Quoted 29 lines · Zhu Yanjun
Quoted 17 lines · Zhu Yanjun
Quoted 15 lines · yanjun.zhu
Quoted 14 lines · Jason Gunthorpe
Quoted 6 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head;
unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas
no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe; | ^I didn't try to fix all the typos, you will need to do that.
Quoted 8 lines
Quoted 2 lines
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
I am not sure if this commit is good or not.
Hi, Jason && Leon
Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Will do. I will send it out very soon.
I don't see how this addresses the flex-array in the middle issue that originated this discussion.
Could you please explain this in more detail?
Also, if you have a better approach, would you mind sending a new commit for discussion?
Thanks a lot.
Yanjun.Zhu
Quoted 2 lines
-Gustavo
Signature · 2 lines
Best Regards, Yanjun.Zhu
Quoted 41 lines · Zhu Yanjun
Quoted 39 lines · Gustavo A. R. Silva
Quoted 36 lines · Yanjun.Zhu
Quoted 33 lines · Leon Romanovsky
Quoted 23 lines · Zhu Yanjun
Quoted 21 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head;
unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas
no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe; | ^Quoted 3 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
Quoted 8 lines
Quoted 4 lines
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
I am not sure if this commit is good or not.
Hi, Jason && Leon
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
Will do. I will send it out very soon.
I don't see how this addresses the flex-array in the middle issue that originated this discussion.
Could you please explain this in more detail?
Also, if you have a better approach, would you mind sending a new commit for discussion?
This is exactly what my original patch is about. I've explained this a couple of times in this thread already.
-Gustavo
Quoted 46 lines · Gustavo A. R. Silva
Quoted 43 lines · Zhu Yanjun
Quoted 41 lines · Gustavo A. R. Silva
Quoted 37 lines · Yanjun.Zhu
Quoted 27 lines · Leon Romanovsky
Quoted 26 lines · Zhu Yanjun
Quoted 24 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines
unsigned int
res_head;
unsigned intres_tail;
struct resp_res *res;
+
+ /* SRQ only. srq_wqe.dma.sge is a flex array */ + structrxe_recv_wqesrq_wqe;
drivers/infiniband/sw/rxe/rxe_resp.c: In functionget_srq_wqe:drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: structrxe_recv_wqehas
no member named wqe
289 |qp->resp.wqe=&qp->resp.srq_wqe.wqe; | ^Quoted 3 lines
Quoted 1 line
I didn't try to fix all the typos, you will need to do that.
Exactly. I will fix this problem. This weekend, I will send out an official commit.
Hi, Jason
The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.
Quoted 6 lines
Quoted 5 lines
Quoted 1 line
I am not sure if this commit is good or not.
Hi, Jason && Leon
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Will do. I will send it out very soon.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
I don't see how this addresses the flex-array in the middle issue that originated this discussion.
Could you please explain this in more detail?
Also, if you have a better approach, would you mind sending a new commit for discussion?This is exactly what my original patch is about. I've explained this a couple of times in this thread already.
In your original patch, I have the following problem. I read all your replies. But I can not find your answers to my problem.
My problem is as below:
1. The followings are the declaration of __TRAILING_OVERLAP
/**
* __TRAILING_OVERLAP() - Overlap a flexible-array member with trailing
* members.
*
* Creates a union between a flexible-array member (FAM) in a struct
and a set
* of additional members that would otherwise follow it.
*
* @TYPE: Flexible structure type name, including "struct" keyword.
* @NAME: Name for a variable to define.
* @FAM: The flexible-array member within @TYPE <----- Here
* @ATTRS: Any struct attributes (usually empty)
* @MEMBERS: Trailing overlapping members.
*/
#define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS) \
union { \
TYPE NAME; \
struct { \
unsigned char __offset_to_FAM[offsetof(TYPE, FAM)]; \
MEMBERS \
} ATTRS; \
}2. I expanded the above MACRO __TRAILING_OVERLAP into the following following the definition in your commit.
union {
struct rxe_recv_wqe wqe;
struct {
unsigned char __offset_to_FAM[offsetof(struct rxe_recv_wqe,
dma.sge)]; \
struct ib_sge sge[RXE_MAX_SGE];
} ;
}srq_wqe;3. In your original commit, dma.sge should be the member of struct rxe_recv_wqe. The struct rxe_recv_wqe is as below.
"@FAM: The flexible-array member within @TYPE"
The struct rxe_recv_wqe is as below.
struct rxe_recv_wqe {
__aligned_u64 wr_id;
__u32 reserved;
__u32 padding;
struct rxe_dma_info dma;
};But I can not find dma.sge in the above struct. Can you explain it?
To be honest, I read your original commit for several times, but I can not get it. Can you explain the MACRO TRAILING_OVERLAP? And how can it replace the following struct?
struct {
struct rxe_recv_wqe wqe;
struct ib_sge sge[RXE_MAX_SGE];
} srq_wqe;Thanks a lot.
Yanjun.Zhu
Quoted 2 lines
-Gustavo
Quoted 8 lines
The struct
rxe_recv_wqeis as below.struct
rxe_recv_wqe{
__aligned_u64wr_id;
__u32 reserved;
__u32 padding;
structrxe_dma_infodma;
Expand struct rxe_dma_info here.
Quoted 5 lines
};
But I can not find dma.sge in the above struct. Can you explain it?
To be honest, I read your original commit for several times, but I can not get it. Can you explain the MACRO
TRAILING_OVERLAP? And how can it replace the following struct?
This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.
-Gustavo
Quoted 10 lines · Gustavo A. R. Silva
Quoted 8 lines
The struct
rxe_recv_wqeis as below.struct
rxe_recv_wqe{
__aligned_u64wr_id;
__u32 reserved;
__u32 padding;
structrxe_dma_infodma;Expand struct
rxe_dma_infohere.
Thanks. In struct rxe_dma_info, the struct is
struct rxe_sge {
__aligned_u64 addr;
__u32 length;
__u32 lkey;
};
But in your commit, struct ib_sge is used.
struct ib_sge {
u64 addr;
u32 length;
u32 lkey;
};__aligned_u64 is a 64-bit integer with a guaranteed 8-byte alignment,
used to preserve ABI correctness across architectures and between
userspace and kernel, while u64 has architecture-dependent alignment.
I am not sure if we can treate "struct rxe_sge" as the same with "struct ib_sge".
Leon and Jason, please comment on it.
Yanjun.Zhu
Quoted 12 lines
Quoted 6 lines
};
But I can not find dma.sge in the above struct. Can you explain it?
To be honest, I read your original commit for several times, but I can not get it. Can you explain the MACRO
TRAILING_OVERLAP? And how can it replace the following struct?This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.
-Gustavo
Quoted 30 lines · Zhu Yanjun
Quoted 10 lines · Gustavo A. R. Silva
Quoted 8 lines
The struct
rxe_recv_wqeis as below.struct
rxe_recv_wqe{
__aligned_u64wr_id;
__u32 reserved;
__u32 padding;
structrxe_dma_infodma;Expand struct
rxe_dma_infohere.Thanks. In struct
rxe_dma_info, the struct isstruct
rxe_sge{
__aligned_u64 addr;
__u32 length;
__u32 lkey;
};But in your commit, struct
ib_sgeis used.struct
ib_sge{
u64 addr;
u32 length;
u32 lkey;
};__aligned_u64is a 64-bit integer with a guaranteed 8-byte alignment,
Quoted 5 lines
used to preserve ABI correctness across architectures and between
userspace and kernel, while u64 has architecture-dependent alignment.
I am not sure if we can treate "struct
rxe_sge" as the same with "structib_sge".
Just notice that the original code is the one actually doing that. See my response in this same thread:
https://lore.kernel.org/linux-hardening/[email protected]/
So, if that code is fine, this is fine. If the original code is wrong, then that code should be fixed first.
-Gustavo
Quoted 20 lines
Leon and Jason, please comment on it.
Yanjun.Zhu
Quoted 12 lines
Quoted 6 lines
};
But I can not find dma.sge in the above struct. Can you explain it?
To be honest, I read your original commit for several times, but I can not get it. Can you explain the MACRO
TRAILING_OVERLAP? And how can it replace the following struct?This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.
-Gustavo
Jason Gunthorpe
maintainerQuoted 8 lines · Gustavo A. R. Silva
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Will do. I will send it out very soon.
I don't see how this addresses the flex-array in the middle issue that originated this discussion.
??
It moved it to the end of every struct it is part of?
Jason
Quoted 39 lines · Gustavo A. R. Silva
Quoted 30 lines · Zhu Yanjun
Quoted 10 lines · Gustavo A. R. Silva
Quoted 8 lines
The struct
rxe_recv_wqeis as below.struct
rxe_recv_wqe{
__aligned_u64wr_id;
__u32 reserved;
__u32 padding;
structrxe_dma_infodma;Expand struct
rxe_dma_infohere.Thanks. In struct
rxe_dma_info, the struct isstruct
rxe_sge{
__aligned_u64 addr;
__u32 length;
__u32 lkey;
};But in your commit, struct
ib_sgeis used.struct
ib_sge{
u64 addr;
u32 length;
u32 lkey;
};__aligned_u64is a 64-bit integer with a guaranteed 8-byte alignment,Quoted 5 lines
used to preserve ABI correctness across architectures and between
userspace and kernel, while u64 has architecture-dependent alignment.
I am not sure if we can treate "struct
rxe_sge" as the same with "structib_sge".
Quoted 7 lines
Just notice that the original code is the one actually doing that. See my response in this same thread:
https://lore.kernel.org/linux-hardening/d3336e9d-2b84-4698-a799- [email protected]/
So, if that code is fine, this is fine. If the original code is wrong, then that code should be fixed first.
Thanks a lot. Because struct ib_sge and struct ib_sge is different,
struct ib_sge {
u64 addr; <--- u64 has architecture-dependent alignment
u32 length;
u32 lkey;
};struct rxe_sge {
__aligned_u64 addr; <---guaranteed 8-byte alignment,
used to preserve ABI correctness across architectures and between
userspace and kernel
__u32 length;
__u32 lkey;
};and struct rxe_sge is used in rxe_mr, it is working between userspace and kernel, thus, I want to keep struct rxe_mr in rxe_mr.
But in other places, I want to replace struct rxe_sge with struct ib_sge. The commit is as below.
In short, the commit "RDMA/rxe: Avoid -Wflex-array-member-not-at-end warnings" && the following commit will work well. I have made tests in my local host. It can work well.
Please Leon and Jason comment.
drivers/infiniband/sw/rxe/rxe_mr.c +2 -2
@@ -341,7 +341,7 @@ int copy_data( enum rxe_mr_copy_dir dir) {int bytes; -struct rxe_sge *sge = &dma->sge[dma->cur_sge]; +struct ib_sge *sge = &dma->sge[dma->cur_sge];int offset = dma->sge_offset;int resid = dma->resid;struct rxe_mr *mr =NULL ;
@@ -580,7 +580,7 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)int advance_dma_data (struct rxe_dma_info *dma,unsigned int length) { -struct rxe_sge *sge = &dma->sge[dma->cur_sge]; +struct ib_sge *sge = &dma->sge[dma->cur_sge];int offset = dma->sge_offset;int resid = dma->resid;
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
@@ -283,7 +283,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)rxe_dbg_qp (qp,"invalid num_sge in SRQ entry\n" );return RESPST_ERR_MALFORMED_WQE ; } - size =sizeof (*wqe) + wqe->dma.num_sge*sizeof (struct rxe_sge ); + size =sizeof (*wqe) + wqe->dma.num_sge*sizeof (struct ib_sge );memcpy (&qp->resp.srq_wqe, wqe, size); qp->resp.wqe = &qp->resp.srq_wqe.wqe;
include/uapi/rdma/rdma_user_rxe.h +1 -1
@@ -154,7 +154,7 @@ struct rxe_dma_info {union { __DECLARE_FLEX_ARRAY(__u8, inline_data); __DECLARE_FLEX_ARRAY(__u8, atomic_wr); - __DECLARE_FLEX_ARRAY(struct rxe_sge, sge); + __DECLARE_FLEX_ARRAY(struct ib_sge, sge); }; };
To this commit, plus the above commit, it should work well.
Yanjun.Zhu
Quoted 24 lines
-Gustavo
Quoted 20 lines
Leon and Jason, please comment on it.
Yanjun.Zhu
Quoted 12 lines
Quoted 6 lines
};
But I can not find dma.sge in the above struct. Can you explain it?
To be honest, I read your original commit for several times, but I can not get it. Can you explain the MACRO
TRAILING_OVERLAP? And how can it replace the following struct?This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.
-Gustavo
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Gustavo A. R. Silva
Quoted 4 lines
Quoted 3 lines
Quoted 2 lines
Any update? If this looks good to you, I will send out an official commit based on the following commit.
You are RXE maintainer, send the official patch.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Will do. I will send it out very soon.
I don't see how this addresses the flex-array in the middle issue that originated this discussion.
??
It moved it to the end of every struct it is part of?
Any update?
Yanjun.Zhu
Quoted 2 lines
Jason