Skip to contentSkip to conversation

RDMA/rxe: Avoid -Wflex-array-member-not-at-end warnings

PATCH In review Gustavo A. R. Silva 6 participants 28 messages Nov 11, 2025 – Dec 26, 2025

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 RDMA/rxe: Avoid -Wflex-array-member-not-at-end warnings

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
 drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +11 -7
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
 	u32			rkey;
 	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
 	/* Responder resources. It's a circular list where the oldest
 	 * resource is dropped first.
 	 */
@@ -232,7 +226,15 @@ struct rxe_resp_info {
 	unsigned int		res_head;
 	unsigned int		res_tail;
 	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;
 };
+static_assert(offsetof(struct rxe_resp_info, srq_wqe.wqe.dma.sge) ==
+	      offsetof(struct rxe_resp_info, srq_wqe.sge));
 
 struct rxe_qp {
 	struct ib_qp		ibqp;
@@ -269,7 +271,6 @@ struct rxe_qp {
 
 	struct rxe_req_info	req;
 	struct rxe_comp_info	comp;
-	struct rxe_resp_info	resp;
 
 	atomic_t		ssn;
 	atomic_t		skb_out;
@@ -289,6 +290,9 @@ struct rxe_qp {
 	spinlock_t		state_lock; /* guard requester and completer */
 
 	struct execute_work	cleanup_work;
+
+	/* Must be last as it ends in a flexible-array member. */
+	struct rxe_resp_info	resp;
 };
 
 enum {
Signature · 1 line
2.43.0
#1 lore raw
Quoted 44 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
 drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
 	u32			rkey;
 	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
 	/* Responder resources. It's a circular list where the oldest
 	 * resource is dropped first.
 	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
 	unsigned int		res_head;
 	unsigned int		res_tail;
 	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +6 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
        u32                     rkey;
        u32                     length;
 
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
        /* Responder resources. It's a circular list where the oldest
         * resource is dropped first.
         */
@@ -232,6 +226,12 @@ struct rxe_resp_info {
        unsigned int            res_head;
        unsigned int            res_tail;
        struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
 };
 
 struct rxe_qp {

~

#2 lore raw
Quoted 75 lines · Leon Romanovsky
Quoted 44 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
  drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
  1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
  	u32			rkey;
  	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
  	/* Responder resources. It's a circular list where the oldest
  	 * resource is dropped first.
  	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
  	unsigned int		res_head;
  	unsigned int		res_tail;
  	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
         u32                     rkey;
         u32                     length;
 
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
         /* Responder resources. It's a circular list where the oldest
          * resource is dropped first.
          */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
         unsigned int            res_head;
         unsigned int            res_tail;
         struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
  };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
        u64     addr;
        u32     length;
        u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
        __aligned_u64 addr;
        __u32   length;
        __u32   lkey;
};

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

Thanks
-Gustavo

#3 lore raw
Quoted 105 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
  drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
  1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
  	u32			rkey;
  	u32			length;
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
  	/* Responder resources. It's a circular list where the oldest
  	 * resource is dropped first.
  	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
  	unsigned int		res_head;
  	unsigned int		res_tail;
  	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
         u32                     rkey;
         u32                     length;
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
         /* Responder resources. It's a circular list where the oldest
          * resource is dropped first.
          */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
         unsigned int            res_head;
         unsigned int            res_tail;
         struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
  };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
        u64     addr;
        u32     length;
        u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
        __aligned_u64 addr;
        __u32   length;
        __u32   lkey;
};
Quoted 7 lines

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".

My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.

Thanks

Quoted 8 lines

Thanks
-Gustavo

#4 lore raw
Quoted 108 lines · Leon Romanovsky
Quoted 107 lines · Gustavo A. R. Silva
Quoted 72 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
   drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
   1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
   	u32			rkey;
   	u32			length;
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
   	/* Responder resources. It's a circular list where the oldest
   	 * resource is dropped first.
   	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
   	unsigned int		res_head;
   	unsigned int		res_tail;
   	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
          u32                     rkey;
          u32                     length;
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
          /* Responder resources. It's a circular list where the oldest
           * resource is dropped first.
           */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
          unsigned int            res_head;
          unsigned int            res_tail;
          struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
   };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
         u64     addr;
         u32     length;
         u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
         __aligned_u64 addr;
         __u32   length;
         __u32   lkey;
};

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Quoted 7 lines
Quoted 2 lines

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".

My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.

Thanks, I agree with this approach. The macro is rather complicated, and this solution fixes the problem in a straightforward way.

Yanjun.Zhu

Quoted 10 lines

Thanks

Quoted 7 lines

Thanks
-Gustavo

Signature · 2 lines
Best Regards,
Yanjun.Zhu
#5 lore raw
In reply to Leon Romanovsky
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
   drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
   1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
   	u32			rkey;
   	u32			length;
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
   	/* Responder resources. It's a circular list where the oldest
   	 * resource is dropped first.
   	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
   	unsigned int		res_head;
   	unsigned int		res_tail;
   	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
          u32                     rkey;
          u32                     length;
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
          /* Responder resources. It's a circular list where the oldest
           * resource is dropped first.
           */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
          unsigned int            res_head;
          unsigned int            res_tail;
          struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
   };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
         u64     addr;
         u32     length;
         u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
         __aligned_u64 addr;
         __u32   length;
         __u32   lkey;
};

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Quoted 8 lines
Quoted 2 lines

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".

My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.

Yep, but as I commented above, that doesn't seem to be the right change.

Look at the following couple of lines:

drivers/infiniband/sw/rxe/rxe_resp.c-286- size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
drivers/infiniband/sw/rxe/rxe_resp.c-287- memcpy(&qp->resp.srq_wqe, wqe, size);

Notice that line 286 is the open-coded arithmetic (struct_size(wqe, dma.sge, wqe->dma.num_sge) is preferred) to get the number of bytes to allocate for a flexible structure, in this case struct rxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].

So, size bytes are written in qp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation of RXE_MAX_SGE number of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:

struct {
	struct rxe_recv_wqe	wqe;
	struct ib_sge		sge[RXE_MAX_SGE];
} srq_wqe;

So, unless I'm missing something, struct ib_sge sge[RXE_MAX_SGE]; should be aligned with struct rxe_recv_wqe wqe::dma.sge[].

The TRAILING_OVERLAP() macro is also designed to ensure alignment in these cases (and the static_assert() to preserve it). See this thread:

https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/

Thanks
-Gustavo

#6 lore raw
Quoted 135 lines · Gustavo A. R. Silva
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
   drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
   1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
   	u32			rkey;
   	u32			length;
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
   	/* Responder resources. It's a circular list where the oldest
   	 * resource is dropped first.
   	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
   	unsigned int		res_head;
   	unsigned int		res_tail;
   	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
          u32                     rkey;
          u32                     length;
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
          /* Responder resources. It's a circular list where the oldest
           * resource is dropped first.
           */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
          unsigned int            res_head;
          unsigned int            res_tail;
          struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
   };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
         u64     addr;
         u32     length;
         u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
         __aligned_u64 addr;
         __u32   length;
         __u32   lkey;
};

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Quoted 8 lines
Quoted 2 lines

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".

My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.

Yep, but as I commented above, that doesn't seem to be the right change.

Look at the following couple of lines:

drivers/infiniband/sw/rxe/rxe_resp.c-286- size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
drivers/infiniband/sw/rxe/rxe_resp.c-287- memcpy(&qp->resp.srq_wqe, wqe, size);

Notice that line 286 is the open-coded arithmetic (struct_size(wqe, dma.sge, wqe->dma.num_sge) is preferred) to get the number of bytes to allocate for a flexible structure, in this case struct rxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].

Quoted 8 lines

So, size bytes are written in qp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation of RXE_MAX_SGE number of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:

struct {
	struct rxe_recv_wqe	wqe;
	struct ib_sge		sge[RXE_MAX_SGE];
} srq_wqe;

So you are saying that it works because it is written properly, so what is the problem? Why do we need to fix properly working and written code to be less readable?

Quoted 3 lines

So, unless I'm missing something, struct ib_sge sge[RXE_MAX_SGE]; should be aligned with struct rxe_recv_wqe wqe::dma.sge[].

It is and moving to the end of struct will continue to keep it aligned.

Quoted 12 lines

The TRAILING_OVERLAP() macro is also designed to ensure alignment in these cases (and the static_assert() to preserve it). See this thread:

https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/

Thanks
-Gustavo

#7 lore raw
Quoted 140 lines · Leon Romanovsky
Quoted 139 lines · Gustavo A. R. Silva
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
    drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
    1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
    	u32			rkey;
    	u32			length;
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
    	/* Responder resources. It's a circular list where the oldest
    	 * resource is dropped first.
    	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
    	unsigned int		res_head;
    	unsigned int		res_tail;
    	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
           u32                     rkey;
           u32                     length;
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
           /* Responder resources. It's a circular list where the oldest
            * resource is dropped first.
            */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
           unsigned int            res_head;
           unsigned int            res_tail;
           struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
    };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
          u64     addr;
          u32     length;
          u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
          __aligned_u64 addr;
          __u32   length;
          __u32   lkey;
};

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Quoted 8 lines
Quoted 2 lines

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".

My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.

Yep, but as I commented above, that doesn't seem to be the right change.

Look at the following couple of lines:

drivers/infiniband/sw/rxe/rxe_resp.c-286- size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
drivers/infiniband/sw/rxe/rxe_resp.c-287- memcpy(&qp->resp.srq_wqe, wqe, size);

Notice that line 286 is the open-coded arithmetic (struct_size(wqe, dma.sge, wqe->dma.num_sge) is preferred) to get the number of bytes to allocate for a flexible structure, in this case struct rxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].

So, size bytes are written in qp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation of RXE_MAX_SGE number of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:

Quoted 8 lines
Quoted 4 lines
struct {
	struct rxe_recv_wqe	wqe;
	struct ib_sge		sge[RXE_MAX_SGE];
} srq_wqe;

So you are saying that it works because it is written properly, so what is the problem? Why do we need to fix properly working and written code to be less readable?

No one said the original code is not working as expected. The issue here is that the FAM is not at the end, and this causes a -Wflex-array-member-not-at-end warning. The change I propose places the FAM at the end, and the functionality remains exactly the same.

You're probably not aware of the work we've been doing to enable -Wflex-array-member-not-at-end in mainline. If you're interested, below you can take a look at other similar changes I (and others) have been doing to complete this work:

https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/log/?qt=grep&q=-Wflex-array-member-not-at-end

Quoted 6 lines
Quoted 4 lines

So, unless I'm missing something, struct ib_sge sge[RXE_MAX_SGE]; should be aligned with struct rxe_recv_wqe wqe::dma.sge[].

It is and moving to the end of struct will continue to keep it aligned.

I think there is something you are missing here. The following pieces of code are no equivalent:

struct {
	struct rxe_recv_wqe	wqe;

struct ib_sge sge[RXE_MAX_SGE];
} srq_wqe;

struct {

 	struct ib_sge		sge[RXE_MAX_SGE];
	struct rxe_recv_wqe	wqe;
} srq_wqe;

What I'm understanding from your last couple of responses is that you think the above are equivalent. My previous response tried to explain why that is not the case.

Quoted 7 lines
Quoted 7 lines

The TRAILING_OVERLAP() macro is also designed to ensure alignment in these cases (and the static_assert() to preserve it). See this thread:

https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/

Thanks
-Gustavo

#8 lore raw
Quoted 157 lines · Gustavo A. R. Silva
Quoted 140 lines · Leon Romanovsky
Quoted 139 lines · Gustavo A. R. Silva
Quoted 111 lines · Leon Romanovsky
Quoted 110 lines · Gustavo A. R. Silva
Quoted 73 lines · Leon Romanovsky
Quoted 43 lines · Gustavo A. R. Silva

-Wflex-array-member-not-at-end was introduced in GCC-14, and we are getting ready to enable it, globally.

Use the new TRAILING_OVERLAP() helper to fix the following warning:

21 drivers/infiniband/sw/rxe/rxe_verbs.h:271:33: warning: structure containing a flexible array member is not at the end of another structure [-Wflex-array-member-not-at-end]

This helper creates a union between a flexible-array member (FAM) and a set of MEMBERS that would otherwise follow it.

This overlays the trailing MEMBER struct ib_sge sge[RXE_MAX_SGE]; onto the FAM struct rxe_recv_wqe::dma.sge, while keeping the FAM and the start of MEMBER aligned.

The static_assert() ensures this alignment remains, and it's intentionally placed inmediately after the related structure --no blank line in between.

Lastly, move the conflicting declaration struct rxe_resp_info resp; to the end of the corresponding structure.

---
    drivers/infiniband/sw/rxe/rxe_verbs.h | 18 +++++++++++-------
    1 file changed, 11 insertions(+), 7 deletions(-)
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
    	u32			rkey;
    	u32			length;
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
    	/* Responder resources. It's a circular list where the oldest
    	 * resource is dropped first.
    	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 234

Quoted 9 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,7 +226,15 @@ struct rxe_resp_info {
    	unsigned int		res_head;
    	unsigned int		res_tail;
    	struct resp_res		*res;
+
+	/* SRQ only */
+	/* Must be last as it ends in a flexible-array member. */
+	TRAILING_OVERLAP(struct rxe_recv_wqe, wqe, dma.sge,
+		struct ib_sge		sge[RXE_MAX_SGE];
+	) srq_wqe;

Will this change be enough?

drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd..9ab11421a585 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
           u32                     rkey;
           u32                     length;
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
           /* Responder resources. It's a circular list where the oldest
            * resource is dropped first.
            */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 235

Quoted 10 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -0
@@ -232,6 +226,12 @@ struct rxe_resp_info {
           unsigned int            res_head;
           unsigned int            res_tail;
           struct resp_res         *res;
+
+       /* SRQ only */
+       struct {
+               struct ib_sge           sge[RXE_MAX_SGE];
+               struct rxe_recv_wqe     wqe;
+       } srq_wqe;
    };

The question is if this is really what you want?

sge[RXE_MAX_SGE] is of the following type:

struct ib_sge {
          u64     addr;
          u32     length;
          u32     lkey;
};

and struct rxe_recv_wqe::dma.sge[] is of type:

struct rxe_sge {
          __aligned_u64 addr;
          __u32   length;
          __u32   lkey;
};

Both types are basically the same, and the original code looks pretty much like what people do when they want to pre-allocate a number of elements (of the same element type as the flex array) for a flexible-array member.

Quoted 8 lines
Quoted 2 lines

Based on the above, the change you suggest seems a bit suspicious, and I'm not sure that's actually what you want?

You wrote about this error: "warning: structure containing a flexible array member is not at the end of another structure".

My suggestion was simply to move that flex array to be the last element and save us from the need to have some complex, magic macro in RXE.

Yep, but as I commented above, that doesn't seem to be the right change.

Look at the following couple of lines:

drivers/infiniband/sw/rxe/rxe_resp.c-286- size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
drivers/infiniband/sw/rxe/rxe_resp.c-287- memcpy(&qp->resp.srq_wqe, wqe, size);

Notice that line 286 is the open-coded arithmetic (struct_size(wqe, dma.sge, wqe->dma.num_sge) is preferred) to get the number of bytes to allocate for a flexible structure, in this case struct rxe_recv_wqe, and its flexible-array member, in this case struct rxe_recv_wqe::dma.sge[].

So, size bytes are written in qp->resp.srq_wqe, and the reason this works seems to be because of the pre-allocation of RXE_MAX_SGE number of elements for flex array struct rxe_recv_wqe::dma.sge[] given by:

Quoted 8 lines
Quoted 4 lines
struct {
	struct rxe_recv_wqe	wqe;
	struct ib_sge		sge[RXE_MAX_SGE];
} srq_wqe;

So you are saying that it works because it is written properly, so what is the problem? Why do we need to fix properly working and written code to be less readable?

No one said the original code is not working as expected. The issue here is that the FAM is not at the end, and this causes a -Wflex-array-member-not-at-end warning. The change I propose places the FAM at the end, and the functionality remains exactly the same.

Quoted 6 lines

You're probably not aware of the work we've been doing to enable -Wflex-array-member-not-at-end in mainline. If you're interested, below you can take a look at other similar changes I (and others) have been doing to complete this work:

https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git/log/?qt=grep&q=-Wflex-array-member-not-at-end

I'm aware of that work, just trying to find a simplest possible solution for this specific driver. RXE is a development driver which won't benefit from this hardening work at all.

Quoted 16 lines
Quoted 7 lines
Quoted 5 lines

So, unless I'm missing something, struct ib_sge sge[RXE_MAX_SGE]; should be aligned with struct rxe_recv_wqe wqe::dma.sge[].

It is and moving to the end of struct will continue to keep it aligned.

I think there is something you are missing here. The following pieces of code are no equivalent:

struct {
	struct rxe_recv_wqe	wqe;

struct ib_sge sge[RXE_MAX_SGE];
} srq_wqe;

Quoted 8 lines

struct {

 	struct ib_sge		sge[RXE_MAX_SGE];
	struct rxe_recv_wqe	wqe;
} srq_wqe;

What I'm understanding from your last couple of responses is that you think the above are equivalent. My previous response tried to explain why that is not the case.

Yes, I already forgot about change in srq_wqe.

Thanks

Quoted 11 lines
Quoted 8 lines
Quoted 8 lines

The TRAILING_OVERLAP() macro is also designed to ensure alignment in these cases (and the static_assert() to preserve it). See this thread:

https://lore.kernel.org/linux-hardening/aLiYrQGdGmaDTtLF@kspp/

Thanks
-Gustavo

maintainer
#9 lore raw
In reply to Gustavo A. R. Silva

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 226

Quoted 13 lines · Gustavo A. R. Silva
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -5
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
 	u32			rkey;
 	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;

I think this existing is just a mess, can we fix it properly?

What this code wants to do is to have rxe_resp_info allocate a rxe_recv_wqe and have a maximum size of its flex array pre-allocated.

Probably like this, though maybe we need a FLEX version of the INIT_RDMA_OBJ_SIZE macro to make it safer.

Zhu, this seems like a good thing for you to look at?

drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
index 38d8c408320f11..189eaa59a5fb14 100644
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {
 	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
 	INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
 	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
-	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp),
+	/* For resp.srq_wqe.dma.sge */
+	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp) + RXE_MAX_SGE*sizeof(struct ib_sge),
 	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
 	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
 	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
drivers/infiniband/sw/rxe/rxe_verbs.h +4 -7
index fd48075810dd10..dda741ec3ac701 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
 	u32			rkey;
 	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
 	/* Responder resources. It's a circular list where the oldest
 	 * resource is dropped first.
 	 */
@@ -232,6 +226,9 @@ struct rxe_resp_info {
 	unsigned int		res_head;
 	unsigned int		res_tail;
 	struct resp_res		*res;
+
+	/* SRQ only. srq_wqe.dma.sge is a flex array */
+	struct rxe_recv_wqe srq_wqe;
 };
 
 struct rxe_qp {
@@ -269,7 +266,6 @@ struct rxe_qp {
 
 	struct rxe_req_info	req;
 	struct rxe_comp_info	comp;
-	struct rxe_resp_info	resp;
 
 	atomic_t		ssn;
 	atomic_t		skb_out;
@@ -289,6 +285,7 @@ struct rxe_qp {
 	spinlock_t		state_lock; /* guard requester and completer */
 
 	struct execute_work	cleanup_work;
+	struct rxe_resp_info	resp;
 };
 
 enum {
#10 lore raw
Quoted 16 lines · Jason Gunthorpe

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 226

Quoted 13 lines · Gustavo A. R. Silva
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -5
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
  	u32			rkey;
  	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;

I think this existing is just a mess, can we fix it properly?

Quoted 7 lines

What this code wants to do is to have rxe_resp_info allocate a rxe_recv_wqe and have a maximum size of its flex array pre-allocated.

Probably like this, though maybe we need a FLEX version of the INIT_RDMA_OBJ_SIZE macro to make it safer.

Zhu, this seems like a good thing for you to look at?

Thanks. I’ll take a closer look at this.

I’ll provide my comments over the weekend.

Zhu Yanjun

Quoted 58 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
index 38d8c408320f11..189eaa59a5fb14 100644
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {
  	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
  	INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
  	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
-	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp),
+	/* For resp.srq_wqe.dma.sge */
+	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp) + RXE_MAX_SGE*sizeof(struct ib_sge),
  	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
  	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
  	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
drivers/infiniband/sw/rxe/rxe_verbs.h +4 -7
index fd48075810dd10..dda741ec3ac701 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
  	u32			rkey;
  	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
  	/* Responder resources. It's a circular list where the oldest
  	 * resource is dropped first.
  	 */
@@ -232,6 +226,9 @@ struct rxe_resp_info {
  	unsigned int		res_head;
  	unsigned int		res_tail;
  	struct resp_res		*res;
+
+	/* SRQ only. srq_wqe.dma.sge is a flex array */
+	struct rxe_recv_wqe srq_wqe;
  };
 
  struct rxe_qp {
@@ -269,7 +266,6 @@ struct rxe_qp {
 
  	struct rxe_req_info	req;
  	struct rxe_comp_info	comp;
-	struct rxe_resp_info	resp;
 
  	atomic_t		ssn;
  	atomic_t		skb_out;
@@ -289,6 +285,7 @@ struct rxe_qp {
  	spinlock_t		state_lock; /* guard requester and completer */
 
  	struct execute_work	cleanup_work;
+	struct rxe_resp_info	resp;
  };
 
  enum {
#11 lore raw
In reply to Jason Gunthorpe
Quoted 57 lines · Jason Gunthorpe

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 226

Quoted 13 lines · Gustavo A. R. Silva
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -5
index fd48075810dd..6498d61e8956 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
  	u32			rkey;
  	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;

I think this existing is just a mess, can we fix it properly?

What this code wants to do is to have rxe_resp_info allocate a rxe_recv_wqe and have a maximum size of its flex array pre-allocated.

Probably like this, though maybe we need a FLEX version of the INIT_RDMA_OBJ_SIZE macro to make it safer.

Zhu, this seems like a good thing for you to look at?

drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
index 38d8c408320f11..189eaa59a5fb14 100644
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {
  	INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
  	INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
  	INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
-	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp),
+	/* For resp.srq_wqe.dma.sge */
+	INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp) + RXE_MAX_SGE*sizeof(struct ib_sge),
  	INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
  	INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
  	INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
drivers/infiniband/sw/rxe/rxe_verbs.h +0 -6
index fd48075810dd10..dda741ec3ac701 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
  	u32			rkey;
  	u32			length;
 
-	/* SRQ only */
-	struct {
-		struct rxe_recv_wqe	wqe;
-		struct ib_sge		sge[RXE_MAX_SGE];
-	} srq_wqe;
-
  	/* Responder resources. It's a circular list where the oldest
  	 * resource is dropped first.
  	 */

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 231

Quoted 6 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +3 -0
@@ -232,6 +226,9 @@ struct rxe_resp_info {
  	unsigned int		res_head;
  	unsigned int		res_tail;
  	struct resp_res		*res;
+
+	/* SRQ only. srq_wqe.dma.sge is a flex array */
+	struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has no member named wqe
289 | qp->resp.wqe = &qp->resp.srq_wqe.wqe;
| ^

struct {
	struct rxe_recv_wqe	wqe;
	struct ib_sge		sge[RXE_MAX_SGE];
} srq_wqe;

IMO, it is better to move this struct to the end of the struct rxe_resp_info.

Yanjun.Zhu

Quoted 19 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +1 -1
  };
 
  struct rxe_qp {
@@ -269,7 +266,6 @@ struct rxe_qp {
 
  	struct rxe_req_info	req;
  	struct rxe_comp_info	comp;
-	struct rxe_resp_info	resp;
 
  	atomic_t		ssn;
  	atomic_t		skb_out;
@@ -289,6 +285,7 @@ struct rxe_qp {
  	spinlock_t		state_lock; /* guard requester and completer */
 
  	struct execute_work	cleanup_work;
+	struct rxe_resp_info	resp;
  };
 
  enum {
maintainer
#12 lore raw
Quoted 4 lines · Zhu Yanjun
Quoted 4 lines

unsigned int res_head;
unsigned int res_tail;
struct resp_res *res;
+

Quoted 8 lines

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 231

Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -0
+	/* SRQ only. srq_wqe.dma.sge is a flex array */
+	struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has no member named wqe
289 | qp->resp.wqe = &qp->resp.srq_wqe.wqe;
| ^

I didn't try to fix all the typos, you will need to do that.

Jason

#13 lore raw
Quoted 14 lines · Jason Gunthorpe
Quoted 4 lines · Zhu Yanjun
Quoted 4 lines
   	unsigned int		res_head;
   	unsigned int		res_tail;
   	struct resp_res		*res;

+

Quoted 8 lines

drivers/infiniband/sw/rxe/rxe_verbs.h struct rxe_resp_info { · line 231

Quoted 2 lines
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -0
+	/* SRQ only. srq_wqe.dma.sge is a flex array */
+	struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has no member named wqe
289 | qp->resp.wqe = &qp->resp.srq_wqe.wqe;
| ^

Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Yanjun.Zhu

Quoted 2 lines

Jason

#14 lore raw
Quoted 15 lines · yanjun.zhu
Quoted 14 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines

       unsigned int        res_head; unsigned int        res_tail;
       struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has no member named wqe
   289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe;
       |                                         ^

Quoted 3 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

I am not sure if this commit is good or not.

drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
index 38d8c408320f..189eaa59a5fb 100644
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {
        INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
        INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
        INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
-       INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp),
+       /* For resp.srq_wqe.dma.sge */
+       INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp) + RXE_MAX_SGE*sizeof(struct ib_sge),
        INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
        INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),
        INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),
drivers/infiniband/sw/rxe/rxe_verbs.h +6 -7
index fd48075810dd..8c17f5f4e318 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
        u32                     rkey;
        u32                     length;
 
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
        /* Responder resources. It's a circular list where the oldest
         * resource is dropped first.
         */
@@ -232,6 +226,12 @@ struct rxe_resp_info {
        unsigned int            res_head;
        unsigned int            res_tail;
        struct resp_res         *res;
+
+       /* SRQ only. srq_wqe.dma.sge is a flex array */
+       struct {
+               struct rxe_recv_wqe     wqe;
+               struct ib_sge           sge[RXE_MAX_SGE];
+       } srq_wqe;
 };
 
 struct rxe_qp {
@@ -269,7 +269,6 @@ struct rxe_qp {
 
        struct rxe_req_info     req;
        struct rxe_comp_info    comp;
-       struct rxe_resp_info    resp;
 
        atomic_t                ssn;
        atomic_t                skb_out;
@@ -289,6 +288,7 @@ struct rxe_qp {
        spinlock_t              state_lock; /* guard requester and 

completer */

        struct execute_work     cleanup_work;

+ struct rxe_resp_info resp;
};

enum {

Yanjun.Zhu

Quoted 6 lines

Yanjun.Zhu

Quoted 2 lines

Jason

Signature · 2 lines
Best Regards,
Yanjun.Zhu
#15 lore raw
Quoted 18 lines · Zhu Yanjun
Quoted 15 lines · yanjun.zhu
Quoted 14 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines

       unsigned int        res_head; unsigned int        res_tail;
       struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has no member named wqe
   289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe;
       |                                         ^

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

I am not sure if this commit is good or not.

Hi, Jason && Leon

Any update? If this looks good to you, I will send out an official commit based on the following commit.

Thanks,
Yanjun.Zhu

Quoted 74 lines
drivers/infiniband/sw/rxe/rxe_verbs.c +2 -1
index 38d8c408320f..189eaa59a5fb 100644
@@ -1524,7 +1524,8 @@ static const struct ib_device_ops rxe_dev_ops = {
         INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah),
         INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq),
         INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd),
-       INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp),
+       /* For resp.srq_wqe.dma.sge */
+       INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp) + 
 RXE_MAX_SGE*sizeof(struct ib_sge),
         INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq),
         INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc),

        INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw),

drivers/infiniband/sw/rxe/rxe_verbs.h +6 -7
index fd48075810dd..8c17f5f4e318 100644
@@ -219,12 +219,6 @@ struct rxe_resp_info {
         u32                     rkey;
         u32                     length;
 
-       /* SRQ only */
-       struct {
-               struct rxe_recv_wqe     wqe;
-               struct ib_sge           sge[RXE_MAX_SGE];
-       } srq_wqe;
-
         /* Responder resources. It's a circular list where the oldest
          * resource is dropped first.
          */
@@ -232,6 +226,12 @@ struct rxe_resp_info {
         unsigned int            res_head;
         unsigned int            res_tail;
         struct resp_res         *res;
+
+       /* SRQ only. srq_wqe.dma.sge is a flex array */
+       struct {
+               struct rxe_recv_wqe     wqe;
+               struct ib_sge           sge[RXE_MAX_SGE];
+       } srq_wqe;
  };
 
  struct rxe_qp {
@@ -269,7 +269,6 @@ struct rxe_qp {
 
         struct rxe_req_info     req;
         struct rxe_comp_info    comp;
-       struct rxe_resp_info    resp;
 
         atomic_t                ssn;
         atomic_t                skb_out;
@@ -289,6 +288,7 @@ struct rxe_qp {
         spinlock_t              state_lock; /* guard requester and 

completer */

        struct execute_work     cleanup_work;
+       struct rxe_resp_info    resp;
 };

 enum {

Yanjun.Zhu

Quoted 6 lines

Yanjun.Zhu

Quoted 2 lines

Jason

Signature · 2 lines
Best Regards,
Yanjun.Zhu
#16 lore raw
Quoted 31 lines · Zhu Yanjun
Quoted 28 lines · Zhu Yanjun
Quoted 16 lines · yanjun.zhu
Quoted 15 lines · Jason Gunthorpe
Quoted 13 lines · Zhu Yanjun
Quoted 6 lines

       unsigned int        res_head; unsigned int        res_tail;
       struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has
no member named wqe
   289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe; |                                         ^

Quoted 4 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

Quoted 6 lines
Quoted 1 line

I am not sure if this commit is good or not.

Hi, Jason && Leon

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Thanks

#17 lore raw
Quoted 27 lines · Leon Romanovsky
Quoted 26 lines · Zhu Yanjun
Quoted 24 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines

       unsigned int        res_head; unsigned int        res_tail;
       struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has
no member named wqe
   289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe; |                                         ^

Quoted 3 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

Quoted 6 lines
Quoted 5 lines
Quoted 1 line

I am not sure if this commit is good or not.

Hi, Jason && Leon

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Will do. I will send it out very soon.

Yanjun.Zhu

Quoted 2 lines

Thanks

#18 lore raw
Quoted 29 lines · Yanjun.Zhu
Quoted 27 lines · Leon Romanovsky
Quoted 26 lines · Zhu Yanjun
Quoted 24 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines

        unsigned int        res_head; unsigned int        res_tail;
        struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has
no member named wqe
    289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe; |                                         ^

Quoted 3 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

Quoted 8 lines
Quoted 6 lines
Quoted 5 lines
Quoted 1 line

I am not sure if this commit is good or not.

Hi, Jason && Leon

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Will do. I will send it out very soon.

I don't see how this addresses the flex-array in the middle issue that originated this discussion.

-Gustavo

#19 lore raw
Quoted 35 lines · Gustavo A. R. Silva
Quoted 32 lines · Yanjun.Zhu
Quoted 30 lines · Leon Romanovsky
Quoted 29 lines · Zhu Yanjun
Quoted 17 lines · Zhu Yanjun
Quoted 15 lines · yanjun.zhu
Quoted 14 lines · Jason Gunthorpe
Quoted 6 lines · Zhu Yanjun
Quoted 6 lines

        unsigned int res_head;
        unsigned int        res_tail;
        struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

Quoted 6 lines

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has
no member named wqe
    289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe; |                                         ^

I didn't try to fix all the typos, you will need to do that.

Quoted 8 lines
Quoted 2 lines

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

I am not sure if this commit is good or not.

Hi, Jason && Leon

Quoted 8 lines
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Will do. I will send it out very soon.

I don't see how this addresses the flex-array in the middle issue that originated this discussion.

Could you please explain this in more detail?
Also, if you have a better approach, would you mind sending a new commit for discussion?

Thanks a lot.

Yanjun.Zhu

Quoted 2 lines

-Gustavo

Signature · 2 lines
Best Regards,
Yanjun.Zhu
#20 lore raw
Quoted 41 lines · Zhu Yanjun
Quoted 39 lines · Gustavo A. R. Silva
Quoted 36 lines · Yanjun.Zhu
Quoted 33 lines · Leon Romanovsky
Quoted 23 lines · Zhu Yanjun
Quoted 21 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines

        unsigned int res_head;
        unsigned int        res_tail;
        struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has
no member named wqe
    289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe; |                                         ^

Quoted 3 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

Quoted 8 lines
Quoted 4 lines

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

I am not sure if this commit is good or not.

Hi, Jason && Leon

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Quoted 7 lines
Quoted 4 lines
Quoted 1 line

Will do. I will send it out very soon.

I don't see how this addresses the flex-array in the middle issue that originated this discussion.

Could you please explain this in more detail?
Also, if you have a better approach, would you mind sending a new commit for discussion?

This is exactly what my original patch is about. I've explained this a couple of times in this thread already.

-Gustavo

#21 lore raw
Quoted 46 lines · Gustavo A. R. Silva
Quoted 43 lines · Zhu Yanjun
Quoted 41 lines · Gustavo A. R. Silva
Quoted 37 lines · Yanjun.Zhu
Quoted 27 lines · Leon Romanovsky
Quoted 26 lines · Zhu Yanjun
Quoted 24 lines · Zhu Yanjun
Quoted 14 lines · yanjun.zhu
Quoted 13 lines · Jason Gunthorpe
Quoted 12 lines · Zhu Yanjun
Quoted 6 lines

        unsigned int res_head;
        unsigned int        res_tail;
        struct resp_res        *res;
+
+    /* SRQ only. srq_wqe.dma.sge is a flex array */ +    struct rxe_recv_wqe srq_wqe;

drivers/infiniband/sw/rxe/rxe_resp.c: In function get_srq_wqe:
drivers/infiniband/sw/rxe/rxe_resp.c:289:41: error: struct rxe_recv_wqe has
no member named wqe
    289 |         qp->resp.wqe = &qp->resp.srq_wqe.wqe; |                                         ^

Quoted 3 lines
Quoted 1 line

I didn't try to fix all the typos, you will need to do that.

Exactly. I will fix this problem. This weekend, I will send out an official commit.

Hi, Jason

The followings are based on your commits and Leon's commits. And it can pass the rdma-core tests.

Quoted 6 lines
Quoted 5 lines
Quoted 1 line

I am not sure if this commit is good or not.

Hi, Jason && Leon

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Will do. I will send it out very soon.

Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

I don't see how this addresses the flex-array in the middle issue that originated this discussion.

Could you please explain this in more detail?
Also, if you have a better approach, would you mind sending a new commit for discussion?

This is exactly what my original patch is about. I've explained this a couple of times in this thread already.

In your original patch, I have the following problem. I read all your replies. But I can not find your answers to my problem.

My problem is as below:

1. The followings are the declaration of __TRAILING_OVERLAP

/**
* __TRAILING_OVERLAP() - Overlap a flexible-array member with trailing
* members.
*
* Creates a union between a flexible-array member (FAM) in a struct
and a set
* of additional members that would otherwise follow it.
*
* @TYPE: Flexible structure type name, including "struct" keyword.
* @NAME: Name for a variable to define.
* @FAM: The flexible-array member within @TYPE <----- Here
* @ATTRS: Any struct attributes (usually empty)
* @MEMBERS: Trailing overlapping members.
*/

#define __TRAILING_OVERLAP(TYPE, NAME, FAM, ATTRS, MEMBERS)         \
    union {                                 \
        TYPE NAME;                          \
        struct {                            \
            unsigned char __offset_to_FAM[offsetof(TYPE, FAM)]; \
            MEMBERS                         \
        } ATTRS;                            \
    }

2. I expanded the above MACRO __TRAILING_OVERLAP into the following following the definition in your commit.

union {

        struct rxe_recv_wqe wqe;

        struct {
            unsigned char __offset_to_FAM[offsetof(struct rxe_recv_wqe, 
dma.sge)]; \
            struct ib_sge sge[RXE_MAX_SGE];
        } ;

    }srq_wqe;

3. In your original commit, dma.sge should be the member of struct rxe_recv_wqe. The struct rxe_recv_wqe is as below.

"@FAM: The flexible-array member within @TYPE"

The struct rxe_recv_wqe is as below.

struct rxe_recv_wqe {
    __aligned_u64       wr_id;
    __u32           reserved;
    __u32           padding;
    struct rxe_dma_info dma;
};

But I can not find dma.sge in the above struct. Can you explain it?

To be honest, I read your original commit for several times, but I can not get it. Can you explain the MACRO TRAILING_OVERLAP? And how can it replace the following struct?

struct {
      struct rxe_recv_wqe    wqe;
      struct ib_sge        sge[RXE_MAX_SGE];
} srq_wqe;

Thanks a lot.

Yanjun.Zhu

Quoted 2 lines

-Gustavo

#22 lore raw
Quoted 8 lines

The struct rxe_recv_wqe is as below.

struct rxe_recv_wqe {
    __aligned_u64       wr_id;
    __u32           reserved;
    __u32           padding;
    struct rxe_dma_info dma;

Expand struct rxe_dma_info here.

Quoted 5 lines

};

But I can not find dma.sge in the above struct. Can you explain it?

To be honest, I read your original commit for several times, but I can not get it.  Can you explain the MACRO TRAILING_OVERLAP? And how can it replace the following struct?

This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.

-Gustavo

#23 lore raw
Quoted 10 lines · Gustavo A. R. Silva
Quoted 8 lines

The struct rxe_recv_wqe is as below.

struct rxe_recv_wqe {
     __aligned_u64       wr_id;
     __u32           reserved;
     __u32           padding;
     struct rxe_dma_info dma;

Expand struct rxe_dma_info here.

Thanks. In struct rxe_dma_info, the struct is

struct rxe_sge {
       __aligned_u64 addr;
       __u32   length;
       __u32   lkey;
};

But in your commit, struct ib_sge is used.

struct ib_sge {
    u64 addr;
    u32 length;
    u32 lkey;
};
__aligned_u64 is a 64-bit integer with a guaranteed 8-byte alignment,

used to preserve ABI correctness across architectures and between

userspace and kernel, while u64 has architecture-dependent alignment.

I am not sure if we can treate "struct rxe_sge" as the same with "struct ib_sge".

Leon and Jason, please comment on it.

Yanjun.Zhu

Quoted 12 lines
Quoted 6 lines

};

But I can not find dma.sge in the above struct. Can you explain it?

To be honest, I read your original commit for several times, but I can not get it.  Can you explain the MACRO TRAILING_OVERLAP? And how can it replace the following struct?

This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.

-Gustavo

#24 lore raw
Quoted 30 lines · Zhu Yanjun
Quoted 10 lines · Gustavo A. R. Silva
Quoted 8 lines

The struct rxe_recv_wqe is as below.

struct rxe_recv_wqe {
     __aligned_u64       wr_id;
     __u32           reserved;
     __u32           padding;
     struct rxe_dma_info dma;

Expand struct rxe_dma_info here.

Thanks. In struct rxe_dma_info, the struct is

struct rxe_sge {
       __aligned_u64 addr;
       __u32   length;
       __u32   lkey;
};

But in your commit, struct ib_sge is used.

struct ib_sge {
    u64 addr;
    u32 length;
    u32 lkey;
};
__aligned_u64 is a 64-bit integer with a guaranteed 8-byte alignment,

Quoted 5 lines

used to preserve ABI correctness across architectures and between

userspace and kernel, while u64 has architecture-dependent alignment.

I am not sure if we can treate "struct rxe_sge" as the same with "struct ib_sge".

Just notice that the original code is the one actually doing that. See my response in this same thread:

https://lore.kernel.org/linux-hardening/[email protected]/

So, if that code is fine, this is fine. If the original code is wrong, then that code should be fixed first.

-Gustavo

Quoted 20 lines

Leon and Jason, please comment on it.

Yanjun.Zhu

Quoted 12 lines
Quoted 6 lines

};

But I can not find dma.sge in the above struct. Can you explain it?

To be honest, I read your original commit for several times, but I can not get it.  Can you explain the MACRO TRAILING_OVERLAP? And how can it replace the following struct?

This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.

-Gustavo

maintainer
#26 lore raw
In reply to Gustavo A. R. Silva
Quoted 8 lines · Gustavo A. R. Silva
Quoted 5 lines
Quoted 3 lines
Quoted 2 lines

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Will do. I will send it out very soon.

I don't see how this addresses the flex-array in the middle issue that originated this discussion.

??

It moved it to the end of every struct it is part of?

Jason

#25 lore raw
In reply to Gustavo A. R. Silva
Quoted 39 lines · Gustavo A. R. Silva
Quoted 30 lines · Zhu Yanjun
Quoted 10 lines · Gustavo A. R. Silva
Quoted 8 lines

The struct rxe_recv_wqe is as below.

struct rxe_recv_wqe {
     __aligned_u64       wr_id;
     __u32           reserved;
     __u32           padding;
     struct rxe_dma_info dma;

Expand struct rxe_dma_info here.

Thanks. In struct rxe_dma_info, the struct is

struct rxe_sge {
        __aligned_u64 addr;
        __u32   length;
        __u32   lkey;
};

But in your commit, struct ib_sge is used.

struct ib_sge {
     u64 addr;
     u32 length;
     u32 lkey;
};
__aligned_u64 is a 64-bit integer with a guaranteed 8-byte alignment,

Quoted 5 lines

used to preserve ABI correctness across architectures and between

userspace and kernel, while u64 has architecture-dependent alignment.

I am not sure if we can treate "struct rxe_sge" as the same with "struct ib_sge".

Quoted 7 lines

Just notice that the original code is the one actually doing that. See my response in this same thread:

https://lore.kernel.org/linux-hardening/d3336e9d-2b84-4698-a799- [email protected]/

So, if that code is fine, this is fine. If the original code is wrong, then that code should be fixed first.

Thanks a lot. Because struct ib_sge and struct ib_sge is different,

struct ib_sge {
    u64 addr; <--- u64 has architecture-dependent alignment
    u32 length;
    u32 lkey;
};

struct rxe_sge {
__aligned_u64 addr; <---guaranteed 8-byte alignment,

used to preserve ABI correctness across architectures and between

userspace and kernel

       __u32   length;
       __u32   lkey;
};

and struct rxe_sge is used in rxe_mr, it is working between userspace and kernel, thus, I want to keep struct rxe_mr in rxe_mr.

But in other places, I want to replace struct rxe_sge with struct ib_sge. The commit is as below.

In short, the commit "RDMA/rxe: Avoid -Wflex-array-member-not-at-end warnings" && the following commit will work well. I have made tests in my local host. It can work well.

Please Leon and Jason comment.

drivers/infiniband/sw/rxe/rxe_mr.c +2 -2
index b1df05238848..390ae01f549d 100644
@@ -341,7 +341,7 @@ int copy_data(
        enum rxe_mr_copy_dir    dir)
 {
        int                     bytes;
-       struct rxe_sge          *sge    = &dma->sge[dma->cur_sge];
+       struct ib_sge *sge      = &dma->sge[dma->cur_sge];
        int                     offset  = dma->sge_offset;
        int                     resid   = dma->resid;
        struct rxe_mr           *mr     = NULL;
@@ -580,7 +580,7 @@ enum resp_states rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
 
 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
 {
-       struct rxe_sge          *sge    = &dma->sge[dma->cur_sge];
+       struct ib_sge *sge      = &dma->sge[dma->cur_sge];
        int                     offset  = dma->sge_offset;
        int                     resid   = dma->resid;
 
drivers/infiniband/sw/rxe/rxe_resp.c +1 -1
index 711f73e0bbb1..74f5b695da7a 100644
@@ -283,7 +283,7 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
                rxe_dbg_qp(qp, "invalid num_sge in SRQ entry\n");
                return RESPST_ERR_MALFORMED_WQE;
        }
-       size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct rxe_sge);
+       size = sizeof(*wqe) + wqe->dma.num_sge*sizeof(struct ib_sge);
        memcpy(&qp->resp.srq_wqe, wqe, size);
 
        qp->resp.wqe = &qp->resp.srq_wqe.wqe;
include/uapi/rdma/rdma_user_rxe.h +1 -1
index bb092fccb813..360839498441 100644
@@ -154,7 +154,7 @@ struct rxe_dma_info {
        union {
                __DECLARE_FLEX_ARRAY(__u8, inline_data);
                __DECLARE_FLEX_ARRAY(__u8, atomic_wr);
-               __DECLARE_FLEX_ARRAY(struct rxe_sge, sge);
+               __DECLARE_FLEX_ARRAY(struct ib_sge, sge);
        };
 };
 

To this commit, plus the above commit, it should work well.

Yanjun.Zhu

Quoted 24 lines

-Gustavo

Quoted 20 lines

Leon and Jason, please comment on it.

Yanjun.Zhu

Quoted 12 lines
Quoted 6 lines

};

But I can not find dma.sge in the above struct. Can you explain it?

To be honest, I read your original commit for several times, but I can not get it.  Can you explain the MACRO TRAILING_OVERLAP? And how can it replace the following struct?

This is clearly explained in the changelog text. I think what you're missing will be clear once you understand how nested structures work. See my comment above.

-Gustavo

#27 lore raw
In reply to Jason Gunthorpe
Quoted 6 lines · Jason Gunthorpe
Quoted 4 lines · Gustavo A. R. Silva
Quoted 4 lines
Quoted 3 lines
Quoted 2 lines

Any update? If this looks good to you, I will send out an official commit based on the following commit.

You are RXE maintainer, send the official patch.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Will do. I will send it out very soon.

I don't see how this addresses the flex-array in the middle issue that originated this discussion.

??

It moved it to the end of every struct it is part of?

Any update?

Yanjun.Zhu

Quoted 2 lines

Jason