On-Demand Paging on SoftRoCE
Subsystems (2)
PATCH v4 1/8 RDMA/rxe: Tentative workqueue implementation
In reply to Daisuke MatsudaThis is a mock patch and not intended to be merged. In order to implement ODP on rxe, it is necessary to convert triple tasklets to workqueue. I expect Bob Pearson will submit his patch to do this very soon.
--- drivers/infiniband/sw/rxe/rxe.c | 9 ++- drivers/infiniband/sw/rxe/rxe_task.c | 84 ++++++++++++++++++---------- drivers/infiniband/sw/rxe/rxe_task.h | 6 +- 3 files changed, 67 insertions(+), 32 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +8 -1
@@ -212,10 +212,16 @@ static int __init rxe_module_init(void) {int err; - err =rxe_net_init (); + err =rxe_alloc_wq ();if (err)return err; + err =rxe_net_init (); +if (err) { +rxe_destroy_wq (); +return err; + } +rdma_link_register (&rxe_link_ops);pr_info ("loaded\n" );return 0 ;
@@ -226,6 +232,7 @@ static void __exit rxe_module_exit(void)rdma_link_unregister (&rxe_link_ops);ib_unregister_driver (RDMA_DRIVER_RXE );rxe_net_exit (); +rxe_destroy_wq ();pr_info ("unloaded\n" ); }
drivers/infiniband/sw/rxe/rxe_task.c +54 -30
200 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_task.h +5 -1
@@ -22,7 +22,7 @@ enum {* called again. */ struct rxe_task { -struct tasklet_struct tasklet; +struct work_struct work;int state;spinlock_t lock;struct rxe_qp *qp;
@@ -32,6 +32,10 @@ struct rxe_task {long num_done; }; +int rxe_alloc_wq (void ); + +void rxe_destroy_wq (void ); +/* * init rxe_task structure * qp => parameter to pass to func
Signature · 1 line
2.39.1
PATCH v4 2/8 RDMA/rxe: Always schedule works before accessing user MRs
In reply to Daisuke MatsudaBoth responder and completer can sleep to execute page-fault when used with ODP. It happens when they are going to access user MRs, so works must be scheduled in such cases.
--- drivers/infiniband/sw/rxe/rxe_comp.c | 12 ++++++++++-- drivers/infiniband/sw/rxe/rxe_loc.h | 4 ++-- drivers/infiniband/sw/rxe/rxe_recv.c | 4 ++-- drivers/infiniband/sw/rxe/rxe_resp.c | 14 +++++++++----- 4 files changed, 23 insertions(+), 11 deletions(-)
drivers/infiniband/sw/rxe/rxe_comp.c +10 -2
@@ -126,13 +126,21 @@ void retransmit_timer(struct timer_list *t)spin_unlock_bh (&qp->state_lock); } -void rxe_comp_queue_pkt (struct rxe_qp *qp,struct sk_buff *skb) +void rxe_comp_queue_pkt (struct rxe_pkt_info *pkt,struct sk_buff *skb) { +struct rxe_qp *qp = pkt->qp;int must_sched;skb_queue_tail (&qp->resp_pkts, skb); - must_sched =skb_queue_len (&qp->resp_pkts) >1 ; +/* Schedule the task if processing Read responses or Atomic acks. +* In these cases, completer may sleep to access ODP-enabled MRs. +*/ +if (pkt->mask | (RXE_PAYLOAD_MASK ||RXE_ATMACK_MASK )) + must_sched =1 ; +else + must_sched =skb_queue_len (&qp->resp_pkts) >1 ; +if (must_sched !=0 )rxe_counter_inc (SKB_TO_PKT (skb)->rxe,RXE_CNT_COMPLETER_SCHED );
drivers/infiniband/sw/rxe/rxe_loc.h +2 -2
@@ -179,9 +179,9 @@ int rxe_icrc_init(struct rxe_dev *rxe);int rxe_icrc_check (struct sk_buff *skb,struct rxe_pkt_info *pkt);void rxe_icrc_generate (struct sk_buff *skb,struct rxe_pkt_info *pkt); -void rxe_resp_queue_pkt (struct rxe_qp *qp,struct sk_buff *skb); +void rxe_resp_queue_pkt (struct rxe_pkt_info *pkt,struct sk_buff *skb); -void rxe_comp_queue_pkt (struct rxe_qp *qp,struct sk_buff *skb); +void rxe_comp_queue_pkt (struct rxe_pkt_info *pkt,struct sk_buff *skb);static inline unsigned int wr_opcode_mask (int opcode,struct rxe_qp *qp) {
drivers/infiniband/sw/rxe/rxe_recv.c +2 -2
@@ -181,9 +181,9 @@ static int hdr_check(struct rxe_pkt_info *pkt)static inline void rxe_rcv_pkt (struct rxe_pkt_info *pkt,struct sk_buff *skb) {if (pkt->mask &RXE_REQ_MASK ) -rxe_resp_queue_pkt (pkt->qp, skb); +rxe_resp_queue_pkt (pkt, skb);else -rxe_comp_queue_pkt (pkt->qp, skb); +rxe_comp_queue_pkt (pkt, skb); }static void rxe_rcv_mcast_pkt (struct rxe_dev *rxe,struct sk_buff *skb)
drivers/infiniband/sw/rxe/rxe_resp.c +9 -5
@@ -47,15 +47,19 @@ static char *resp_state_name[] = { };/* rxe_recv calls here to add a request packet to the input queue */ -void rxe_resp_queue_pkt (struct rxe_qp *qp,struct sk_buff *skb) +void rxe_resp_queue_pkt (struct rxe_pkt_info *pkt,struct sk_buff *skb) { -int must_sched; -struct rxe_pkt_info *pkt =SKB_TO_PKT (skb); +int must_sched =1 ; +struct rxe_qp *qp = pkt->qp;skb_queue_tail (&qp->req_pkts, skb); - must_sched = (pkt->opcode ==IB_OPCODE_RC_RDMA_READ_REQUEST ) || - (skb_queue_len (&qp->req_pkts) >1 ); +/* responder can sleep to access an ODP-enabled MR. Always schedule +* tasks for non-zero-byte operations, RDMA Read, and Atomic. +*/ +if ((skb_queue_len (&qp->req_pkts) ==1 ) && (payload_size (pkt) ==0 ) + && !(pkt->mask &RXE_READ_OR_ATOMIC_MASK )) + must_sched =0 ;if (must_sched)rxe_sched_task (&qp->resp.task);
Signature · 1 line
2.39.1
PATCH v4 4/8 RDMA/rxe: Move resp_states definition to rxe_verbs.h
In reply to Daisuke MatsudaTo use the resp_states values in rxe_loc.h, it is necessary to move the definition to rxe_verbs.h, where other internal states of this driver are defined.
--- drivers/infiniband/sw/rxe/rxe.h | 37 --------------------------- drivers/infiniband/sw/rxe/rxe_verbs.h | 37 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 37 deletions(-)
drivers/infiniband/sw/rxe/rxe.h +0 -37
@@ -100,43 +100,6 @@#define rxe_info_mw (mw, fmt, ...) ibdev_info_ratelimited((mw)->ibmw.device, \ "mw#%d %s: " fmt, (mw)->elem.index, __func__, ##__VA_ARGS__) -/* responder states */ -enum resp_states { -RESPST_NONE , -RESPST_GET_REQ , -RESPST_CHK_PSN , -RESPST_CHK_OP_SEQ , -RESPST_CHK_OP_VALID , -RESPST_CHK_RESOURCE , -RESPST_CHK_LENGTH , -RESPST_CHK_RKEY , -RESPST_EXECUTE , -RESPST_READ_REPLY , -RESPST_ATOMIC_REPLY , -RESPST_ATOMIC_WRITE_REPLY , -RESPST_PROCESS_FLUSH , -RESPST_COMPLETE , -RESPST_ACKNOWLEDGE , -RESPST_CLEANUP , -RESPST_DUPLICATE_REQUEST , -RESPST_ERR_MALFORMED_WQE , -RESPST_ERR_UNSUPPORTED_OPCODE , -RESPST_ERR_MISALIGNED_ATOMIC , -RESPST_ERR_PSN_OUT_OF_SEQ , -RESPST_ERR_MISSING_OPCODE_FIRST , -RESPST_ERR_MISSING_OPCODE_LAST_C , -RESPST_ERR_MISSING_OPCODE_LAST_D1E , -RESPST_ERR_TOO_MANY_RDMA_ATM_REQ , -RESPST_ERR_RNR , -RESPST_ERR_RKEY_VIOLATION , -RESPST_ERR_INVALIDATE_RKEY , -RESPST_ERR_LENGTH , -RESPST_ERR_CQ_OVERFLOW , -RESPST_ERROR , -RESPST_DONE , -RESPST_EXIT , -}; -void rxe_set_mtu (struct rxe_dev *rxe,unsigned int dev_mtu);int rxe_add (struct rxe_dev *rxe,unsigned int mtu,const char *ibdev_name);
drivers/infiniband/sw/rxe/rxe_verbs.h +37 -0
@@ -127,6 +127,43 @@ struct rxe_comp_info {struct rxe_task task; }; +/* responder states */ +enum resp_states { +RESPST_NONE , +RESPST_GET_REQ , +RESPST_CHK_PSN , +RESPST_CHK_OP_SEQ , +RESPST_CHK_OP_VALID , +RESPST_CHK_RESOURCE , +RESPST_CHK_LENGTH , +RESPST_CHK_RKEY , +RESPST_EXECUTE , +RESPST_READ_REPLY , +RESPST_ATOMIC_REPLY , +RESPST_ATOMIC_WRITE_REPLY , +RESPST_PROCESS_FLUSH , +RESPST_COMPLETE , +RESPST_ACKNOWLEDGE , +RESPST_CLEANUP , +RESPST_DUPLICATE_REQUEST , +RESPST_ERR_MALFORMED_WQE , +RESPST_ERR_UNSUPPORTED_OPCODE , +RESPST_ERR_MISALIGNED_ATOMIC , +RESPST_ERR_PSN_OUT_OF_SEQ , +RESPST_ERR_MISSING_OPCODE_FIRST , +RESPST_ERR_MISSING_OPCODE_LAST_C , +RESPST_ERR_MISSING_OPCODE_LAST_D1E , +RESPST_ERR_TOO_MANY_RDMA_ATM_REQ , +RESPST_ERR_RNR , +RESPST_ERR_RKEY_VIOLATION , +RESPST_ERR_INVALIDATE_RKEY , +RESPST_ERR_LENGTH , +RESPST_ERR_CQ_OVERFLOW , +RESPST_ERROR , +RESPST_DONE , +RESPST_EXIT , +}; +enum rdatm_res_state { rdatm_res_state_next, rdatm_res_state_new,
Signature · 1 line
2.39.1
PATCH v4 7/8 RDMA/rxe: Add support for Send/Recv/Write/Read with ODP
In reply to Daisuke Matsudarxe_mr_copy() is used widely to copy data to/from a user MR. requester uses it to load payloads of requesting packets; responder uses it to process Send, Write, and Read operaetions; completer uses it to copy data from response packets of Read and Atomic operations to a user MR.
Allow these operations to be used with ODP by adding a subordinate function rxe_odp_mr_copy(). It is comprised of the following steps:
1. Check the driver page table(umem_odp->dma_list) to see if pages being accessed are present with appropriate permission.
2. If necessary, trigger page fault to map the pages.
3. Update the MR xarray using PFNs in umem_odp->pfn_list.
4. Execute data copy to/from the pages.
umem_mutex is used to ensure that dma_list (an array of addresses of an MR) is not changed while it is being checked and that mapped pages are not invalidated before data copy completes.
--- drivers/infiniband/sw/rxe/rxe.c | 10 +++ drivers/infiniband/sw/rxe/rxe_loc.h | 8 ++ drivers/infiniband/sw/rxe/rxe_mr.c | 2 +- drivers/infiniband/sw/rxe/rxe_odp.c | 109 ++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe.c +10 -0
@@ -79,6 +79,16 @@ static void rxe_init_device_param(struct rxe_dev *rxe)/* IB_ODP_SUPPORT_IMPLICIT is not supported right now. */ rxe->attr.odp_caps.general_caps |=IB_ODP_SUPPORT ; + + rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |=IB_ODP_SUPPORT_SEND ; + rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |=IB_ODP_SUPPORT_RECV ; + rxe->attr.odp_caps.per_transport_caps.ud_odp_caps |=IB_ODP_SUPPORT_SRQ_RECV ; + + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_SEND ; + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_RECV ; + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_WRITE ; + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_READ ; + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_SRQ_RECV ; } }
drivers/infiniband/sw/rxe/rxe_loc.h +8 -0
@@ -206,6 +206,8 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp)#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING int rxe_odp_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova,int access_flags,struct rxe_mr *mr); +int rxe_odp_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length, +enum rxe_mr_copy_dir dir);#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -213,6 +215,12 @@ rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, {return -EOPNOTSUPP ; } +static inline int +rxe_odp_mr_copy (struct rxe_mr *mr,u64 iova,void *addr, +int length,enum rxe_mr_copy_dir dir) +{ +return -EOPNOTSUPP ; +}#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
drivers/infiniband/sw/rxe/rxe_mr.c +1 -1
@@ -319,7 +319,7 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, }if (mr->odp_enabled) -return -EOPNOTSUPP ; +return rxe_odp_mr_copy (mr, iova, addr, length, dir);else return rxe_mr_copy_xarray (mr, iova, addr, length, dir); }
drivers/infiniband/sw/rxe/rxe_odp.c +109 -0
112 lines not on this page · raw message
Signature · 1 line
2.39.1
PATCH v4 5/8 RDMA/rxe: Add page invalidation support
In reply to Daisuke MatsudaOn page invalidation, an MMU notifier callback is invoked to unmap DMA addresses and update the driver page table(umem_odp->dma_list). It also set the corresponding entries in MR xarray to NULL to prevent any access. The callback is registered when an ODP-enabled MR is created.
--- drivers/infiniband/sw/rxe/Makefile | 2 ++ drivers/infiniband/sw/rxe/rxe_odp.c | 56 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c
drivers/infiniband/sw/rxe/Makefile +2 -0
@@ -23,3 +23,5 @@ rdma_rxe-y := \ rxe_task.o \ rxe_net.o \ rxe_hw_counters.o + +rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
drivers/infiniband/sw/rxe/rxe_odp.c new file +56 -0
56 lines not on this page · raw message
Signature · 1 line
2.39.1
PATCH v4 6/8 RDMA/rxe: Allow registering MRs for On-Demand Paging
In reply to Daisuke MatsudaAllow applications to register an ODP-enabled MR, in which case the flag IB_ACCESS_ON_DEMAND is passed to rxe_reg_user_mr(). However, there is no RDMA operation supported right now. They will be enabled later in the subsequent two patches.
rxe_odp_do_pagefault() is called to initialize an ODP-enabled MR. It syncs process address space from the CPU page table to the driver page table (dma_list/pfn_list in umem_odp) when called with RXE_PAGEFAULT_SNAPSHOT flag. Additionally, It can be used to trigger page fault when pages being accessed are not present or do not have proper read/write permissions, and possibly to prefetch pages in the future.
--- drivers/infiniband/sw/rxe/rxe.c | 7 ++ drivers/infiniband/sw/rxe/rxe_loc.h | 14 +++ drivers/infiniband/sw/rxe/rxe_mr.c | 9 +- drivers/infiniband/sw/rxe/rxe_odp.c | 120 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_resp.c | 15 +++- drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 2 + 7 files changed, 166 insertions(+), 6 deletions(-)
drivers/infiniband/sw/rxe/rxe.c +7 -0
@@ -73,6 +73,13 @@ static void rxe_init_device_param(struct rxe_dev *rxe) rxe->ndev->dev_addr); rxe->max_ucontext =RXE_MAX_UCONTEXT ; + +if (IS_ENABLED (CONFIG_INFINIBAND_ON_DEMAND_PAGING )) { + rxe->attr.kernel_cap_flags |=IBK_ON_DEMAND_PAGING ; + +/* IB_ODP_SUPPORT_IMPLICIT is not supported right now. */ + rxe->attr.odp_caps.general_caps |=IB_ODP_SUPPORT ; + } }/* initialize port attributes */
drivers/infiniband/sw/rxe/rxe_loc.h +14 -0
@@ -202,4 +202,18 @@ static inline unsigned int wr_opcode_mask(int opcode, struct rxe_qp *qp) return rxe_wr_opcode_info[opcode].mask[qp->ibqp.qp_type]; } +/* rxe_odp.c */ +#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING +int rxe_odp_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length, +u64 iova,int access_flags,struct rxe_mr *mr); +#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +static inline int +rxe_odp_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova, +int access_flags,struct rxe_mr *mr) +{ +return -EOPNOTSUPP ; +} + +#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ +#endif /* RXE_LOC_H */
drivers/infiniband/sw/rxe/rxe_mr.c +8 -1
@@ -318,7 +318,10 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,return err; } -return rxe_mr_copy_xarray (mr, iova, addr, length, dir); +if (mr->odp_enabled) +return -EOPNOTSUPP ; +else +return rxe_mr_copy_xarray (mr, iova, addr, length, dir); }/* copy data in or out of a wqe, i.e. sg list
@@ -527,6 +530,10 @@ int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)struct page *page;u64 *va; +/* ODP is not supported right now. WIP. */ +if (mr->odp_enabled) +return RESPST_ERR_UNSUPPORTED_OPCODE ; +/* See IBA oA19-28 */ if (unlikely (mr->state !=RXE_MR_STATE_VALID )) {rxe_dbg_mr (mr,"mr not in valid state" );
drivers/infiniband/sw/rxe/rxe_odp.c +120 -0
129 lines not on this page · raw message
drivers/infiniband/sw/rxe/rxe_resp.c +11 -4
@@ -641,6 +641,10 @@ static enum resp_states process_flush(struct rxe_qp *qp,struct rxe_mr *mr = qp->resp.mr;struct resp_res *res = qp->resp.res; +/* ODP is not supported right now. WIP. */ +if (mr->odp_enabled) +return RESPST_ERR_UNSUPPORTED_OPCODE ; +/* oA19-14, oA19-15 */ if (res && res->replay)return RESPST_ACKNOWLEDGE ;
@@ -694,10 +698,13 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,if (!res->replay) {u64 iova = qp->resp.va + qp->resp.offset; - err =rxe_mr_do_atomic_op (mr, iova, pkt->opcode, -atmeth_comp (pkt), -atmeth_swap_add (pkt), - &res->atomic.orig_val); +if (mr->odp_enabled) + err =RESPST_ERR_UNSUPPORTED_OPCODE ; +else + err =rxe_mr_do_atomic_op (mr, iova, pkt->opcode, +atmeth_comp (pkt), +atmeth_swap_add (pkt), + &res->atomic.orig_val);if (err)return err;
drivers/infiniband/sw/rxe/rxe_verbs.c +4 -1
@@ -1274,7 +1274,10 @@ static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start, mr->ibmr.pd = ibpd; mr->ibmr.device = ibpd->device; - err =rxe_mr_init_user (rxe, start, length, iova, access, mr); +if (access &IB_ACCESS_ON_DEMAND ) + err =rxe_odp_mr_init_user (rxe, start, length, iova, access, mr); +else + err =rxe_mr_init_user (rxe, start, length, iova, access, mr);if (err) {rxe_dbg_mr (mr,"reg_user_mr failed, err = %d" , err); goto err_cleanup;
drivers/infiniband/sw/rxe/rxe_verbs.h +2 -0
@@ -333,6 +333,8 @@ struct rxe_mr {u32 nbuf;struct xarray page_list; + +bool odp_enabled; };static inline unsigned int mr_page_size (struct rxe_mr *mr)
Signature · 1 line
2.39.1
PATCH v4 8/8 RDMA/rxe: Add support for the traditional Atomic operations with ODP
In reply to Daisuke MatsudaEnable 'fetch and add' and 'compare and swap' operations to manipulate data in an ODP-enabled MR. This is comprised of the following steps:
1. Check the driver page table(umem_odp->dma_list) to see if the target page is both readable and writable.
2. If not, then trigger page fault to map the page.
3. Update the entry in the MR xarray.
4. Execute the operation.
umem_mutex is used to ensure that dma_list (an array of addresses of an MR) is not changed while it is being checked and that the target page is not invalidated before data access completes.
--- drivers/infiniband/sw/rxe/rxe.c | 1 + drivers/infiniband/sw/rxe/rxe_loc.h | 9 +++++++++ drivers/infiniband/sw/rxe/rxe_odp.c | 26 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_resp.c | 5 ++++- 4 files changed, 40 insertions(+), 1 deletion(-)
drivers/infiniband/sw/rxe/rxe.c +1 -0
@@ -88,6 +88,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe) rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_RECV ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_WRITE ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_READ ; + rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_ATOMIC ; rxe->attr.odp_caps.per_transport_caps.rc_odp_caps |=IB_ODP_SUPPORT_SRQ_RECV ; } }
drivers/infiniband/sw/rxe/rxe_loc.h +9 -0
@@ -208,6 +208,9 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, int access_flags, struct rxe_mr *mr);int rxe_odp_mr_copy (struct rxe_mr *mr,u64 iova,void *addr,int length,enum rxe_mr_copy_dir dir); +int rxe_odp_mr_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val); +#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */ static inline int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
@@ -221,6 +224,12 @@ rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, {return -EOPNOTSUPP ; } +static inline int +rxe_odp_mr_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val) +{ +return RESPST_ERR_UNSUPPORTED_OPCODE ; +}#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
drivers/infiniband/sw/rxe/rxe_odp.c +26 -0
@@ -283,3 +283,29 @@ int rxe_odp_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,return err; } + +int rxe_odp_mr_atomic_op (struct rxe_mr *mr,u64 iova,int opcode, +u64 compare,u64 swap_add,u64 *orig_val) +{ +int err; +struct ib_umem_odp *umem_odp =to_ib_umem_odp (mr->umem); + +/* If pagefault is not required, umem mutex will be held until the +* atomic operation completes. Otherwise, it is released and locked +* again in rxe_odp_map_range() to let invalidation handler do its +* work meanwhile. +*/ +mutex_lock (&umem_odp->umem_mutex); + +/* Atomic operations manipulate a single char. */ + err =rxe_odp_map_range (mr, iova,sizeof (char ),0 ); +if (err) +return err; + + err =rxe_mr_do_atomic_op (mr, iova, opcode, compare, + swap_add, orig_val); + +mutex_unlock (&umem_odp->umem_mutex); + +return err; +}
drivers/infiniband/sw/rxe/rxe_resp.c +4 -1
@@ -699,7 +699,10 @@ static enum resp_states atomic_reply(struct rxe_qp *qp,u64 iova = qp->resp.va + qp->resp.offset;if (mr->odp_enabled) - err =RESPST_ERR_UNSUPPORTED_OPCODE ; + err =rxe_odp_mr_atomic_op (mr, iova, pkt->opcode, +atmeth_comp (pkt), +atmeth_swap_add (pkt), + &res->atomic.orig_val);else err = rxe_mr_do_atomic_op(mr, iova, pkt->opcode, atmeth_comp(pkt),
Signature · 1 line
2.39.1
PATCH v4 3/8 RDMA/rxe: Make MR functions accessible from other rxe source code
In reply to Daisuke MatsudaSome functions in rxe_mr.c are going to be used in rxe_odp.c, which is to be created in the subsequent patch. List the declarations of the functions in rxe_loc.h.
--- drivers/infiniband/sw/rxe/rxe_loc.h | 14 ++++++++++++++ drivers/infiniband/sw/rxe/rxe_mr.c | 18 ++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-)
drivers/infiniband/sw/rxe/rxe_loc.h +14 -0
@@ -60,7 +60,9 @@ int rxe_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);/* rxe_mr.c */ u8 rxe_get_next_key (u32 last_key); +void rxe_mr_init (int access,struct rxe_mr *mr);void rxe_mr_init_dma (int access,struct rxe_mr *mr); +int rxe_mr_fill_pages_from_sgt (struct rxe_mr *mr,struct sg_table *sgt);int rxe_mr_init_user (struct rxe_dev *rxe,u64 start,u64 length,u64 iova,int access,struct rxe_mr *mr);int rxe_mr_init_fast (int max_pages,struct rxe_mr *mr);
@@ -71,6 +73,8 @@ int copy_data(struct rxe_pd *pd, int access, struct rxe_dma_info *dma, void *addr, int length, enum rxe_mr_copy_dir dir);int rxe_map_mr_sg (struct ib_mr *ibmr,struct scatterlist *sg,int sg_nents,unsigned int *sg_offset); +int rxe_mr_copy_xarray (struct rxe_mr *mr,u64 iova,void *addr, +unsigned int length,enum rxe_mr_copy_dir dir);int rxe_mr_do_atomic_op (struct rxe_mr *mr,u64 iova,int opcode,u64 compare,u64 swap_add,u64 *orig_val);int rxe_mr_do_atomic_write (struct rxe_mr *mr,u64 iova,u64 value);
@@ -82,6 +86,16 @@ int rxe_invalidate_mr(struct rxe_qp *qp, u32 key);int rxe_reg_fast_mr (struct rxe_qp *qp,struct rxe_send_wqe *wqe);void rxe_mr_cleanup (struct rxe_pool_elem *elem); +static inline unsigned long rxe_mr_iova_to_index (struct rxe_mr *mr,u64 iova) +{ +return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); +} + +static inline unsigned long rxe_mr_iova_to_page_offset (struct rxe_mr *mr,u64 iova) +{ +return iova & (mr_page_size (mr) -1 ); +} +/* rxe_mw.c */ int rxe_alloc_mw (struct ib_mw *ibmw,struct ib_udata *udata);int rxe_dealloc_mw (struct ib_mw *ibmw);
drivers/infiniband/sw/rxe/rxe_mr.c +4 -14
@@ -49,7 +49,7 @@ int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) | IB_ACCESS_REMOTE_WRITE \ | IB_ACCESS_REMOTE_ATOMIC) -static void rxe_mr_init (int access,struct rxe_mr *mr) +void rxe_mr_init (int access,struct rxe_mr *mr) {u32 lkey = mr->elem.index <<8 |rxe_get_next_key (-1 );u32 rkey = (access &IB_ACCESS_REMOTE ) ? lkey :0 ;
@@ -77,16 +77,6 @@ void rxe_mr_init_dma(int access, struct rxe_mr *mr) mr->ibmr.type =IB_MR_TYPE_DMA ; } -static unsigned long rxe_mr_iova_to_index (struct rxe_mr *mr,u64 iova) -{ -return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); -} - -static unsigned long rxe_mr_iova_to_page_offset (struct rxe_mr *mr,u64 iova) -{ -return iova & (mr_page_size (mr) -1 ); -} -static bool is_pmem_page (struct page *pg) {unsigned long paddr =page_to_phys (pg);
@@ -96,7 +86,7 @@ static bool is_pmem_page(struct page *pg) IORES_DESC_PERSISTENT_MEMORY); } -static int rxe_mr_fill_pages_from_sgt (struct rxe_mr *mr,struct sg_table *sgt) +int rxe_mr_fill_pages_from_sgt (struct rxe_mr *mr,struct sg_table *sgt) {XA_STATE (xas, &mr->page_list,0 );struct sg_page_iter sg_iter;
@@ -247,8 +237,8 @@ int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl,return ib_sg_to_pages (ibmr, sgl, sg_nents, sg_offset, rxe_set_page); } -static int rxe_mr_copy_xarray (struct rxe_mr *mr,u64 iova,void *addr, -unsigned int length,enum rxe_mr_copy_dir dir) +int rxe_mr_copy_xarray (struct rxe_mr *mr,u64 iova,void *addr, +unsigned int length,enum rxe_mr_copy_dir dir) {unsigned int page_offset =rxe_mr_iova_to_page_offset (mr, iova);unsigned long index =rxe_mr_iova_to_index (mr, iova);
Signature · 1 line
2.39.1
v4 cover On-Demand Paging on SoftRoCE
This patch series implements the On-Demand Paging feature on SoftRoCE(rxe) driver, which has been available only in mlx5 driver[1] so far.
The first patch of this series is provided for testing purpose, and it should be dropped in the end. It converts triple tasklets to use workqueue in order to let them sleep during page-fault. Bob Pearson says he will post the patch to do this, and I think we can adopt that. The other patches in this series are, I believe, completed works.
I omitted some contents like the motive behind this series for simplicity. Please see the cover letter of v3 for more details[2].
[Overview]
When applications register a memory region(MR), RDMA drivers normally pin pages in the MR so that physical addresses are never changed during RDMA communication. This requires the MR to fit in physical memory and inevitably leads to memory pressure. On the other hand, On-Demand Paging (ODP) allows applications to register MRs without pinning pages. They are paged-in when the driver requires and paged-out when the OS reclaims. As a result, it is possible to register a large MR that does not fit in physical memory without taking up so much physical memory.
[How does ODP work?]
"struct ib_umem_odp" is used to manage pages. It is created for each ODP-enabled MR on its registration. This struct holds a pair of arrays (dma_list/pfn_list) that serve as a driver page table. DMA addresses and PFNs are stored in the driver page table. They are updated on page-in and page-out, both of which use the common interfaces in the ib_uverbs layer.
Page-in can occur when requester, responder or completer access an MR in order to process RDMA operations. If they find that the pages being accessed are not present on physical memory or requisite permissions are not set on the pages, they provoke page fault to make the pages present with proper permissions and at the same time update the driver page table. After confirming the presence of the pages, they execute memory access such as read, write or atomic operations.
Page-out is triggered by page reclaim or filesystem events (e.g. metadata update of a file that is being used as an MR). When creating an ODP-enabled MR, the driver registers an MMU notifier callback. When the kernel issues a page invalidation notification, the callback is provoked to unmap DMA addresses and update the driver page table. After that, the kernel releases the pages.
[Supported operations]
All traditional operations are supported on RC connection. The new Atomic write[3] and RDMA Flush[4] operations are not included in this patchset. I will post them later after this patchset is merged. On UD connection, Send, Recv, and SRQ-Recv are supported.
[How to test ODP?]
There are only a few resources available for testing. pyverbs testcases in rdma-core and perftest[5] are recommendable ones. Other than them, the ibv_rc_pingpong command can also used for testing. Note that you may have to build perftest from upstream because older versions do not handle ODP capabilities correctly.
The tree is available from github:
https://github.com/daimatsuda/linux/tree/odp_v4
While this series is based on commit f605f26ea196, the tree includes an additional bugfix, which is yet to be merged as of today (Apr 19th, 2023).
https://lore.kernel.org/linux-rdma/[email protected]/
[Future work]
My next work is to enable the new Atomic write[3] and RDMA Flush[4] operations with ODP. After that, I am going to implement the prefetch feature. It allows applications to trigger page fault using ibv_advise_mr(3) to optimize performance. Some existing software like librpma[6] use this feature. Additionally, I think we can also add the implicit ODP feature in the future.
[1] [RFC 00/20] On demand paging
https://www.spinics.net/lists/linux-rdma/msg18906.html
[2] [PATCH for-next v3 0/7] On-Demand Paging on SoftRoCE
https://lore.kernel.org/lkml/[email protected]/
[3] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation
https://lore.kernel.org/linux-rdma/[email protected]/
[4] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation
https://lore.kernel.org/lkml/[email protected]/
[5] linux-rdma/perftest: Infiniband Verbs Performance Tests
https://github.com/linux-rdma/perftest
[6] librpma: Remote Persistent Memory Access Library
https://github.com/pmem/rpma
v3->v4:
1) Re-designed functions that access MRs to use the MR xarray.
2) Rebased onto the latest jgg-for-next tree.
v2->v3:
1) Removed a patch that changes the common ib_uverbs layer.
2) Re-implemented patches for conversion to workqueue.
3) Fixed compile errors (happened when CONFIG_INFINIBAND_ON_DEMAND_PAGING=n).
4) Fixed some functions that returned incorrect errors.
5) Temporarily disabled ODP for RDMA Flush and Atomic Write.
v1->v2:
1) Fixed a crash issue reported by Haris Iqbal.
2) Tried to make lock patters clearer as pointed out by Romanovsky.
3) Minor clean ups and fixes.
Daisuke Matsuda (8):
RDMA/rxe: Tentative workqueue implementation
RDMA/rxe: Always schedule works before accessing user MRs RDMA/rxe: Make MR functions accessible from other rxe source code RDMA/rxe: Move resp_states definition to rxe_verbs.h RDMA/rxe: Add page invalidation support
RDMA/rxe: Allow registering MRs for On-Demand Paging RDMA/rxe: Add support for Send/Recv/Write/Read with ODP RDMA/rxe: Add support for the traditional Atomic operations with ODP
drivers/infiniband/sw/rxe/Makefile | 2 + drivers/infiniband/sw/rxe/rxe.c | 27 ++- drivers/infiniband/sw/rxe/rxe.h | 37 --- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +- drivers/infiniband/sw/rxe/rxe_loc.h | 49 +++- drivers/infiniband/sw/rxe/rxe_mr.c | 27 +-- drivers/infiniband/sw/rxe/rxe_odp.c | 311 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_recv.c | 4 +- drivers/infiniband/sw/rxe/rxe_resp.c | 32 ++- drivers/infiniband/sw/rxe/rxe_task.c | 84 ++++--- drivers/infiniband/sw/rxe/rxe_task.h | 6 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 39 ++++ 13 files changed, 535 insertions(+), 100 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c
base-commit: f605f26ea196a3b49bea249330cbd18dba61a33e
Signature · 1 line
2.39.1
The work queue patch has been submitted and is waiting for some action. -- Bob
-----Original Message-----
From: Daisuke Matsuda <[email protected]>
Sent: Wednesday, April 19, 2023 12:52 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Daisuke Matsuda <[email protected]>
Subject: [PATCH for-next v4 0/8] On-Demand Paging on SoftRoCE
This patch series implements the On-Demand Paging feature on SoftRoCE(rxe) driver, which has been available only in mlx5 driver[1] so far.
The first patch of this series is provided for testing purpose, and it should be dropped in the end. It converts triple tasklets to use workqueue in order to let them sleep during page-fault. Bob Pearson says he will post the patch to do this, and I think we can adopt that. The other patches in this series are, I believe, completed works.
I omitted some contents like the motive behind this series for simplicity. Please see the cover letter of v3 for more details[2].
[Overview]
When applications register a memory region(MR), RDMA drivers normally pin pages in the MR so that physical addresses are never changed during RDMA communication. This requires the MR to fit in physical memory and inevitably leads to memory pressure. On the other hand, On-Demand Paging (ODP) allows applications to register MRs without pinning pages. They are paged-in when the driver requires and paged-out when the OS reclaims. As a result, it is possible to register a large MR that does not fit in physical memory without taking up so much physical memory.
[How does ODP work?]
"struct ib_umem_odp" is used to manage pages. It is created for each ODP-enabled MR on its registration. This struct holds a pair of arrays
(dma_list/pfn_list) that serve as a driver page table. DMA addresses and PFNs are stored in the driver page table. They are updated on page-in and page-out, both of which use the common interfaces in the ib_uverbs layer.
Page-in can occur when requester, responder or completer access an MR in order to process RDMA operations. If they find that the pages being accessed are not present on physical memory or requisite permissions are not set on the pages, they provoke page fault to make the pages present with proper permissions and at the same time update the driver page table. After confirming the presence of the pages, they execute memory access such as read, write or atomic operations.
Page-out is triggered by page reclaim or filesystem events (e.g. metadata update of a file that is being used as an MR). When creating an ODP-enabled MR, the driver registers an MMU notifier callback. When the kernel issues a page invalidation notification, the callback is provoked to unmap DMA addresses and update the driver page table. After that, the kernel releases the pages.
[Supported operations]
All traditional operations are supported on RC connection. The new Atomic write[3] and RDMA Flush[4] operations are not included in this patchset. I will post them later after this patchset is merged. On UD connection, Send, Recv, and SRQ-Recv are supported.
[How to test ODP?]
There are only a few resources available for testing. pyverbs testcases in rdma-core and perftest[5] are recommendable ones. Other than them, the ibv_rc_pingpong command can also used for testing. Note that you may have to build perftest from upstream because older versions do not handle ODP capabilities correctly.
The tree is available from github:
https://github.com/daimatsuda/linux/tree/odp_v4
While this series is based on commit f605f26ea196, the tree includes an additional bugfix, which is yet to be merged as of today (Apr 19th, 2023).
https://lore.kernel.org/linux-rdma/[email protected]/
[Future work]
My next work is to enable the new Atomic write[3] and RDMA Flush[4] operations with ODP. After that, I am going to implement the prefetch feature. It allows applications to trigger page fault using ibv_advise_mr(3) to optimize performance. Some existing software like librpma[6] use this feature. Additionally, I think we can also add the implicit ODP feature in the future.
[1] [RFC 00/20] On demand paging
https://www.spinics.net/lists/linux-rdma/msg18906.html
[2] [PATCH for-next v3 0/7] On-Demand Paging on SoftRoCE https://lore.kernel.org/lkml/[email protected]/
[3] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation https://lore.kernel.org/linux-rdma/[email protected]/
[4] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation https://lore.kernel.org/lkml/[email protected]/
[5] linux-rdma/perftest: Infiniband Verbs Performance Tests https://github.com/linux-rdma/perftest
[6] librpma: Remote Persistent Memory Access Library https://github.com/pmem/rpma
v3->v4:
1) Re-designed functions that access MRs to use the MR xarray.
2) Rebased onto the latest jgg-for-next tree.
v2->v3:
1) Removed a patch that changes the common ib_uverbs layer.
2) Re-implemented patches for conversion to workqueue.
3) Fixed compile errors (happened when CONFIG_INFINIBAND_ON_DEMAND_PAGING=n).
4) Fixed some functions that returned incorrect errors.
5) Temporarily disabled ODP for RDMA Flush and Atomic Write.
v1->v2:
1) Fixed a crash issue reported by Haris Iqbal.
2) Tried to make lock patters clearer as pointed out by Romanovsky.
3) Minor clean ups and fixes.
Daisuke Matsuda (8):
RDMA/rxe: Tentative workqueue implementation
RDMA/rxe: Always schedule works before accessing user MRs RDMA/rxe: Make MR functions accessible from other rxe source code RDMA/rxe: Move resp_states definition to rxe_verbs.h RDMA/rxe: Add page invalidation support
RDMA/rxe: Allow registering MRs for On-Demand Paging RDMA/rxe: Add support for Send/Recv/Write/Read with ODP RDMA/rxe: Add support for the traditional Atomic operations with ODP
drivers/infiniband/sw/rxe/Makefile | 2 + drivers/infiniband/sw/rxe/rxe.c | 27 ++- drivers/infiniband/sw/rxe/rxe.h | 37 --- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +- drivers/infiniband/sw/rxe/rxe_loc.h | 49 +++- drivers/infiniband/sw/rxe/rxe_mr.c | 27 +-- drivers/infiniband/sw/rxe/rxe_odp.c | 311 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_recv.c | 4 +- drivers/infiniband/sw/rxe/rxe_resp.c | 32 ++- drivers/infiniband/sw/rxe/rxe_task.c | 84 ++++--- drivers/infiniband/sw/rxe/rxe_task.h | 6 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 39 ++++ 13 files changed, 535 insertions(+), 100 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.c
base-commit: f605f26ea196a3b49bea249330cbd18dba61a33e
--
2.39.1
kernel test robot
build botRe: [PATCH for-next v4 2/8] RDMA/rxe: Always schedule works before accessing user MRs
In reply to Daisuke Matsudakernel test robot noticed the following build warnings:
Hi Daisuke,
kernel test robot noticed the following build warnings:
[auto build test WARNING on f605f26ea196a3b49bea249330cbd18dba61a33e]
url: https://github.com/intel-lab-lkp/linux/commits/Daisuke-Matsuda/RDMA-rxe-Tentative-workqueue-implementation/20230419-135731 base: f605f26ea196a3b49bea249330cbd18dba61a33e patch link: https://lore.kernel.org/r/7441c59fcea601c03c70ec03b5d17a69032e51f8.1681882651.git.matsuda-daisuke%40fujitsu.com patch subject: [PATCH for-next v4 2/8] RDMA/rxe: Always schedule works before accessing user MRs config: x86_64-randconfig-a011-20230417 (https://download.01.org/0day-ci/archive/20230420/[email protected]/config) compiler: clang version 14.0.6 (https://github.com/llvm/llvm-project f28c006a5895fc0e329fe15fead81e37457cb1d1)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/493fb0777100e2e1b6358176e84b4b29372105ae
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Daisuke-Matsuda/RDMA-rxe-Tentative-workqueue-implementation/20230419-135731 git checkout 493fb0777100e2e1b6358176e84b4b29372105ae
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 olddefconfig COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash drivers/infiniband/sw/rxe/
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <[email protected]>
| Link: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> drivers/infiniband/sw/rxe/rxe_comp.c:139:36: warning: converting the enum constant to a boolean [-Wint-in-bool-context]
if (pkt->mask | (RXE_PAYLOAD_MASK || RXE_ATMACK_MASK))
^1 warning generated.
vim +139 drivers/infiniband/sw/rxe/rxe_comp.c
128
129 void rxe_comp_queue_pkt(struct rxe_pkt_info *pkt, struct sk_buff *skb)
130 {
131 struct rxe_qp *qp = pkt->qp;
132 int must_sched;
133
134 skb_queue_tail(&qp->resp_pkts, skb);
135
136 /* Schedule the task if processing Read responses or Atomic acks.
137 * In these cases, completer may sleep to access ODP-enabled MRs.
138 */
> 139 if (pkt->mask | (RXE_PAYLOAD_MASK || RXE_ATMACK_MASK))
140 must_sched = 1;
141 else
142 must_sched = skb_queue_len(&qp->resp_pkts) > 1;
143
144 if (must_sched != 0)
145 rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
146
147 if (must_sched)
148 rxe_sched_task(&qp->comp.task);
149 else
150 rxe_run_task(&qp->comp.task);
151 }
152
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests
Quoted 2 lines · Robert B
The work queue patch has been submitted and is waiting for some action. -- Bob
Hi,
Could you tell me which is it? I am willing to review it.
This seems to be your latest work queue patch:
https://lore.kernel.org/all/TYCPR01MB8455A2D0B3303FD90B3BB6F1E58B9@TYCPR01MB8455.jpnprd01.prod.outlook.com/
I cannot find any one newer on the mailing list nor on the Patchwork.
Daisuke
Quoted 126 lines
-----Original Message-----
From: Daisuke Matsuda <[email protected]>
Sent: Wednesday, April 19, 2023 12:52 AM
To: [email protected]; [email protected]; [email protected]; [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Daisuke Matsuda <[email protected]>
Subject: [PATCH for-next v4 0/8] On-Demand Paging on SoftRoCEThis patch series implements the On-Demand Paging feature on SoftRoCE(rxe) driver, which has been available only in mlx5 driver[1] so far.
The first patch of this series is provided for testing purpose, and it should be dropped in the end. It converts triple tasklets to use workqueue in order to let them sleep during page-fault. Bob Pearson says he will post the patch to do this, and I think we can adopt that. The other patches in this series are, I believe, completed works.
I omitted some contents like the motive behind this series for simplicity. Please see the cover letter of v3 for more details[2].
[Overview]
When applications register a memory region(MR), RDMA drivers normally pin pages in the MR so that physical addresses are never changed during RDMA communication. This requires the MR to fit in physical memory and inevitably leads to memory pressure. On the other hand, On-Demand Paging
(ODP) allows applications to register MRs without pinning pages. They are paged-in when the driver requires and paged-out when the OS reclaims. As a result, it is possible to register a large MR that does not fit in physical memory without taking up so much physical memory.[How does ODP work?]
"structib_umem_odp" is used to manage pages. It is created for each ODP-enabled MR on its registration. This struct holds a pair of arrays
(dma_list/pfn_list) that serve as a driver page table. DMA addresses and PFNs are stored in the driver page table. They are updated on page-in and page-out, both of which use the common interfaces in theib_uverbslayer.Page-in can occur when requester, responder or completer access an MR in order to process RDMA operations. If they find that the pages being accessed are not present on physical memory or requisite permissions are not set on the pages, they provoke page fault to make the pages present with proper permissions and at the same time update the driver page table.
After confirming the presence of the pages, they execute memory access such as read, write or atomic operations.Page-out is triggered by page reclaim or filesystem events (e.g. metadata update of a file that is being used as an MR). When creating an ODP-enabled MR, the driver registers an MMU notifier callback. When the kernel issues a page invalidation notification, the callback is provoked to unmap DMA addresses and update the driver page table. After that, the kernel releases the pages.
[Supported operations]
All traditional operations are supported on RC connection. The new Atomic write[3] and RDMA Flush[4] operations are not included in this patchset. I will post them later after this patchset is merged. On UD connection, Send, Recv, and SRQ-Recv are supported.[How to test ODP?]
There are only a few resources available for testing. pyverbs testcases in rdma-core and perftest[5] are recommendable ones. Other than them, theibv_rc_pingpongcommand can also used for testing. Note that you may have to build perftest from upstream because older versions do not handle ODP capabilities correctly.The tree is available from github:
https://github.com/daimatsuda/linux/tree/odp_v4
While this series is based on commitf605f26ea196, the tree includes an additional bugfix, which is yet to be merged as of today (Apr 19th, 2023).
https://lore.kernel.org/linux-rdma/[email protected]/[Future work]
My next work is to enable the new Atomic write[3] and RDMA Flush[4] operations with ODP. After that, I am going to implement the prefetch feature. It allows applications to trigger page fault usingibv_advise_mr(3)to optimize performance. Some existing software like librpma[6] use this feature. Additionally, I think we can also add the implicit ODP feature in the future.[1] [RFC 00/20] On demand paging
https://www.spinics.net/lists/linux-rdma/msg18906.html[2] [PATCH for-next v3 0/7] On-Demand Paging on SoftRoCE
https://lore.kernel.org/lkml/[email protected]/[3] [PATCH v7 0/8] RDMA/rxe: Add atomic write operation
https://lore.kernel.org/linux-rdma/[email protected]/[4] [for-next PATCH 00/10] RDMA/rxe: Add RDMA FLUSH operation
https://lore.kernel.org/lkml/[email protected]/[5] linux-rdma/perftest: Infiniband Verbs Performance Tests https://github.com/linux-rdma/perftest
[6] librpma: Remote Persistent Memory Access Library https://github.com/pmem/rpma
v3->v4:
1) Re-designed functions that access MRs to use the MR xarray.
2) Rebased onto the latest jgg-for-next tree.
v2->v3:
1) Removed a patch that changes the commonib_uverbslayer.
2) Re-implemented patches for conversion to workqueue.
3) Fixed compile errors (happened whenCONFIG_INFINIBAND_ON_DEMAND_PAGING=n).
4) Fixed some functions that returned incorrect errors.
5) Temporarily disabled ODP for RDMA Flush and Atomic Write.
v1->v2:
1) Fixed a crash issue reported by Haris Iqbal.
2) Tried to make lock patters clearer as pointed out by Romanovsky.
3) Minor clean ups and fixes.Daisuke Matsuda (8):
RDMA/rxe: Tentative workqueue implementation
RDMA/rxe: Always schedule works before accessing user MRs RDMA/rxe: Make MR functions accessible from other rxe source code RDMA/rxe: Moveresp_statesdefinition torxe_verbs.hRDMA/rxe: Add page invalidation support
RDMA/rxe: Allow registering MRs for On-Demand Paging RDMA/rxe: Add support for Send/Recv/Write/Read with ODP RDMA/rxe: Add support for the traditional Atomic operations with ODPdrivers/infiniband/sw/rxe/Makefile | 2 + drivers/infiniband/sw/rxe/rxe.c | 27 ++- drivers/infiniband/sw/rxe/rxe.h | 37 --- drivers/infiniband/sw/rxe/rxe_comp.c | 12 +- drivers/infiniband/sw/rxe/rxe_loc.h | 49 +++- drivers/infiniband/sw/rxe/rxe_mr.c | 27 +-- drivers/infiniband/sw/rxe/rxe_odp.c | 311 ++++++++++++++++++++++++++ drivers/infiniband/sw/rxe/rxe_recv.c | 4 +- drivers/infiniband/sw/rxe/rxe_resp.c | 32 ++- drivers/infiniband/sw/rxe/rxe_task.c | 84 ++++--- drivers/infiniband/sw/rxe/rxe_task.h | 6 +- drivers/infiniband/sw/rxe/rxe_verbs.c | 5 +- drivers/infiniband/sw/rxe/rxe_verbs.h | 39 ++++ 13 files changed, 535 insertions(+), 100 deletions(-) create mode 100644 drivers/infiniband/sw/rxe/rxe_odp.cbase-commit:
f605f26ea196a3b49bea249330cbd18dba61a33e--
2.39.1
Quoted 4 lines · Daisuke Matsuda (Fujitsu)
Quoted 2 lines · Robert B
The work queue patch has been submitted and is waiting for some action. -- Bob
Quoted 8 lines
Hi,
Could you tell me which is it? I am willing to review it.This seems to be your latest work queue patch:
https://lore.kernel.org/all/TYCPR01MB8455A2D0B3303FD90B3BB6F1E58B9@TYCPR01MB8455.jpnprd01.prod.outlook.com/
I cannot find any one newer on the mailing list nor on the Patchwork.Daisuke
Daisuke,
Sorry for the delay. I've been on another project for a few days.
I can't either. After the fix qp counting in task.c the work queue patch is almost trivial. I'll send it again.
Bob