ublk: auto buffer helpers rename & document io reference
Subsystems (2)
cover ublk: auto buffer helpers rename & document io reference
Hi Jens,
The 1st patch improves auto buffer helpers naming, and improve documentation.
The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.
Thanks,
Ming Lei (2):
ublk: rename auto buffer registration helpers
ublk: document IO reference counting design
drivers/block/ublk_drv.c | 103 ++++++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 13 deletions(-)
Signature · 1 line
2.47.0
PATCH 1/2 ublk: rename auto buffer registration helpers
Rename the auto buffer registration functions for clarity:
- __ublk_do_auto_buf_reg() -> ublk_auto_buf_register()
- ublk_prep_auto_buf_reg_io() -> ublk_auto_buf_io_setup()
- ublk_do_auto_buf_reg() -> ublk_auto_buf_dispatch()
Add comments documenting the locking requirements for each function.
No functional change.
--- drivers/block/ublk_drv.c | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-)
drivers/block/ublk_drv.c +26 -13
@@ -1509,10 +1509,16 @@ enum auto_buf_reg_res { AUTO_BUF_REG_OK, }; -static void ublk_prep_auto_buf_reg_io (const struct ublk_queue *ubq, -struct request *req,struct ublk_io *io, -struct io_uring_cmd *cmd, -enum auto_buf_reg_res res) +/* +* Setup io state after auto buffer registration. +* +* Must be called after ublk_auto_buf_register() is done. +* Caller must hold io->lock in batch context. +*/ +static void ublk_auto_buf_io_setup (const struct ublk_queue *ubq, +struct request *req,struct ublk_io *io, +struct io_uring_cmd *cmd, +enum auto_buf_reg_res res) {if (res ==AUTO_BUF_REG_OK ) { io->task_registered_buffers =1 ;
@@ -1523,8 +1529,9 @@ static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq,__ublk_prep_compl_io_cmd (io, req); } +/* Register request bvec to io_uring for auto buffer registration. */ static enum auto_buf_reg_res -__ublk_do_auto_buf_reg (const struct ublk_queue *ubq,struct request *req, +ublk_auto_buf_register (const struct ublk_queue *ubq,struct request *req,struct ublk_io *io,struct io_uring_cmd *cmd,unsigned int issue_flags) {
@@ -1544,15 +1551,21 @@ __ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,return AUTO_BUF_REG_OK ; } -static void ublk_do_auto_buf_reg (const struct ublk_queue *ubq,struct request *req, -struct ublk_io *io,struct io_uring_cmd *cmd, -unsigned int issue_flags) +/* +* Dispatch IO to userspace with auto buffer registration. +* +* Only called in non-batch context from task work, io->lock not held. +*/ +static void ublk_auto_buf_dispatch (const struct ublk_queue *ubq, +struct request *req,struct ublk_io *io, +struct io_uring_cmd *cmd, +unsigned int issue_flags) { -enum auto_buf_reg_res res =__ublk_do_auto_buf_reg (ubq, req, io, cmd, +enum auto_buf_reg_res res =ublk_auto_buf_register (ubq, req, io, cmd, issue_flags);if (res !=AUTO_BUF_REG_FAIL ) { -ublk_prep_auto_buf_reg_io (ubq, req, io, cmd, res); +ublk_auto_buf_io_setup (ubq, req, io, cmd, res);io_uring_cmd_done (cmd,UBLK_IO_RES_OK , issue_flags); } }
@@ -1627,7 +1640,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)return ;if (ublk_support_auto_buf_reg (ubq) &&ublk_rq_has_data (req)) { -ublk_do_auto_buf_reg (ubq, req, io, io->cmd, issue_flags); +ublk_auto_buf_dispatch (ubq, req, io, io->cmd, issue_flags); }else {ublk_init_req_ref (ubq, io);ublk_complete_io_cmd (io, req,UBLK_IO_RES_OK , issue_flags);
@@ -1648,7 +1661,7 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq,return false;if (ublk_support_auto_buf_reg (ubq) &&ublk_rq_has_data (req)) { - res =__ublk_do_auto_buf_reg (ubq, req, io, cmd, + res =ublk_auto_buf_register (ubq, req, io, cmd, data->issue_flags);if (res ==AUTO_BUF_REG_FAIL )
@@ -1656,7 +1669,7 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq, }ublk_io_lock (io); -ublk_prep_auto_buf_reg_io (ubq, req, io, cmd, res); +ublk_auto_buf_io_setup (ubq, req, io, cmd, res);ublk_io_unlock (io);return true;
Signature · 1 line
2.47.0
PATCH 2/2 ublk: document IO reference counting design
In reply to Ming LeiAdd comprehensive documentation for ublk's split reference counting model (io->ref + io->task_registered_buffers) above ublk_init_req_ref() given this model isn't very straightforward.
--- drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)
drivers/block/ublk_drv.c +64 -0
@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)ublk_dev_support_auto_buf_reg (ub); } +/* +* ublk IO Reference Counting Design +* ================================== +* +* For user-copy and zero-copy modes, ublk uses a split reference model with +* two counters that together track IO lifetime: +* +* - io->ref: refcount for off-task buffer registrations and user-copy ops +* - io->task_registered_buffers: count of buffers registered on the IO task +* +* Key Invariant: +* -------------- +* The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT +* when no active references exist. This invariant is checked by +* ublk_check_and_reset_active_ref() during daemon exit to determine if all +* references have been released. +* +* Why Split Counters: +* ------------------- +* Buffers registered on the IO daemon task can use the lightweight +* task_registered_buffers counter (simple increment/decrement) instead of +* atomic refcount operations. The ublk_io_release() callback checks if +* current == io->task to decide which counter to update. +* +* Reference Lifecycle: +* -------------------- +* 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch +* +* 2. During IO processing: +* - On-task buffer reg: task_registered_buffers++ (no ref change) +* - Off-task buffer reg: ref++ via ublk_get_req_ref() +* - Buffer unregister callback (ublk_io_release): +* * If on-task: task_registered_buffers-- +* * If off-task: ref-- via ublk_put_req_ref() +* +* 3. ublk_sub_req_ref() at IO completion: +* - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers +* - Subtracts sub_refs from ref +* - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task +* buffers that were already counted in task_registered_buffers +* +* Example (zero-copy, register on-task, unregister off-task): +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0 +* - Register buffer on-task: task_registered_buffers = 1 +* - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1 +* - Completion via ublk_sub_req_ref(): +* sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0 +* +* Example (auto buffer registration): +* Auto buffer registration sets task_registered_buffers = 1 at dispatch. +* +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1 +* - Buffer unregister: task_registered_buffers-- (becomes 0) +* - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0 +* - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT +* - Sum equals UBLK_REFCOUNT_INIT, so no active reference exists +* +* Batch IO Special Case: +* ---------------------- +* In batch IO mode, io->task is NULL. This means ublk_io_release() always +* takes the off-task path (ublk_put_req_ref), decrementing io->ref. The +* task_registered_buffers counter still tracks registered buffers for the +* invariant check, even though the callback doesn't decrement it. +*/ static inline void ublk_init_req_ref (const struct ublk_queue *ubq,struct ublk_io *io) {
Signature · 1 line
2.47.0
Re: [PATCH 2/2] ublk: document IO reference counting design
Quoted 21 lines · Ming Lei
Add comprehensive documentation for ublk's split reference counting model (
io->ref+io->task_registered_buffers) aboveublk_init_req_ref()given this model isn't very straightforward.Signed-off-by Ming Lei <[email protected]>--- drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)drivers/block/ublk_drv.c +3
-0@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)ublk_dev_support_auto_buf_reg (ub); } +/* +* ublk IO Reference Counting Design +* ==================================
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* +* For user-copy and zero-copy modes, ublk uses a split reference model with +* two counters that together track IO lifetime: +* +* - io->ref: refcount for off-task buffer registrations and user-copy ops +* - io->task_registered_buffers: count of buffers registered on the IO task +* +* Key Invariant: +* -------------- +* The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT
This is only true when UBLK_IO_FLAG_OWNED_BY_SRV is set. If the ublk_io isn't currently dispatched to the ublk server, ref and task_registered_buffers would both be 0.
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* when no active references exist. This invariant is checked by +* ublk_check_and_reset_active_ref() during daemon exit to determine if all +* references have been released. +* +* Why Split Counters: +* ------------------- +* Buffers registered on the IO daemon task can use the lightweight +* task_registered_buffers counter (simple increment/decrement) instead of +* atomic refcount operations. The ublk_io_release() callback checks if +* current == io->task to decide which counter to update.
However, this optimization can only be used before the I/O is completed, since task_registered_buffers is collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.
Quoted 7 lines
drivers/block/ublk_drv.c +7
-0+* +* Reference Lifecycle: +* -------------------- +* 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch +* +* 2. During IO processing: +* - On-task buffer reg: task_registered_buffers++ (no ref change)
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Off-task buffer reg: ref++ via ublk_get_req_ref() +* - Buffer unregister callback (ublk_io_release): +* * If on-task: task_registered_buffers-- +* * If off-task: ref-- via ublk_put_req_ref() +* +* 3. ublk_sub_req_ref() at IO completion: +* - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers +* - Subtracts sub_refs from ref +* - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task +* buffers that were already counted in task_registered_buffers
I would note that task_registered_buffers is also zeroed during ublk_sub_req_ref(). Effectively, it's transferring/collapsing task_registered_buffers into the atomic ref.
Quoted 4 lines
drivers/block/ublk_drv.c +4
-0+* +* Example (zero-copy, register on-task, unregister off-task): +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0 +* - Register buffer on-task: task_registered_buffers = 1
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1041
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1 +* - Completion via ublk_sub_req_ref(): +* sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0 +* +* Example (auto buffer registration): +* Auto buffer registration sets task_registered_buffers = 1 at dispatch. +* +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1 +* - Buffer unregister: task_registered_buffers-- (becomes 0) +* - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0
I think the order here was actually reversed in commit b749965edda8 "ublk: remove ublk_commit_and_fetch()". ublk_need_complete_req() will call ublk_sub_req_ref(), and io_buffer_unregister_bvec() is called *afterwards*, which results in a call to ublk_io_release(). So the buffer unregister will end up performing an atomic decrement of ref.
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042
Quoted 1 line
drivers/block/ublk_drv.c +1
-0+* - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT
Both ref and task_registered_buffers are 0 at this point, no?
Best,
Caleb
Quoted 15 lines
drivers/block/ublk_drv.c +9
-1+* - Sum equals UBLK_REFCOUNT_INIT, so no active reference exists +* +* Batch IO Special Case: +* ---------------------- +* In batch IO mode, io->task is NULL. This means ublk_io_release() always +* takes the off-task path (ublk_put_req_ref), decrementing io->ref. The +* task_registered_buffers counter still tracks registered buffers for the +* invariant check, even though the callback doesn't decrement it. +*/ static inline void ublk_init_req_ref (const struct ublk_queue *ubq,struct ublk_io *io) { --2.47.0
Re: [PATCH 2/2] ublk: document IO reference counting design
Quoted 28 lines · Caleb Sander Mateos
Quoted 27 lines · Ming Lei
Add comprehensive documentation for ublk's split reference counting model (
io->ref+io->task_registered_buffers) aboveublk_init_req_ref()given this model isn't very straightforward.Signed-off-by Ming Lei <[email protected]>--- drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)drivers/block/ublk_drv.c +9
-0@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)ublk_dev_support_auto_buf_reg (ub); } +/* +* ublk IO Reference Counting Design +* ================================== +* +* For user-copy and zero-copy modes, ublk uses a split reference model with +* two counters that together track IO lifetime: +* +* - io->ref: refcount for off-task buffer registrations and user-copy ops +* - io->task_registered_buffers: count of buffers registered on the IO task
Quoted 8 lines
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000
Quoted 4 lines
drivers/block/ublk_drv.c +4
-0+* +* Key Invariant: +* -------------- +* The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT This is only true when
UBLK_IO_FLAG_OWNED_BY_SRVis set. If theublk_ioisn't currently dispatched to the ublk server, ref andtask_registered_bufferswould both be 0.
Yeah, it also means the reference counter is initialized, which is implied in the whole doc.
Quoted 12 lines
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010
Quoted 11 lines
drivers/block/ublk_drv.c +10
-0+* when no active references exist. This invariant is checked by +* ublk_check_and_reset_active_ref() during daemon exit to determine if all +* references have been released. +* +* Why Split Counters: +* ------------------- +* Buffers registered on the IO daemon task can use the lightweight +* task_registered_buffers counter (simple increment/decrement) instead of +* atomic refcount operations. The ublk_io_release() callback checks if +* current == io->task to decide which counter to update.
Quoted 4 lines
However, this optimization can only be used before the I/O is completed, since
task_registered_buffersis collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.
The point is ublk_need_complete_req()/ublk_sub_req_ref(), we can make it explicit.
Quoted 19 lines
Quoted 8 lines
drivers/block/ublk_drv.c +7
-0+* +* Reference Lifecycle: +* -------------------- +* 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch +* +* 2. During IO processing: +* - On-task buffer reg: task_registered_buffers++ (no ref change) drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Off-task buffer reg: ref++ via ublk_get_req_ref() +* - Buffer unregister callback (ublk_io_release): +* * If on-task: task_registered_buffers-- +* * If off-task: ref-- via ublk_put_req_ref() +* +* 3. ublk_sub_req_ref() at IO completion: +* - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers +* - Subtracts sub_refs from ref +* - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task +* buffers that were already counted in task_registered_buffers
Quoted 3 lines
I would note that
task_registered_buffersis also zeroed duringublk_sub_req_ref(). Effectively, it's transferring/collapsingtask_registered_buffersinto the atomic ref.
OK.
Quoted 16 lines
Quoted 5 lines
drivers/block/ublk_drv.c +4
-0+* +* Example (zero-copy, register on-task, unregister off-task): +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0 +* - Register buffer on-task: task_registered_buffers = 1 drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1041
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1 +* - Completion via ublk_sub_req_ref(): +* sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0 +* +* Example (auto buffer registration): +* Auto buffer registration sets task_registered_buffers = 1 at dispatch. +* +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1 +* - Buffer unregister: task_registered_buffers-- (becomes 0) +* - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0
Quoted 4 lines
I think the order here was actually reversed in commit
b749965edda8"ublk: removeublk_commit_and_fetch()".ublk_need_complete_req()will callublk_sub_req_ref(), andio_buffer_unregister_bvec()is called *afterwards*, which results in a call toublk_io_release(). So the
Right.
Quoted 5 lines
buffer unregister will end up performing an atomic decrement of ref.
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042
Quoted 1 line
drivers/block/ublk_drv.c +1
-0+* - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT Both ref and
task_registered_buffersare 0 at this point, no?
It can be UBLK_REFCOUNT_INIT or zero, please see ublk_check_and_reset_active_ref(), the doc mentions Daemon exit check. But yes, both reset finally when we confirm that the io hasn't active references.
Thanks,
Ming
Re: [PATCH 2/2] ublk: document IO reference counting design
Quoted 39 lines · Ming Lei
Quoted 28 lines · Caleb Sander Mateos
Quoted 27 lines · Ming Lei
Add comprehensive documentation for ublk's split reference counting model (
io->ref+io->task_registered_buffers) aboveublk_init_req_ref()given this model isn't very straightforward.Signed-off-by Ming Lei <[email protected]>--- drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)drivers/block/ublk_drv.c +9
-0@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)ublk_dev_support_auto_buf_reg (ub); } +/* +* ublk IO Reference Counting Design +* ================================== +* +* For user-copy and zero-copy modes, ublk uses a split reference model with +* two counters that together track IO lifetime: +* +* - io->ref: refcount for off-task buffer registrations and user-copy ops +* - io->task_registered_buffers: count of buffers registered on the IO task Quoted 8 lines
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000
Quoted 4 lines
drivers/block/ublk_drv.c +4
-0+* +* Key Invariant: +* -------------- +* The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT This is only true when
UBLK_IO_FLAG_OWNED_BY_SRVis set. If theublk_ioisn't currently dispatched to the ublk server, ref andtask_registered_bufferswould both be 0.
Quoted 2 lines
Yeah, it also means the reference counter is initialized, which is implied in the whole doc.
Would you mind mentioning explicitly that both reference counts are 0 for I/Os not currently dispatched to the ublk server?
Quoted 62 lines
Quoted 13 lines
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010
Quoted 12 lines
drivers/block/ublk_drv.c +10
-0+* when no active references exist. This invariant is checked by +* ublk_check_and_reset_active_ref() during daemon exit to determine if all +* references have been released. +* +* Why Split Counters: +* ------------------- +* Buffers registered on the IO daemon task can use the lightweight +* task_registered_buffers counter (simple increment/decrement) instead of +* atomic refcount operations. The ublk_io_release() callback checks if +* current == io->task to decide which counter to update. Quoted 4 lines
However, this optimization can only be used before the I/O is completed, since
task_registered_buffersis collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.The point is ublk_need_complete_req()/ublk_sub_req_ref(), we can make it explicit.
Quoted 19 lines
Quoted 8 lines
drivers/block/ublk_drv.c +7
-0+* +* Reference Lifecycle: +* -------------------- +* 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch +* +* 2. During IO processing: +* - On-task buffer reg: task_registered_buffers++ (no ref change) drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Off-task buffer reg: ref++ via ublk_get_req_ref() +* - Buffer unregister callback (ublk_io_release): +* * If on-task: task_registered_buffers-- +* * If off-task: ref-- via ublk_put_req_ref() +* +* 3. ublk_sub_req_ref() at IO completion: +* - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers +* - Subtracts sub_refs from ref +* - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task +* buffers that were already counted in task_registered_buffers Quoted 3 lines
I would note that
task_registered_buffersis also zeroed duringublk_sub_req_ref(). Effectively, it's transferring/collapsingtask_registered_buffersinto the atomic ref.OK.
Quoted 16 lines
Quoted 15 lines
drivers/block/ublk_drv.c +14
-0+* +* Example (zero-copy, register on-task, unregister off-task): +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0 +* - Register buffer on-task: task_registered_buffers = 1 +* - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1 +* - Completion via ublk_sub_req_ref(): +* sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0 +* +* Example (auto buffer registration): +* Auto buffer registration sets task_registered_buffers = 1 at dispatch. +* +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1 +* - Buffer unregister: task_registered_buffers-- (becomes 0) +* - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0
Quoted 6 lines
Quoted 4 lines
I think the order here was actually reversed in commit
b749965edda8"ublk: removeublk_commit_and_fetch()".ublk_need_complete_req()will callublk_sub_req_ref(), andio_buffer_unregister_bvec()is called *afterwards*, which results in a call toublk_io_release(). So theRight.
Should I send out a patch to restore the original order to recover the non-atomic refcount optimization?
Quoted 10 lines
Quoted 6 lines
buffer unregister will end up performing an atomic decrement of ref.
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042
Quoted 1 line
drivers/block/ublk_drv.c +1
-0+* - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT Both ref and
task_registered_buffersare 0 at this point, no?It can be
UBLK_REFCOUNT_INITor zero, please seeublk_check_and_reset_active_ref(), the doc mentionsDaemon exit check. But yes, both reset finally when we confirm that the io hasn't active references.
I mainly found it confusing that "Daemon exit check" is listed under "Example (auto buffer registration)". That makes it sounds like it's a continuation of the steps above, in which case ref and task_registered_buffers should be 0 at this point. Consider separating the part about the daemon exit check into its own section?
Best,
Caleb
Re: [PATCH 2/2] ublk: document IO reference counting design
Quoted 40 lines · Caleb Sander Mateos
Quoted 39 lines · Ming Lei
Quoted 36 lines · Caleb Sander Mateos
Quoted 21 lines · Ming Lei
Add comprehensive documentation for ublk's split reference counting model (
io->ref+io->task_registered_buffers) aboveublk_init_req_ref()given this model isn't very straightforward.Signed-off-by Ming Lei <[email protected]>--- drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+)drivers/block/ublk_drv.c +3
-0@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)ublk_dev_support_auto_buf_reg (ub); } +/* +* ublk IO Reference Counting Design +* ================================== drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* +* For user-copy and zero-copy modes, ublk uses a split reference model with +* two counters that together track IO lifetime: +* +* - io->ref: refcount for off-task buffer registrations and user-copy ops +* - io->task_registered_buffers: count of buffers registered on the IO task +* +* Key Invariant: +* -------------- +* The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT This is only true when
UBLK_IO_FLAG_OWNED_BY_SRVis set. If theublk_ioisn't currently dispatched to the ublk server, ref andtask_registered_bufferswould both be 0.
Quoted 5 lines
Quoted 2 lines
Yeah, it also means the reference counter is initialized, which is implied in the whole doc.
Would you mind mentioning explicitly that both reference counts are 0 for I/Os not currently dispatched to the ublk server?
Sure.
Quoted 68 lines
Quoted 68 lines
Quoted 14 lines
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010
Quoted 13 lines
drivers/block/ublk_drv.c +10
-0+* when no active references exist. This invariant is checked by +* ublk_check_and_reset_active_ref() during daemon exit to determine if all +* references have been released. +* +* Why Split Counters: +* ------------------- +* Buffers registered on the IO daemon task can use the lightweight +* task_registered_buffers counter (simple increment/decrement) instead of +* atomic refcount operations. The ublk_io_release() callback checks if +* current == io->task to decide which counter to update. Quoted 4 lines
However, this optimization can only be used before the I/O is completed, since
task_registered_buffersis collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.The point is ublk_need_complete_req()/ublk_sub_req_ref(), we can make it explicit.
Quoted 19 lines
Quoted 8 lines
drivers/block/ublk_drv.c +7
-0+* +* Reference Lifecycle: +* -------------------- +* 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch +* +* 2. During IO processing: +* - On-task buffer reg: task_registered_buffers++ (no ref change) drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Off-task buffer reg: ref++ via ublk_get_req_ref() +* - Buffer unregister callback (ublk_io_release): +* * If on-task: task_registered_buffers-- +* * If off-task: ref-- via ublk_put_req_ref() +* +* 3. ublk_sub_req_ref() at IO completion: +* - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers +* - Subtracts sub_refs from ref +* - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task +* buffers that were already counted in task_registered_buffers Quoted 3 lines
I would note that
task_registered_buffersis also zeroed duringublk_sub_req_ref(). Effectively, it's transferring/collapsingtask_registered_buffersinto the atomic ref.OK.
Quoted 20 lines
Quoted 5 lines
drivers/block/ublk_drv.c +4
-0+* +* Example (zero-copy, register on-task, unregister off-task): +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0 +* - Register buffer on-task: task_registered_buffers = 1 drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1041
Quoted 10 lines
drivers/block/ublk_drv.c +10
-0+* - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1 +* - Completion via ublk_sub_req_ref(): +* sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0 +* +* Example (auto buffer registration): +* Auto buffer registration sets task_registered_buffers = 1 at dispatch. +* +* - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1 +* - Buffer unregister: task_registered_buffers-- (becomes 0) +* - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0 I think the order here was actually reversed in commit
b749965edda8"ublk: removeublk_commit_and_fetch()".ublk_need_complete_req()will callublk_sub_req_ref(), andio_buffer_unregister_bvec()is called *afterwards*, which results in a call toublk_io_release(). So the
Quoted 4 lines
Quoted 1 line
Right.
Should I send out a patch to restore the original order to recover the non-atomic refcount optimization?
Yeah, I think it is good to make the optimization back.
Quoted 12 lines
Quoted 11 lines
Quoted 7 lines
buffer unregister will end up performing an atomic decrement of ref.
drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042
Quoted 1 line
drivers/block/ublk_drv.c +1
-0+* - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT Both ref and
task_registered_buffersare 0 at this point, no?It can be
UBLK_REFCOUNT_INITor zero, please seeublk_check_and_reset_active_ref(), the doc mentionsDaemon exit check. But yes, both reset finally when we confirm that the io hasn't active references.
Quoted 5 lines
I mainly found it confusing that "Daemon exit check" is listed under "Example (auto buffer registration)". That makes it sounds like it's a continuation of the steps above, in which case ref and
task_registered_buffersshould be 0 at this point. Consider separating the part about the daemon exit check into its own section?
OK.
Also both may not be zero in Daemon exit check when the server is killed, which should be added as example too.
Thanks,
Ming
Quoted 9 lines · Ming Lei
The 1st patch improves auto buffer helpers naming, and improve documentation.
The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.
Thanks,
[...]
Applied, thanks!
[1/2] ublk: rename auto buffer registration helpers
commit: f50af896932f5edb1ff7b407753ecfa285c30b7a
[2/2] ublk: document IO reference counting design
commit: 420bcbf67b4575d6455a4fa0de06695d98d340b1Best regards,
Signature · 1 line
Jens Axboe
Quoted 10 lines · Jens Axboe
Quoted 8 lines · Ming Lei
The 1st patch improves auto buffer helpers naming, and improve documentation.
The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.
Thanks,
Quoted 8 lines
Quoted 1 line
[...]
Applied, thanks!
[1/2] ublk: rename auto buffer registration helpers
commit: f50af896932f5edb1ff7b407753ecfa285c30b7a [2/2] ublk: document IO reference counting design commit: 420bcbf67b4575d6455a4fa0de06695d98d340b1
Hi Jens,
I'd address Caleb's comment on the 2nd patch, so can you drop it?
Thanks,
Ming
Jens Axboe
maintainerQuoted 15 lines · Ming Lei
Quoted 14 lines · Jens Axboe
Quoted 9 lines · Ming Lei
The 1st patch improves auto buffer helpers naming, and improve documentation.
The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.
Thanks,
[...]
Applied, thanks!
Quoted 8 lines
Quoted 4 lines
[1/2] ublk: rename auto buffer registration helpers
commit: f50af896932f5edb1ff7b407753ecfa285c30b7a [2/2] ublk: document IO reference counting design commit: 420bcbf67b4575d6455a4fa0de06695d98d340b1Hi Jens,
I'd address Caleb's comment on the 2nd patch, so can you drop it?
Dropped
Signature · 1 line
Jens Axboe