Skip to contentSkip to conversation

ublk: auto buffer helpers rename & document io reference

PATCH Applied Ming Lei 3 participants 10 messages Jan 23 – Jan 24

Subsystems (2)
lore
Share & tools
lore

Ming Lei

author
#0 lore raw

cover ublk: auto buffer helpers rename & document io reference

Hi Jens,

The 1st patch improves auto buffer helpers naming, and improve documentation.

The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.

Thanks,

Ming Lei (2):
ublk: rename auto buffer registration helpers
ublk: document IO reference counting design

 drivers/block/ublk_drv.c | 103 ++++++++++++++++++++++++++++++++++-----
 1 file changed, 90 insertions(+), 13 deletions(-)
Signature · 1 line
2.47.0
author
#1 lore raw

PATCH 1/2 ublk: rename auto buffer registration helpers

Rename the auto buffer registration functions for clarity:

- __ublk_do_auto_buf_reg() -> ublk_auto_buf_register()
- ublk_prep_auto_buf_reg_io() -> ublk_auto_buf_io_setup()
- ublk_do_auto_buf_reg() -> ublk_auto_buf_dispatch()

Add comments documenting the locking requirements for each function.

No functional change.

---
 drivers/block/ublk_drv.c | 39 ++++++++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 13 deletions(-)
drivers/block/ublk_drv.c +26 -13
index 31fda782c47c..7981decd1cee 100644
@@ -1509,10 +1509,16 @@ enum auto_buf_reg_res {
 	AUTO_BUF_REG_OK,
 };
 
-static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq,
-				      struct request *req, struct ublk_io *io,
-				      struct io_uring_cmd *cmd,
-				      enum auto_buf_reg_res res)
+/*
+ * Setup io state after auto buffer registration.
+ *
+ * Must be called after ublk_auto_buf_register() is done.
+ * Caller must hold io->lock in batch context.
+ */
+static void ublk_auto_buf_io_setup(const struct ublk_queue *ubq,
+				   struct request *req, struct ublk_io *io,
+				   struct io_uring_cmd *cmd,
+				   enum auto_buf_reg_res res)
 {
 	if (res == AUTO_BUF_REG_OK) {
 		io->task_registered_buffers = 1;
@@ -1523,8 +1529,9 @@ static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq,
 	__ublk_prep_compl_io_cmd(io, req);
 }
 
+/* Register request bvec to io_uring for auto buffer registration. */
 static enum auto_buf_reg_res
-__ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
+ublk_auto_buf_register(const struct ublk_queue *ubq, struct request *req,
 		       struct ublk_io *io, struct io_uring_cmd *cmd,
 		       unsigned int issue_flags)
 {
@@ -1544,15 +1551,21 @@ __ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
 	return AUTO_BUF_REG_OK;
 }
 
-static void ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req,
-				 struct ublk_io *io, struct io_uring_cmd *cmd,
-				 unsigned int issue_flags)
+/*
+ * Dispatch IO to userspace with auto buffer registration.
+ *
+ * Only called in non-batch context from task work, io->lock not held.
+ */
+static void ublk_auto_buf_dispatch(const struct ublk_queue *ubq,
+				   struct request *req, struct ublk_io *io,
+				   struct io_uring_cmd *cmd,
+				   unsigned int issue_flags)
 {
-	enum auto_buf_reg_res res = __ublk_do_auto_buf_reg(ubq, req, io, cmd,
+	enum auto_buf_reg_res res = ublk_auto_buf_register(ubq, req, io, cmd,
 			issue_flags);
 
 	if (res != AUTO_BUF_REG_FAIL) {
-		ublk_prep_auto_buf_reg_io(ubq, req, io, cmd, res);
+		ublk_auto_buf_io_setup(ubq, req, io, cmd, res);
 		io_uring_cmd_done(cmd, UBLK_IO_RES_OK, issue_flags);
 	}
 }
@@ -1627,7 +1640,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
 		return;
 
 	if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) {
-		ublk_do_auto_buf_reg(ubq, req, io, io->cmd, issue_flags);
+		ublk_auto_buf_dispatch(ubq, req, io, io->cmd, issue_flags);
 	} else {
 		ublk_init_req_ref(ubq, io);
 		ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags);
@@ -1648,7 +1661,7 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq,
 		return false;
 
 	if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) {
-		res = __ublk_do_auto_buf_reg(ubq, req, io, cmd,
+		res = ublk_auto_buf_register(ubq, req, io, cmd,
 				data->issue_flags);
 
 		if (res == AUTO_BUF_REG_FAIL)
@@ -1656,7 +1669,7 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq,
 	}
 
 	ublk_io_lock(io);
-	ublk_prep_auto_buf_reg_io(ubq, req, io, cmd, res);
+	ublk_auto_buf_io_setup(ubq, req, io, cmd, res);
 	ublk_io_unlock(io);
 
 	return true;
Signature · 1 line
2.47.0
author
#2 lore raw

PATCH 2/2 ublk: document IO reference counting design

In reply to Ming Lei

Add comprehensive documentation for ublk's split reference counting model (io->ref + io->task_registered_buffers) above ublk_init_req_ref() given this model isn't very straightforward.

---
 drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
drivers/block/ublk_drv.c +64 -0
index 7981decd1cee..91218b78e711 100644
@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
 	       ublk_dev_support_auto_buf_reg(ub);
 }
 
+/*
+ * ublk IO Reference Counting Design
+ * ==================================
+ *
+ * For user-copy and zero-copy modes, ublk uses a split reference model with
+ * two counters that together track IO lifetime:
+ *
+ *   - io->ref: refcount for off-task buffer registrations and user-copy ops
+ *   - io->task_registered_buffers: count of buffers registered on the IO task
+ *
+ * Key Invariant:
+ * --------------
+ * The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT
+ * when no active references exist. This invariant is checked by
+ * ublk_check_and_reset_active_ref() during daemon exit to determine if all
+ * references have been released.
+ *
+ * Why Split Counters:
+ * -------------------
+ * Buffers registered on the IO daemon task can use the lightweight
+ * task_registered_buffers counter (simple increment/decrement) instead of
+ * atomic refcount operations. The ublk_io_release() callback checks if
+ * current == io->task to decide which counter to update.
+ *
+ * Reference Lifecycle:
+ * --------------------
+ * 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch
+ *
+ * 2. During IO processing:
+ *    - On-task buffer reg: task_registered_buffers++ (no ref change)
+ *    - Off-task buffer reg: ref++ via ublk_get_req_ref()
+ *    - Buffer unregister callback (ublk_io_release):
+ *      * If on-task: task_registered_buffers--
+ *      * If off-task: ref-- via ublk_put_req_ref()
+ *
+ * 3. ublk_sub_req_ref() at IO completion:
+ *    - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers
+ *    - Subtracts sub_refs from ref
+ *    - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task
+ *      buffers that were already counted in task_registered_buffers
+ *
+ * Example (zero-copy, register on-task, unregister off-task):
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0
+ *   - Register buffer on-task: task_registered_buffers = 1
+ *   - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1
+ *   - Completion via ublk_sub_req_ref():
+ *     sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0
+ *
+ * Example (auto buffer registration):
+ *   Auto buffer registration sets task_registered_buffers = 1 at dispatch.
+ *
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1
+ *   - Buffer unregister: task_registered_buffers-- (becomes 0)
+ *   - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0
+ *   - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT
+ *   - Sum equals UBLK_REFCOUNT_INIT, so no active reference exists
+ *
+ * Batch IO Special Case:
+ * ----------------------
+ * In batch IO mode, io->task is NULL. This means ublk_io_release() always
+ * takes the off-task path (ublk_put_req_ref), decrementing io->ref. The
+ * task_registered_buffers counter still tracks registered buffers for the
+ * invariant check, even though the callback doesn't decrement it.
+ */
 static inline void ublk_init_req_ref(const struct ublk_queue *ubq,
 		struct ublk_io *io)
 {
Signature · 1 line
2.47.0
#3 lore raw

Re: [PATCH 2/2] ublk: document IO reference counting design

Quoted 21 lines · Ming Lei

Add comprehensive documentation for ublk's split reference counting model (io->ref + io->task_registered_buffers) above ublk_init_req_ref() given this model isn't very straightforward.

---
 drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
drivers/block/ublk_drv.c +3 -0
index 7981decd1cee..91218b78e711 100644
@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
               ublk_dev_support_auto_buf_reg(ub);
 }
 
+/*
+ * ublk IO Reference Counting Design
+ * ==================================

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *
+ * For user-copy and zero-copy modes, ublk uses a split reference model with
+ * two counters that together track IO lifetime:
+ *
+ *   - io->ref: refcount for off-task buffer registrations and user-copy ops
+ *   - io->task_registered_buffers: count of buffers registered on the IO task
+ *
+ * Key Invariant:
+ * --------------
+ * The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT

This is only true when UBLK_IO_FLAG_OWNED_BY_SRV is set. If the ublk_io isn't currently dispatched to the ublk server, ref and task_registered_buffers would both be 0.

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ * when no active references exist. This invariant is checked by
+ * ublk_check_and_reset_active_ref() during daemon exit to determine if all
+ * references have been released.
+ *
+ * Why Split Counters:
+ * -------------------
+ * Buffers registered on the IO daemon task can use the lightweight
+ * task_registered_buffers counter (simple increment/decrement) instead of
+ * atomic refcount operations. The ublk_io_release() callback checks if
+ * current == io->task to decide which counter to update.

However, this optimization can only be used before the I/O is completed, since task_registered_buffers is collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.

Quoted 7 lines
drivers/block/ublk_drv.c +7 -0
+ *
+ * Reference Lifecycle:
+ * --------------------
+ * 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch
+ *
+ * 2. During IO processing:
+ *    - On-task buffer reg: task_registered_buffers++ (no ref change)

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *    - Off-task buffer reg: ref++ via ublk_get_req_ref()
+ *    - Buffer unregister callback (ublk_io_release):
+ *      * If on-task: task_registered_buffers--
+ *      * If off-task: ref-- via ublk_put_req_ref()
+ *
+ * 3. ublk_sub_req_ref() at IO completion:
+ *    - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers
+ *    - Subtracts sub_refs from ref
+ *    - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task
+ *      buffers that were already counted in task_registered_buffers

I would note that task_registered_buffers is also zeroed during ublk_sub_req_ref(). Effectively, it's transferring/collapsing task_registered_buffers into the atomic ref.

Quoted 4 lines
drivers/block/ublk_drv.c +4 -0
+ *
+ * Example (zero-copy, register on-task, unregister off-task):
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0
+ *   - Register buffer on-task: task_registered_buffers = 1

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1041

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *   - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1
+ *   - Completion via ublk_sub_req_ref():
+ *     sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0
+ *
+ * Example (auto buffer registration):
+ *   Auto buffer registration sets task_registered_buffers = 1 at dispatch.
+ *
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1
+ *   - Buffer unregister: task_registered_buffers-- (becomes 0)
+ *   - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0

I think the order here was actually reversed in commit b749965edda8 "ublk: remove ublk_commit_and_fetch()". ublk_need_complete_req() will call ublk_sub_req_ref(), and io_buffer_unregister_bvec() is called *afterwards*, which results in a call to ublk_io_release(). So the buffer unregister will end up performing an atomic decrement of ref.

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042

Quoted 1 line
drivers/block/ublk_drv.c +1 -0
+ *   - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT

Both ref and task_registered_buffers are 0 at this point, no?

Best,
Caleb

Quoted 15 lines
drivers/block/ublk_drv.c +9 -1
+ *   - Sum equals UBLK_REFCOUNT_INIT, so no active reference exists
+ *
+ * Batch IO Special Case:
+ * ----------------------
+ * In batch IO mode, io->task is NULL. This means ublk_io_release() always
+ * takes the off-task path (ublk_put_req_ref), decrementing io->ref. The
+ * task_registered_buffers counter still tracks registered buffers for the
+ * invariant check, even though the callback doesn't decrement it.
+ */
 static inline void ublk_init_req_ref(const struct ublk_queue *ubq,
                struct ublk_io *io)
 {
--

2.47.0

author
#4 lore raw

Re: [PATCH 2/2] ublk: document IO reference counting design

Quoted 28 lines · Caleb Sander Mateos
Quoted 27 lines · Ming Lei

Add comprehensive documentation for ublk's split reference counting model (io->ref + io->task_registered_buffers) above ublk_init_req_ref() given this model isn't very straightforward.

---
 drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
drivers/block/ublk_drv.c +9 -0
index 7981decd1cee..91218b78e711 100644
@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
               ublk_dev_support_auto_buf_reg(ub);
 }
 
+/*
+ * ublk IO Reference Counting Design
+ * ==================================
+ *
+ * For user-copy and zero-copy modes, ublk uses a split reference model with
+ * two counters that together track IO lifetime:
+ *
+ *   - io->ref: refcount for off-task buffer registrations and user-copy ops
+ *   - io->task_registered_buffers: count of buffers registered on the IO task
Quoted 8 lines

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000

Quoted 4 lines
drivers/block/ublk_drv.c +4 -0
+ *
+ * Key Invariant:
+ * --------------
+ * The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT

This is only true when UBLK_IO_FLAG_OWNED_BY_SRV is set. If the ublk_io isn't currently dispatched to the ublk server, ref and task_registered_buffers would both be 0.

Yeah, it also means the reference counter is initialized, which is implied in the whole doc.

Quoted 12 lines

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010

Quoted 11 lines
drivers/block/ublk_drv.c +10 -0
+ * when no active references exist. This invariant is checked by
+ * ublk_check_and_reset_active_ref() during daemon exit to determine if all
+ * references have been released.
+ *
+ * Why Split Counters:
+ * -------------------
+ * Buffers registered on the IO daemon task can use the lightweight
+ * task_registered_buffers counter (simple increment/decrement) instead of
+ * atomic refcount operations. The ublk_io_release() callback checks if
+ * current == io->task to decide which counter to update.
Quoted 4 lines

However, this optimization can only be used before the I/O is completed, since task_registered_buffers is collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.

The point is ublk_need_complete_req()/ublk_sub_req_ref(), we can make it explicit.

Quoted 19 lines
Quoted 8 lines
drivers/block/ublk_drv.c +7 -0
+ *
+ * Reference Lifecycle:
+ * --------------------
+ * 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch
+ *
+ * 2. During IO processing:
+ *    - On-task buffer reg: task_registered_buffers++ (no ref change)

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *    - Off-task buffer reg: ref++ via ublk_get_req_ref()
+ *    - Buffer unregister callback (ublk_io_release):
+ *      * If on-task: task_registered_buffers--
+ *      * If off-task: ref-- via ublk_put_req_ref()
+ *
+ * 3. ublk_sub_req_ref() at IO completion:
+ *    - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers
+ *    - Subtracts sub_refs from ref
+ *    - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task
+ *      buffers that were already counted in task_registered_buffers
Quoted 3 lines

I would note that task_registered_buffers is also zeroed during ublk_sub_req_ref(). Effectively, it's transferring/collapsing task_registered_buffers into the atomic ref.

OK.

Quoted 16 lines
Quoted 5 lines
drivers/block/ublk_drv.c +4 -0
+ *
+ * Example (zero-copy, register on-task, unregister off-task):
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0
+ *   - Register buffer on-task: task_registered_buffers = 1

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1041

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *   - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1
+ *   - Completion via ublk_sub_req_ref():
+ *     sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0
+ *
+ * Example (auto buffer registration):
+ *   Auto buffer registration sets task_registered_buffers = 1 at dispatch.
+ *
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1
+ *   - Buffer unregister: task_registered_buffers-- (becomes 0)
+ *   - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0
Quoted 4 lines

I think the order here was actually reversed in commit b749965edda8 "ublk: remove ublk_commit_and_fetch()". ublk_need_complete_req() will call ublk_sub_req_ref(), and io_buffer_unregister_bvec() is called *afterwards*, which results in a call to ublk_io_release(). So the

Right.

Quoted 5 lines

buffer unregister will end up performing an atomic decrement of ref.

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042

Quoted 1 line
drivers/block/ublk_drv.c +1 -0
+ *   - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT

Both ref and task_registered_buffers are 0 at this point, no?

It can be UBLK_REFCOUNT_INIT or zero, please see ublk_check_and_reset_active_ref(), the doc mentions Daemon exit check. But yes, both reset finally when we confirm that the io hasn't active references.

Thanks,
Ming

#5 lore raw

Re: [PATCH 2/2] ublk: document IO reference counting design

Quoted 39 lines · Ming Lei
Quoted 28 lines · Caleb Sander Mateos
Quoted 27 lines · Ming Lei

Add comprehensive documentation for ublk's split reference counting model (io->ref + io->task_registered_buffers) above ublk_init_req_ref() given this model isn't very straightforward.

---
 drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
drivers/block/ublk_drv.c +9 -0
index 7981decd1cee..91218b78e711 100644
@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
               ublk_dev_support_auto_buf_reg(ub);
 }
 
+/*
+ * ublk IO Reference Counting Design
+ * ==================================
+ *
+ * For user-copy and zero-copy modes, ublk uses a split reference model with
+ * two counters that together track IO lifetime:
+ *
+ *   - io->ref: refcount for off-task buffer registrations and user-copy ops
+ *   - io->task_registered_buffers: count of buffers registered on the IO task
Quoted 8 lines

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000

Quoted 4 lines
drivers/block/ublk_drv.c +4 -0
+ *
+ * Key Invariant:
+ * --------------
+ * The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT

This is only true when UBLK_IO_FLAG_OWNED_BY_SRV is set. If the ublk_io isn't currently dispatched to the ublk server, ref and task_registered_buffers would both be 0.

Quoted 2 lines

Yeah, it also means the reference counter is initialized, which is implied in the whole doc.

Would you mind mentioning explicitly that both reference counts are 0 for I/Os not currently dispatched to the ublk server?

Quoted 62 lines
Quoted 13 lines

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010

Quoted 12 lines
drivers/block/ublk_drv.c +10 -0
+ * when no active references exist. This invariant is checked by
+ * ublk_check_and_reset_active_ref() during daemon exit to determine if all
+ * references have been released.
+ *
+ * Why Split Counters:
+ * -------------------
+ * Buffers registered on the IO daemon task can use the lightweight
+ * task_registered_buffers counter (simple increment/decrement) instead of
+ * atomic refcount operations. The ublk_io_release() callback checks if
+ * current == io->task to decide which counter to update.
Quoted 4 lines

However, this optimization can only be used before the I/O is completed, since task_registered_buffers is collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.

The point is ublk_need_complete_req()/ublk_sub_req_ref(), we can make it explicit.

Quoted 19 lines
Quoted 8 lines
drivers/block/ublk_drv.c +7 -0
+ *
+ * Reference Lifecycle:
+ * --------------------
+ * 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch
+ *
+ * 2. During IO processing:
+ *    - On-task buffer reg: task_registered_buffers++ (no ref change)

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *    - Off-task buffer reg: ref++ via ublk_get_req_ref()
+ *    - Buffer unregister callback (ublk_io_release):
+ *      * If on-task: task_registered_buffers--
+ *      * If off-task: ref-- via ublk_put_req_ref()
+ *
+ * 3. ublk_sub_req_ref() at IO completion:
+ *    - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers
+ *    - Subtracts sub_refs from ref
+ *    - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task
+ *      buffers that were already counted in task_registered_buffers
Quoted 3 lines

I would note that task_registered_buffers is also zeroed during ublk_sub_req_ref(). Effectively, it's transferring/collapsing task_registered_buffers into the atomic ref.

OK.

Quoted 16 lines
Quoted 15 lines
drivers/block/ublk_drv.c +14 -0
+ *
+ * Example (zero-copy, register on-task, unregister off-task):
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0
+ *   - Register buffer on-task: task_registered_buffers = 1
+ *   - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1
+ *   - Completion via ublk_sub_req_ref():
+ *     sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0
+ *
+ * Example (auto buffer registration):
+ *   Auto buffer registration sets task_registered_buffers = 1 at dispatch.
+ *
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1
+ *   - Buffer unregister: task_registered_buffers-- (becomes 0)
+ *   - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0
Quoted 6 lines
Quoted 4 lines

I think the order here was actually reversed in commit b749965edda8 "ublk: remove ublk_commit_and_fetch()". ublk_need_complete_req() will call ublk_sub_req_ref(), and io_buffer_unregister_bvec() is called *afterwards*, which results in a call to ublk_io_release(). So the

Right.

Should I send out a patch to restore the original order to recover the non-atomic refcount optimization?

Quoted 10 lines
Quoted 6 lines

buffer unregister will end up performing an atomic decrement of ref.

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042

Quoted 1 line
drivers/block/ublk_drv.c +1 -0
+ *   - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT

Both ref and task_registered_buffers are 0 at this point, no?

It can be UBLK_REFCOUNT_INIT or zero, please see ublk_check_and_reset_active_ref(), the doc mentions Daemon exit check. But yes, both reset finally when we confirm that the io hasn't active references.

I mainly found it confusing that "Daemon exit check" is listed under "Example (auto buffer registration)". That makes it sounds like it's a continuation of the steps above, in which case ref and task_registered_buffers should be 0 at this point. Consider separating the part about the daemon exit check into its own section?

Best,
Caleb

author
#6 lore raw

Re: [PATCH 2/2] ublk: document IO reference counting design

Quoted 40 lines · Caleb Sander Mateos
Quoted 39 lines · Ming Lei
Quoted 36 lines · Caleb Sander Mateos
Quoted 21 lines · Ming Lei

Add comprehensive documentation for ublk's split reference counting model (io->ref + io->task_registered_buffers) above ublk_init_req_ref() given this model isn't very straightforward.

---
 drivers/block/ublk_drv.c | 64 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
drivers/block/ublk_drv.c +3 -0
index 7981decd1cee..91218b78e711 100644
@@ -985,6 +985,70 @@ static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub)
               ublk_dev_support_auto_buf_reg(ub);
 }
 
+/*
+ * ublk IO Reference Counting Design
+ * ==================================

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1000

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *
+ * For user-copy and zero-copy modes, ublk uses a split reference model with
+ * two counters that together track IO lifetime:
+ *
+ *   - io->ref: refcount for off-task buffer registrations and user-copy ops
+ *   - io->task_registered_buffers: count of buffers registered on the IO task
+ *
+ * Key Invariant:
+ * --------------
+ * The sum (io->ref + io->task_registered_buffers) must equal UBLK_REFCOUNT_INIT

This is only true when UBLK_IO_FLAG_OWNED_BY_SRV is set. If the ublk_io isn't currently dispatched to the ublk server, ref and task_registered_buffers would both be 0.

Quoted 5 lines
Quoted 2 lines

Yeah, it also means the reference counter is initialized, which is implied in the whole doc.

Would you mind mentioning explicitly that both reference counts are 0 for I/Os not currently dispatched to the ublk server?

Sure.

Quoted 68 lines
Quoted 68 lines
Quoted 14 lines

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1010

Quoted 13 lines
drivers/block/ublk_drv.c +10 -0
+ * when no active references exist. This invariant is checked by
+ * ublk_check_and_reset_active_ref() during daemon exit to determine if all
+ * references have been released.
+ *
+ * Why Split Counters:
+ * -------------------
+ * Buffers registered on the IO daemon task can use the lightweight
+ * task_registered_buffers counter (simple increment/decrement) instead of
+ * atomic refcount operations. The ublk_io_release() callback checks if
+ * current == io->task to decide which counter to update.
Quoted 4 lines

However, this optimization can only be used before the I/O is completed, since task_registered_buffers is collapsed into the atomic ref at that point. All subsequent buffer unregistrations need to use the atomic ref since they may be releasing the last reference.

The point is ublk_need_complete_req()/ublk_sub_req_ref(), we can make it explicit.

Quoted 19 lines
Quoted 8 lines
drivers/block/ublk_drv.c +7 -0
+ *
+ * Reference Lifecycle:
+ * --------------------
+ * 1. ublk_init_req_ref(): Sets io->ref = UBLK_REFCOUNT_INIT at IO dispatch
+ *
+ * 2. During IO processing:
+ *    - On-task buffer reg: task_registered_buffers++ (no ref change)

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1027

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *    - Off-task buffer reg: ref++ via ublk_get_req_ref()
+ *    - Buffer unregister callback (ublk_io_release):
+ *      * If on-task: task_registered_buffers--
+ *      * If off-task: ref-- via ublk_put_req_ref()
+ *
+ * 3. ublk_sub_req_ref() at IO completion:
+ *    - Computes: sub_refs = UBLK_REFCOUNT_INIT - task_registered_buffers
+ *    - Subtracts sub_refs from ref
+ *    - This accounts for the initial UBLK_REFCOUNT_INIT minus any on-task
+ *      buffers that were already counted in task_registered_buffers
Quoted 3 lines

I would note that task_registered_buffers is also zeroed during ublk_sub_req_ref(). Effectively, it's transferring/collapsing task_registered_buffers into the atomic ref.

OK.

Quoted 20 lines
Quoted 5 lines
drivers/block/ublk_drv.c +4 -0
+ *
+ * Example (zero-copy, register on-task, unregister off-task):
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 0
+ *   - Register buffer on-task: task_registered_buffers = 1

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1041

Quoted 10 lines
drivers/block/ublk_drv.c +10 -0
+ *   - Unregister off-task: ref-- (UBLK_REFCOUNT_INIT - 1), task_registered_buffers stays 1
+ *   - Completion via ublk_sub_req_ref():
+ *     sub_refs = UBLK_REFCOUNT_INIT - 1, ref = (UBLK_REFCOUNT_INIT - 1) - (UBLK_REFCOUNT_INIT - 1) = 0
+ *
+ * Example (auto buffer registration):
+ *   Auto buffer registration sets task_registered_buffers = 1 at dispatch.
+ *
+ *   - Dispatch: ref = UBLK_REFCOUNT_INIT, task_registered_buffers = 1
+ *   - Buffer unregister: task_registered_buffers-- (becomes 0)
+ *   - Completion via ublk_sub_req_ref(): sub_refs = UBLK_REFCOUNT_INIT - 0, ref becomes 0

I think the order here was actually reversed in commit b749965edda8 "ublk: remove ublk_commit_and_fetch()". ublk_need_complete_req() will call ublk_sub_req_ref(), and io_buffer_unregister_bvec() is called *afterwards*, which results in a call to ublk_io_release(). So the

Quoted 4 lines
Quoted 1 line

Right.

Should I send out a patch to restore the original order to recover the non-atomic refcount optimization?

Yeah, I think it is good to make the optimization back.

Quoted 12 lines
Quoted 11 lines
Quoted 7 lines

buffer unregister will end up performing an atomic decrement of ref.

drivers/block/ublk_drv.c static inline bool ublk_dev_need_req_ref(const struct ublk_device *ub) · line 1042

Quoted 1 line
drivers/block/ublk_drv.c +1 -0
+ *   - Daemon exit check: sum = ref + task_registered_buffers = UBLK_REFCOUNT_INIT

Both ref and task_registered_buffers are 0 at this point, no?

It can be UBLK_REFCOUNT_INIT or zero, please see ublk_check_and_reset_active_ref(), the doc mentions Daemon exit check. But yes, both reset finally when we confirm that the io hasn't active references.

Quoted 5 lines

I mainly found it confusing that "Daemon exit check" is listed under "Example (auto buffer registration)". That makes it sounds like it's a continuation of the steps above, in which case ref and task_registered_buffers should be 0 at this point. Consider separating the part about the daemon exit check into its own section?

OK.

Also both may not be zero in Daemon exit check when the server is killed, which should be added as example too.

Thanks,
Ming

maintainer Applied
#7 lore raw
In reply to Ming Lei
Quoted 9 lines · Ming Lei

The 1st patch improves auto buffer helpers naming, and improve documentation.

The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.

Thanks,

[...]

Applied, thanks!

[1/2] ublk: rename auto buffer registration helpers

      commit: f50af896932f5edb1ff7b407753ecfa285c30b7a
[2/2] ublk: document IO reference counting design
      commit: 420bcbf67b4575d6455a4fa0de06695d98d340b1

Best regards,

Signature · 1 line
Jens Axboe
author
#8 lore raw
Quoted 10 lines · Jens Axboe
Quoted 8 lines · Ming Lei

The 1st patch improves auto buffer helpers naming, and improve documentation.

The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.

Thanks,

Quoted 8 lines
Quoted 1 line

[...]

Applied, thanks!

[1/2] ublk: rename auto buffer registration helpers

      commit: f50af896932f5edb1ff7b407753ecfa285c30b7a
[2/2] ublk: document IO reference counting design
      commit: 420bcbf67b4575d6455a4fa0de06695d98d340b1

Hi Jens,

I'd address Caleb's comment on the 2nd patch, so can you drop it?

Thanks,
Ming

maintainer
#9 lore raw
Quoted 15 lines · Ming Lei
Quoted 14 lines · Jens Axboe
Quoted 9 lines · Ming Lei

The 1st patch improves auto buffer helpers naming, and improve documentation.

The 2nd patch documents ublk io reference counting design, given it isn't very straightforward.

Thanks,

[...]

Applied, thanks!

Quoted 8 lines
Quoted 4 lines

[1/2] ublk: rename auto buffer registration helpers

      commit: f50af896932f5edb1ff7b407753ecfa285c30b7a
[2/2] ublk: document IO reference counting design
      commit: 420bcbf67b4575d6455a4fa0de06695d98d340b1

Hi Jens,

I'd address Caleb's comment on the 2nd patch, so can you drop it?

Dropped

Signature · 1 line
Jens Axboe