io_uring: add async data clear/free helpers
Jens Axboe
maintainerPATCH 1/1 io_uring: add async data clear/free helpers
Futex recently had an issue where it mishandled how ->async_data and REQ_F_ASYNC_DATA is handled. To avoid future issues like that, add a set of helpers that either clear or clear-and-free the async data assigned to a struct io_kiocb.
Convert existing manual handling of that to use the helpers. No intended functional changes in this patch.
---
io_uring/futex.c +4 -9
@@ -43,7 +43,6 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)static void __io_futex_complete (struct io_kiocb *req,io_tw_token_t tw) { - req->async_data =NULL ;hlist_del_init (&req->hash_node);io_req_task_complete (req, tw); }
@@ -54,6 +53,7 @@ static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)io_tw_lock (ctx, tw);io_cache_free (&ctx->futex_cache, req->async_data); +io_req_async_data_clear (req,0 );__io_futex_complete (req, tw); }
@@ -72,8 +72,7 @@ static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)io_req_set_res (req, res,0 ); } -kfree (req->async_data); - req->flags &= ~REQ_F_ASYNC_DATA ; +io_req_async_data_free (req);__io_futex_complete (req, tw); }
@@ -232,9 +231,7 @@ int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)io_ring_submit_unlock (ctx, issue_flags);req_set_fail (req);io_req_set_res (req, ret,0 ); -kfree (futexv); - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; +io_req_async_data_free (req);return IOU_COMPLETE ; }
@@ -310,9 +307,7 @@ int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)if (ret <0 )req_set_fail (req);io_req_set_res (req, ret,0 ); - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; -kfree (ifd); +io_req_async_data_free (req);return IOU_COMPLETE ; }
io_uring/io_uring.h +13 -0
@@ -281,6 +281,19 @@ static inline bool req_has_async_data(struct io_kiocb *req)return req->flags &REQ_F_ASYNC_DATA ; } +static inline void io_req_async_data_clear (struct io_kiocb *req, +io_req_flags_t extra_flags) +{ + req->flags &= ~(REQ_F_ASYNC_DATA |extra_flags); + req->async_data =NULL ; +} + +static inline void io_req_async_data_free (struct io_kiocb *req) +{ +kfree (req->async_data); +io_req_async_data_clear (req,0 ); +} +static inline void io_put_file (struct io_kiocb *req) {if (!(req->flags &REQ_F_FIXED_FILE ) && req->file)
io_uring/net.c +2 -4
@@ -178,10 +178,8 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)if (hdr->vec.nr >IO_VEC_CACHE_SOFT_CAP )io_vec_free (&hdr->vec); -if (io_alloc_cache_put (&req->ctx->netmsg_cache, hdr)) { - req->async_data =NULL ; - req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); - } +if (io_alloc_cache_put (&req->ctx->netmsg_cache, hdr)) +io_req_async_data_clear (req,REQ_F_NEED_CLEANUP ); }static struct io_async_msghdr *io_msg_alloc_async (struct io_kiocb *req)
io_uring/rw.c +2 -4
@@ -154,10 +154,8 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)if (rw->vec.nr >IO_VEC_CACHE_SOFT_CAP )io_vec_free (&rw->vec); -if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) { - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; - } +if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) +io_req_async_data_clear (req,0 ); }static void io_req_rw_cleanup (struct io_kiocb *req,unsigned int issue_flags)
io_uring/uring_cmd.c +1 -2
@@ -37,8 +37,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)if (io_alloc_cache_put (&req->ctx->cmd_cache, ac)) { ioucmd->sqe =NULL ; - req->async_data =NULL ; - req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); +io_req_async_data_clear (req,0 ); } }
io_uring/waitid.c +1 -3
@@ -37,9 +37,7 @@ static void io_waitid_free(struct io_kiocb *req)struct io_waitid_async *iwa = req->async_data;put_pid (iwa->wo.wo_pid); -kfree (req->async_data); - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; +io_req_async_data_free (req); }static bool io_waitid_compat_copy_si (struct io_waitid *iw,int signo)
Signature · 1 line
Jens Axboe
Quoted 75 lines · Jens Axboe
Futex recently had an issue where it mishandled how ->async_data and
REQ_F_ASYNC_DATAis handled. To avoid future issues like that, add a set of helpers that either clear or clear-and-free the async data assigned to a structio_kiocb.Convert existing manual handling of that to use the helpers. No intended functional changes in this patch.
Signed-off-by Jens Axboe <[email protected]>---
io_uring/futex.c +4
-9@@ -43,7 +43,6 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)static void __io_futex_complete (struct io_kiocb *req,io_tw_token_t tw) { - req->async_data =NULL ;hlist_del_init (&req->hash_node);io_req_task_complete (req, tw); }@@ -54,6 +53,7 @@ static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)io_tw_lock (ctx, tw);io_cache_free (&ctx->futex_cache, req->async_data); +io_req_async_data_clear (req,0 );__io_futex_complete (req, tw); }@@ -72,8 +72,7 @@ static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)io_req_set_res (req, res,0 ); } -kfree (req->async_data); - req->flags &= ~REQ_F_ASYNC_DATA ; +io_req_async_data_free (req);__io_futex_complete (req, tw); }@@ -232,9 +231,7 @@ int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags)io_ring_submit_unlock (ctx, issue_flags);req_set_fail (req);io_req_set_res (req, ret,0 ); -kfree (futexv); - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; +io_req_async_data_free (req);return IOU_COMPLETE ; }@@ -310,9 +307,7 @@ int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags)if (ret <0 )req_set_fail (req);io_req_set_res (req, ret,0 ); - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; -kfree (ifd); +io_req_async_data_free (req);return IOU_COMPLETE ; }io_uring/io_uring.h +2
-0@@ -281,6 +281,19 @@ static inline bool req_has_async_data(struct io_kiocb *req)return req->flags &REQ_F_ASYNC_DATA ; } +static inline void io_req_async_data_clear (struct io_kiocb *req, +io_req_flags_t extra_flags)
io_uring/io_uring.h static inline bool req_has_async_data(struct io_kiocb *req) · line 295
Quoted 10 lines
io_uring/io_uring.h +10
-0+{ + req->flags &= ~(REQ_F_ASYNC_DATA |extra_flags); + req->async_data =NULL ; +} + +static inline void io_req_async_data_free (struct io_kiocb *req) +{ +kfree (req->async_data); +io_req_async_data_clear (req,0 ); +}
Would it make sense to also add a helper for assigning async_data that would also make sure to set REQ_F_ASYNC_DATA?
Quoted 43 lines
io_uring/io_uring.h +1
-0+static inline void io_put_file (struct io_kiocb *req) {if (!(req->flags &REQ_F_FIXED_FILE ) && req->file)io_uring/net.c +2
-4@@ -178,10 +178,8 @@ static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)if (hdr->vec.nr >IO_VEC_CACHE_SOFT_CAP )io_vec_free (&hdr->vec); -if (io_alloc_cache_put (&req->ctx->netmsg_cache, hdr)) { - req->async_data =NULL ; - req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); - } +if (io_alloc_cache_put (&req->ctx->netmsg_cache, hdr)) +io_req_async_data_clear (req,REQ_F_NEED_CLEANUP ); }static struct io_async_msghdr *io_msg_alloc_async (struct io_kiocb *req)io_uring/rw.c +2
-4@@ -154,10 +154,8 @@ static void io_rw_recycle(struct io_kiocb *req, unsigned int issue_flags)if (rw->vec.nr >IO_VEC_CACHE_SOFT_CAP )io_vec_free (&rw->vec); -if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) { - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; - } +if (io_alloc_cache_put (&req->ctx->rw_cache, rw)) +io_req_async_data_clear (req,0 ); }static void io_req_rw_cleanup (struct io_kiocb *req,unsigned int issue_flags)io_uring/uring_cmd.c +0
-0
io_uring/uring_cmd.c static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) · line 40
Quoted 6 lines
io_uring/uring_cmd.c +1
-2@@ -37,8 +37,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)if (io_alloc_cache_put (&req->ctx->cmd_cache, ac)) { ioucmd->sqe =NULL ; - req->async_data =NULL ; - req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); +io_req_async_data_clear (req,0 );
Looks like the REQ_F_NEED_CLEANUP got lost here. Other than that,
Quoted 23 lines
io_uring/uring_cmd.c +0
-0} }io_uring/waitid.c +1
-4@@ -37,9 +37,7 @@ static void io_waitid_free(struct io_kiocb *req)struct io_waitid_async *iwa = req->async_data;put_pid (iwa->wo.wo_pid); -kfree (req->async_data); - req->async_data =NULL ; - req->flags &= ~REQ_F_ASYNC_DATA ; +io_req_async_data_free (req); }static bool io_waitid_compat_copy_si (struct io_waitid *iw,int signo) --Jens Axboe
Jens Axboe
maintainerQuoted 8 lines · Caleb Sander Mateos
Quoted 8 lines
io_uring/io_uring.h +0
-0@@ -281,6 +281,19 @@ static inline bool req_has_async_data(struct io_kiocb *req)return req->flags &REQ_F_ASYNC_DATA ; }
Quoted 15 lines
io_uring/io_uring.h static inline bool req_has_async_data(struct io_kiocb *req) · line 295
Quoted 12 lines
io_uring/io_uring.h +12
-0+static inline void io_req_async_data_clear (struct io_kiocb *req, +io_req_flags_t extra_flags) +{ + req->flags &= ~(REQ_F_ASYNC_DATA |extra_flags); + req->async_data =NULL ; +} + +static inline void io_req_async_data_free (struct io_kiocb *req) +{ +kfree (req->async_data); +io_req_async_data_clear (req,0 ); +}Would it make sense to also add a helper for assigning
async_datathat would also make sure to setREQ_F_ASYNC_DATA?
I did consider that, but it's only futex that'd use it and just in those two spots. So decided against it. At least for now.
Quoted 6 lines
Quoted 6 lines
io_uring/uring_cmd.c +0
-0@@ -37,8 +37,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
Quoted 7 lines
io_uring/uring_cmd.c static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) · line 40
Quoted 5 lines
io_uring/uring_cmd.c +1
-2if (io_alloc_cache_put (&req->ctx->cmd_cache, ac)) { ioucmd->sqe =NULL ; - req->async_data =NULL ; - req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); +io_req_async_data_clear (req,0 );Looks like the
REQ_F_NEED_CLEANUPgot lost here. Other than that,
Oops indeed, added back.
Quoted 1 line
Reviewed-by Caleb Sander Mateos <[email protected]>
Thanks for the review!
Signature · 1 line
Jens Axboe
Quoted 26 lines · Jens Axboe
Quoted 8 lines · Caleb Sander Mateos
Quoted 8 lines
io_uring/io_uring.h +0
-0@@ -281,6 +281,19 @@ static inline bool req_has_async_data(struct io_kiocb *req)return req->flags &REQ_F_ASYNC_DATA ; }Quoted 15 lines
io_uring/io_uring.h static inline bool req_has_async_data(struct io_kiocb *req) · line 295
Quoted 12 lines
io_uring/io_uring.h +12
-0+static inline void io_req_async_data_clear (struct io_kiocb *req, +io_req_flags_t extra_flags) +{ + req->flags &= ~(REQ_F_ASYNC_DATA |extra_flags); + req->async_data =NULL ; +} + +static inline void io_req_async_data_free (struct io_kiocb *req) +{ +kfree (req->async_data); +io_req_async_data_clear (req,0 ); +}Would it make sense to also add a helper for assigning
async_datathat would also make sure to setREQ_F_ASYNC_DATA?
Quoted 2 lines
I did consider that, but it's only futex that'd use it and just in those two spots. So decided against it. At least for now.
Makes sense. I think it could also be used in io_uring_alloc_async_data(), but it's true there are fewer sites assigning to async_data than clearing it.
Best,
Caleb
Quoted 23 lines
Quoted 7 lines
Quoted 7 lines
io_uring/uring_cmd.c +0
-0@@ -37,8 +37,7 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)Quoted 7 lines
io_uring/uring_cmd.c static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) · line 40
Quoted 5 lines
io_uring/uring_cmd.c +1
-2if (io_alloc_cache_put (&req->ctx->cmd_cache, ac)) { ioucmd->sqe =NULL ; - req->async_data =NULL ; - req->flags &= ~(REQ_F_ASYNC_DATA |REQ_F_NEED_CLEANUP ); +io_req_async_data_clear (req,0 );Looks like the
REQ_F_NEED_CLEANUPgot lost here. Other than that,Oops indeed, added back.
Quoted 1 line
Reviewed-by Caleb Sander Mateos <[email protected]>Thanks for the review!
--
Jens Axboe
Jens Axboe
maintainerQuoted 27 lines · Caleb Sander Mateos
Quoted 26 lines · Jens Axboe
Quoted 23 lines · Caleb Sander Mateos
Quoted 10 lines
io_uring/io_uring.h +2
-0@@ -281,6 +281,19 @@ static inline bool req_has_async_data(struct io_kiocb *req)return req->flags &REQ_F_ASYNC_DATA ; } +static inline void io_req_async_data_clear (struct io_kiocb *req, +io_req_flags_t extra_flags)io_uring/io_uring.h static inline bool req_has_async_data(struct io_kiocb *req) · line 295
Quoted 10 lines
io_uring/io_uring.h +10
-0+{ + req->flags &= ~(REQ_F_ASYNC_DATA |extra_flags); + req->async_data =NULL ; +} + +static inline void io_req_async_data_free (struct io_kiocb *req) +{ +kfree (req->async_data); +io_req_async_data_clear (req,0 ); +}Would it make sense to also add a helper for assigning
async_datathat would also make sure to setREQ_F_ASYNC_DATA?
Quoted 6 lines
Quoted 2 lines
I did consider that, but it's only futex that'd use it and just in those two spots. So decided against it. At least for now.
Makes sense. I think it could also be used in
io_uring_alloc_async_data(), but it's true there are fewer sites assigning toasync_datathan clearing it.
Indeed, most of the setting happens through the alloc async data helper already. And yes it could get unified with that, but then we'd probably want to extend that a bit more to return -ENOMEM too for the cached case, and futex has one of each. IOW, I just stopped at that point :-)
Signature · 1 line
Jens Axboe