io_uring/zc fix and improvements
cover io_uring/zc fix and improvements
It mainly fixes net/zc memory accounting error handling and then makes it more consistent with registered buffers mem accounting.
Pavel Begunkov (4):
io_uring/net: improve io_get_notif_slot types io_uring/net: checks errors of zc mem accounting io_uring/net: make page accounting more consistent io_uring/net: use unsigned for flags
io_uring/net.c | 8 +++++--- io_uring/notif.c | 9 ++++----- io_uring/notif.h | 21 ++++++++++++++++++++- io_uring/rsrc.c | 12 ++++-------- io_uring/rsrc.h | 9 +++++++++ 5 files changed, 42 insertions(+), 17 deletions(-)
Signature · 1 line
2.37.0
PATCH 1/4 io_uring/net: improve io_get_notif_slot types
Don't use signed int for slot indexing.
--- io_uring/notif.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/notif.h +1 -1
@@ -67,7 +67,7 @@ static inline struct io_notif *io_get_notif(struct io_ring_ctx *ctx, }static inline struct io_notif_slot *io_get_notif_slot (struct io_ring_ctx *ctx, -int idx) +unsigned idx) __must_hold(&ctx->uring_lock) {if (idx >= ctx->nr_notif_slots)
Signature · 1 line
2.37.0
PATCH 2/4 io_uring/net: checks errors of zc mem accounting
In reply to Pavel Begunkovmm_account_pinned_pages() may fail, don't ignore the return value.
--- io_uring/net.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
io_uring/net.c +3 -1
@@ -985,7 +985,9 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) &msg.msg_iter);if (unlikely (ret))return ret; -mm_account_pinned_pages (¬if->uarg.mmp, zc->len); + ret =mm_account_pinned_pages (¬if->uarg.mmp, zc->len); +if (unlikely (ret)) +return ret; }if (zc->addr) {
Signature · 1 line
2.37.0
PATCH 3/4 io_uring/net: make page accounting more consistent
In reply to Pavel BegunkovMake network page accounting more consistent with how buffer registration is working, i.e. account all memory to ctx->user.
--- io_uring/net.c | 2 +- io_uring/notif.c | 9 ++++----- io_uring/notif.h | 19 +++++++++++++++++++ io_uring/rsrc.c | 12 ++++-------- io_uring/rsrc.h | 9 +++++++++ 5 files changed, 37 insertions(+), 14 deletions(-)
io_uring/net.c +1 -1
@@ -985,7 +985,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) &msg.msg_iter);if (unlikely (ret))return ret; - ret =mm_account_pinned_pages (¬if->uarg.mmp, zc->len); + ret =io_notif_account_mem (notif, zc->len);if (unlikely (ret))return ret; }
io_uring/notif.c +4 -5
@@ -14,12 +14,10 @@ static void __io_notif_complete_tw(struct callback_head *cb) struct io_notif *notif = container_of(cb, struct io_notif, task_work);struct io_rsrc_node *rsrc_node = notif->rsrc_node;struct io_ring_ctx *ctx = notif->ctx; -struct mmpin *mmp = ¬if->uarg.mmp; -if (mmp->user) { -atomic_long_sub (mmp->num_pg, &mmp->user->locked_vm); -free_uid (mmp->user); - mmp->user =NULL ; +if (notif->account_pages && ctx->user) { +__io_unaccount_mem (ctx->user, notif->account_pages); + notif->account_pages =0 ; }if (likely (notif->task)) {io_put_task (notif->task,1 );
@@ -121,6 +119,7 @@ struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, notif->ctx = ctx; notif->uarg.flags =SKBFL_ZEROCOPY_FRAG |SKBFL_DONT_ORPHAN ; notif->uarg.callback = io_uring_tx_zerocopy_callback; + notif->account_pages =0 ; } notif->seq = slot->seq++;
io_uring/notif.h +19 -0
@@ -5,6 +5,8 @@#include <net/sock.h> #include <linux/nospec.h> +#include "rsrc.h" +#define IO_NOTIF_SPLICE_BATCH 32#define IORING_MAX_NOTIF_SLOTS (1U << 10)
@@ -23,6 +25,8 @@ struct io_notif {/* hook into ctx->notif_list and ctx->notif_list_locked */ struct list_head cache_node; +unsigned long account_pages; +union {struct callback_head task_work;struct work_struct commit_work;
@@ -85,3 +89,18 @@ static inline void io_notif_slot_flush_submit(struct io_notif_slot *slot, }io_notif_slot_flush (slot); } + +static inline int io_notif_account_mem (struct io_notif *notif,unsigned len) +{ +struct io_ring_ctx *ctx = notif->ctx; +unsigned nr_pages = (len >>PAGE_SHIFT ) +2 ; +int ret; + +if (ctx->user) { + ret =__io_account_mem (ctx->user, nr_pages); +if (ret) +return ret; + notif->account_pages += nr_pages; + } +return 0 ; +}
io_uring/rsrc.c +4 -8
@@ -44,17 +44,13 @@ void io_rsrc_refs_drop(struct io_ring_ctx *ctx) } } -static inline void __io_unaccount_mem (struct user_struct *user, -unsigned long nr_pages) -{ -atomic_long_sub (nr_pages, &user->locked_vm); -} - -static inline int __io_account_mem (struct user_struct *user, -unsigned long nr_pages) +int __io_account_mem (struct user_struct *user,unsigned long nr_pages) {unsigned long page_limit, cur_pages, new_pages; +if (!nr_pages) +return 0 ; +/* Don't allow more pages than we can safely lock */ page_limit =rlimit (RLIMIT_MEMLOCK ) >>PAGE_SHIFT ;
io_uring/rsrc.h +9 -0
@@ -169,4 +169,13 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)int io_rsrc_update (struct io_kiocb *req,unsigned int issue_flags);int io_rsrc_update_prep (struct io_kiocb *req,const struct io_uring_sqe *sqe); + +int __io_account_mem (struct user_struct *user,unsigned long nr_pages); + +static inline void __io_unaccount_mem (struct user_struct *user, +unsigned long nr_pages) +{ +atomic_long_sub (nr_pages, &user->locked_vm); +} +#endif
Signature · 1 line
2.37.0
PATCH 4/4 io_uring/net: use unsigned for flags
In reply to Pavel BegunkovUse unsigned int type for msg flags.
--- io_uring/net.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
io_uring/net.c +2 -2
@@ -55,10 +55,10 @@ struct io_sr_msg {struct user_msghdr __user *umsg;void __user *buf; }; -int msg_flags; +unsigned msg_flags; +unsigned flags;size_t len;size_t done_io; -unsigned int flags; };struct io_sendzc {
Signature · 1 line
2.37.0
Quoted 10 lines · Pavel Begunkov
It mainly fixes net/zc memory accounting error handling and then makes it more consistent with registered buffers mem accounting.
Pavel Begunkov (4):
io_uring/net: improveio_get_notif_slottypesio_uring/net: checks errors of zc mem accountingio_uring/net: make page accounting more consistentio_uring/net: use unsigned for flags[...]
Applied, thanks!
[1/4] io_uring/net: improve io_get_notif_slot types
commit: cb309ae49da7a7c28f0051deea13970291134fac
[2/4] io_uring/net: checks errors of zc mem accounting
commit: 2e32ba5607ee2b668baa8831dd74f7cc867a1f7e
[3/4] io_uring/net: make page accounting more consistent
commit: 6a9ce66f4d0872861e0bbc67eee6ce5dca5dd886
[4/4] io_uring/net: use unsigned for flags
commit: 293402e564a7391f38541c7694e736f5fde20aeaBest regards,
Signature · 1 line
Jens Axboe