io_uring/rsrc: add huge page accounting for registered buffers
Jens Axboe
maintainerPATCH v4 1/1 io_uring/rsrc: add huge page accounting for registered buffers
Track huge page references in a per-ring xarray to prevent double accounting when the same huge page is used by multiple registered buffers, either within the same ring or across cloned rings.
When registering buffers backed by huge pages, we need to account for RLIMIT_MEMLOCK. But if multiple buffers share the same huge page (common with cloned buffers), we must not account for the same page multiple times. Similarly, we must only unaccount when the last reference to a huge page is released.
Maintain a per-ring xarray (hpage_acct) that tracks reference counts for each huge page. When registering a buffer, for each unique huge page, increment its accounting reference count, and only account pages that are newly added.
When unregistering a buffer, for each unique huge page, decrement its refcount. Once the refcount hits zero, the page is unaccounted.
Note: any account is done against the ctx->user that was assigned when the ring was setup. As before, if root is running the operation, no accounting is done.
With these changes, any use of imu->acct_pages is also dead, hence kill it from struct io_mapped_ubuf. This shrinks it from 56b to 48b on a 64-bit arch. Additionally, hpage_already_acct() is gone, which was an O(M*M) scan over current + previous registrations.
---
Changes since v3:
- Code cleanups based on feedback from Clément
include/linux/io_uring_types.h +3 -0
@@ -446,6 +446,9 @@ struct io_ring_ctx {/* Stores zcrx object pointers of type struct io_zcrx_ifq */ struct xarray zcrx_ctxs; +/* Used for accounting references on pages in registered buffers */ +struct xarray hpage_acct; +u32 pers_next;struct xarray personalities;
io_uring/io_uring.c +3 -0
@@ -233,6 +233,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)return NULL ;xa_init (&ctx->io_bl_xa); +xa_init (&ctx->hpage_acct);/* * Use 5 bits less than the max cq entries, that should give us around
@@ -302,6 +303,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)io_free_alloc_caches (ctx);kvfree (ctx->cancel_table.hbs);xa_destroy (&ctx->io_bl_xa); +xa_destroy (&ctx->hpage_acct);kfree (ctx);return NULL ; }
@@ -2198,6 +2200,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)io_napi_free (ctx);kvfree (ctx->cancel_table.hbs);xa_destroy (&ctx->io_bl_xa); +xa_destroy (&ctx->hpage_acct);kfree (ctx); }
io_uring/rsrc.c +200 -60
360 lines not on this page · raw message
io_uring/rsrc.h +0 -1
@@ -38,7 +38,6 @@ struct io_mapped_ubuf {unsigned int nr_bvecs;unsigned int folio_shift;refcount_t refs; -unsigned long acct_pages;void (*release)(void *);void *priv;u8 flags;
Signature · 1 line
Jens Axboe