next zcrx cleanups
Subsystems (2)
Pavel Begunkov
maintainercover next zcrx cleanups
Flushing for review some of zcrx cleanups I had for a while now. This includes consolidating dma sync, optimising refilling, and using lock guards.
For a full branch with all relevant patches see
https://github.com/isilence/linux.git zcrx/for-next
Pavel Begunkov (10):
io_uring/zcrx: replace memchar_inv with is_zero io_uring/zcrx: use page_pool_unref_and_test() io_uring/zcrx: remove extra io_zcrx_drop_netdev io_uring/zcrx: rename dma lock
io_uring/zcrx: protect netdev with pp_lock io_uring/zcrx: unify allocation dma sync io_uring/zcrx: reduce netmem scope in refill io_uring/zcrx: use guards for the refill lock io_uring/zcrx: don't adjust free cache space io_uring/zcrx: rely on cache size truncation on refill
io_uring/zcrx.c | 92 ++++++++++++++++++++++--------------------------- io_uring/zcrx.h | 8 +++-- 2 files changed, 48 insertions(+), 52 deletions(-)
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 1/10 io_uring/zcrx: replace memchar_inv with is_zero
memchr_inv() is more ambiguous than mem_is_zero(), so use the latter for zero checks.
--- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/zcrx.c +1 -1
@@ -561,7 +561,7 @@ int io_register_zcrx_ifq(struct io_ring_ctx *ctx,return -EFAULT ;if (copy_from_user (&rd,u64_to_user_ptr (reg.region_ptr),sizeof (rd)))return -EFAULT ; -if (memchr_inv (®.__resv,0 ,sizeof (reg.__resv)) || +if (!mem_is_zero (®.__resv,sizeof (reg.__resv)) || reg.__resv2 || reg.zcrx_id)return -EINVAL ;if (reg.if_rxq ==-1 || !reg.rq_entries || reg.flags)
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 2/10 io_uring/zcrx: use page_pool_unref_and_test()
In reply to Pavel Begunkovpage_pool_unref_and_test() tries to better follow usuall refcount semantics, use it instead of page_pool_unref_netmem().
--- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/zcrx.c +1 -1
@@ -785,7 +785,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp,continue ; netmem =net_iov_to_netmem (niov); -if (page_pool_unref_netmem (netmem,1 ) !=0 ) +if (!page_pool_unref_and_test (netmem))continue ;if (unlikely (niov->pp != pp)) {
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 3/10 io_uring/zcrx: remove extra io_zcrx_drop_netdev
In reply to Pavel Begunkovio_close_queue() already detaches the netdev, don't unnecessary call io_zcrx_drop_netdev() right after.
--- io_uring/zcrx.c | 1 - 1 file changed, 1 deletion(-)
io_uring/zcrx.c +0 -1
@@ -513,7 +513,6 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)static void io_zcrx_ifq_free (struct io_zcrx_ifq *ifq) {io_close_queue (ifq); -io_zcrx_drop_netdev (ifq);if (ifq->area)io_zcrx_free_area (ifq->area);
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 4/10 io_uring/zcrx: rename dma lock
In reply to Pavel BegunkovIn preparation for reusing the lock for other purposes, rename it to "pp_lock". As before, it can be taken deeper inside the networking stack by page pool, and so the syscall io_uring must avoid holding it while doing queue reconfiguration or anything that can result in immediate pp init/destruction.
--- io_uring/zcrx.c | 8 ++++---- io_uring/zcrx.h | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-)
io_uring/zcrx.c +4 -4
@@ -247,7 +247,7 @@ static void io_zcrx_unmap_area(struct io_zcrx_ifq *ifq, {int i; -guard (mutex)(&ifq->dma_lock); +guard (mutex)(&ifq->pp_lock);if (!area->is_mapped)return ; area->is_mapped = false;
@@ -278,7 +278,7 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area) {int ret; -guard (mutex)(&ifq->dma_lock); +guard (mutex)(&ifq->pp_lock);if (area->is_mapped)return 0 ;
@@ -471,7 +471,7 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) ifq->ctx = ctx;spin_lock_init (&ifq->lock);spin_lock_init (&ifq->rq_lock); -mutex_init (&ifq->dma_lock); +mutex_init (&ifq->pp_lock);return ifq; }
@@ -520,7 +520,7 @@ static void io_zcrx_ifq_free(struct io_zcrx_ifq *ifq)put_device (ifq->dev);io_free_rbuf_ring (ifq); -mutex_destroy (&ifq->dma_lock); +mutex_destroy (&ifq->pp_lock);kfree (ifq); }
io_uring/zcrx.h +6 -1
@@ -54,7 +54,12 @@ struct io_zcrx_ifq {struct net_device *netdev;netdevice_tracker netdev_tracker;spinlock_t lock; -struct mutex dma_lock; + +/* +* Page pool and net configuration lock, can be taken deeper in the +* net stack. +*/ +struct mutex pp_lock;struct io_mapped_region region; };
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 5/10 io_uring/zcrx: protect netdev with pp_lock
In reply to Pavel BegunkovRemove ifq->lock and reuse pp_lock to protect the netdev pointer.
--- io_uring/zcrx.c | 23 +++++++++++------------ io_uring/zcrx.h | 1 - 2 files changed, 11 insertions(+), 13 deletions(-)
io_uring/zcrx.c +11 -12
@@ -469,7 +469,6 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx) ifq->if_rxq =-1 ; ifq->ctx = ctx; -spin_lock_init (&ifq->lock);spin_lock_init (&ifq->rq_lock);mutex_init (&ifq->pp_lock);return ifq;
@@ -477,12 +476,12 @@ static struct io_zcrx_ifq *io_zcrx_ifq_alloc(struct io_ring_ctx *ctx)static void io_zcrx_drop_netdev (struct io_zcrx_ifq *ifq) { -spin_lock (&ifq->lock); -if (ifq->netdev) { -netdev_put (ifq->netdev, &ifq->netdev_tracker); - ifq->netdev =NULL ; - } -spin_unlock (&ifq->lock); +guard (mutex)(&ifq->pp_lock); + +if (!ifq->netdev) +return ; +netdev_put (ifq->netdev, &ifq->netdev_tracker); + ifq->netdev =NULL ; }static void io_close_queue (struct io_zcrx_ifq *ifq)
@@ -497,11 +496,11 @@ static void io_close_queue(struct io_zcrx_ifq *ifq)if (ifq->if_rxq ==-1 )return ; -spin_lock (&ifq->lock); - netdev = ifq->netdev; - netdev_tracker = ifq->netdev_tracker; - ifq->netdev =NULL ; -spin_unlock (&ifq->lock); + scoped_guard(mutex, &ifq->pp_lock) { + netdev = ifq->netdev; + netdev_tracker = ifq->netdev_tracker; + ifq->netdev =NULL ; + }if (netdev) {net_mp_close_rxq (netdev, ifq->if_rxq, &p);
io_uring/zcrx.h +0 -1
@@ -53,7 +53,6 @@ struct io_zcrx_ifq {struct device *dev;struct net_device *netdev;netdevice_tracker netdev_tracker; -spinlock_t lock;/* * Page pool and net configuration lock, can be taken deeper in the
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 6/10 io_uring/zcrx: unify allocation dma sync
In reply to Pavel BegunkovFirst, I want niov dma sync'ing during page pool allocation out of spinlocked sections, i.e. rq_lock for ring refilling and freelist_lock for slow path allocation. Move it all to a common section, which can further optimise by checking dma_dev_need_sync() only once per batch.
--- io_uring/zcrx.c | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-)
io_uring/zcrx.c +20 -19
@@ -292,21 +292,6 @@ static int io_zcrx_map_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *area)return ret; } -static void io_zcrx_sync_for_device (const struct page_pool *pool, -struct net_iov *niov) -{ -#if defined(CONFIG_HAS_DMA ) && defined(CONFIG_DMA_NEED_SYNC ) -dma_addr_t dma_addr; - -if (!dma_dev_need_sync (pool->p.dev)) -return ; - - dma_addr =page_pool_get_dma_addr_netmem (net_iov_to_netmem (niov)); -__dma_sync_single_for_device (pool->p.dev, dma_addr + pool->p.offset, -PAGE_SIZE , pool->p.dma_dir); -#endif -} -#define IO_RQ_MAX_ENTRIES 32768#define IO_SKBS_PER_CALL_LIMIT 20
@@ -791,7 +776,6 @@ static void io_zcrx_ring_refill(struct page_pool *pp,continue ; } -io_zcrx_sync_for_device (pp, niov);net_mp_netmem_place_in_cache (pp, netmem); }while (--entries);
@@ -806,15 +790,31 @@ static void io_zcrx_refill_slow(struct page_pool *pp, struct io_zcrx_ifq *ifq)spin_lock_bh (&area->freelist_lock);while (area->free_count && pp->alloc.count <PP_ALLOC_CACHE_REFILL ) {struct net_iov *niov =__io_zcrx_get_free_niov (area); -netmem_ref netmem =net_iov_to_netmem (niov);net_mp_niov_set_page_pool (pp, niov); -io_zcrx_sync_for_device (pp, niov); -net_mp_netmem_place_in_cache (pp, netmem); +net_mp_netmem_place_in_cache (pp,net_iov_to_netmem (niov)); }spin_unlock_bh (&area->freelist_lock); } +static void io_sync_allocated_niovs (struct page_pool *pp) +{ +#if defined(CONFIG_HAS_DMA ) && defined(CONFIG_DMA_NEED_SYNC ) +int i; + +if (!dma_dev_need_sync (pp->p.dev)) +return ; + +for (i =0 ; i < pp->alloc.count; i++) { +netmem_ref netmem = pp->alloc.cache[i]; +dma_addr_t dma_addr =page_pool_get_dma_addr_netmem (netmem); + +__dma_sync_single_for_device (pp->p.dev, dma_addr + pp->p.offset, +PAGE_SIZE , pp->p.dma_dir); + } +#endif +} +static netmem_ref io_pp_zc_alloc_netmems (struct page_pool *pp,gfp_t gfp) {struct io_zcrx_ifq *ifq =io_pp_to_ifq (pp);
@@ -831,6 +831,7 @@ static netmem_ref io_pp_zc_alloc_netmems(struct page_pool *pp, gfp_t gfp)if (!pp->alloc.count)return 0 ; out_return: +io_sync_allocated_niovs (pp);return pp->alloc.cache[--pp->alloc.count]; }
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 7/10 io_uring/zcrx: reduce netmem scope in refill
In reply to Pavel BegunkovReduce the scope of a local var netmem in io_zcrx_ring_refill.
--- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/zcrx.c +1 -1
@@ -735,7 +735,6 @@ static void io_zcrx_ring_refill(struct page_pool *pp, {unsigned int mask = ifq->rq_entries -1 ;unsigned int entries; -netmem_ref netmem;spin_lock_bh (&ifq->rq_lock);
@@ -751,6 +750,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp,struct io_zcrx_area *area;struct net_iov *niov;unsigned niov_idx, area_idx; +netmem_ref netmem; area_idx = rqe->off >>IORING_ZCRX_AREA_SHIFT ; niov_idx = (rqe->off & ~IORING_ZCRX_AREA_MASK ) >>PAGE_SHIFT ;
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 8/10 io_uring/zcrx: use guards for the refill lock
In reply to Pavel BegunkovUse guards for rq_lock in io_zcrx_ring_refill(), makes it a tad simpler.
--- io_uring/zcrx.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)
io_uring/zcrx.c +2 -5
@@ -736,14 +736,12 @@ static void io_zcrx_ring_refill(struct page_pool *pp,unsigned int mask = ifq->rq_entries -1 ;unsigned int entries; -spin_lock_bh (&ifq->rq_lock); +guard (spinlock_bh)(&ifq->rq_lock); entries =io_zcrx_rqring_entries (ifq); entries =min_t (unsigned, entries,PP_ALLOC_CACHE_REFILL - pp->alloc.count); -if (unlikely (!entries)) { -spin_unlock_bh (&ifq->rq_lock); +if (unlikely (!entries))return ; - } do { struct io_uring_zcrx_rqe *rqe = io_zcrx_get_rqe(ifq, mask);
@@ -780,7 +778,6 @@ static void io_zcrx_ring_refill(struct page_pool *pp, }while (--entries);smp_store_release (&ifq->rq_ring->head, ifq->cached_rq_head); -spin_unlock_bh (&ifq->rq_lock); }static void io_zcrx_refill_slow (struct page_pool *pp,struct io_zcrx_ifq *ifq)
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 9/10 io_uring/zcrx: don't adjust free cache space
In reply to Pavel BegunkovThe cache should be empty when io_pp_zc_alloc_netmems() is called, that's promised by page pool and further checked, so there is no need to recalculate the available space in io_zcrx_ring_refill().
--- io_uring/zcrx.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
io_uring/zcrx.c +1 -1
@@ -739,7 +739,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp,guard (spinlock_bh)(&ifq->rq_lock); entries =io_zcrx_rqring_entries (ifq); - entries =min_t (unsigned, entries,PP_ALLOC_CACHE_REFILL - pp->alloc.count); + entries =min_t (unsigned, entries,PP_ALLOC_CACHE_REFILL );if (unlikely (!entries))return ;
Signature · 1 line
2.49.0
Pavel Begunkov
maintainerPATCH 10/10 io_uring/zcrx: rely on cache size truncation on refill
In reply to Pavel BegunkovThe return value of io_zcrx_rqring_entries() is truncated by rq_size so that refilling doesn't loop indefinitely. However, io_zcrx_ring_refill() is already protected against that by iterating no more than PP_ALLOC_CACHE_REFILL times. Remove the truncation.
--- io_uring/zcrx.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-)
io_uring/zcrx.c +2 -6
@@ -716,10 +716,7 @@ void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)static inline u32 io_zcrx_rqring_entries (struct io_zcrx_ifq *ifq) { -u32 entries; - - entries =smp_load_acquire (&ifq->rq_ring->tail) - ifq->cached_rq_head; -return min (entries, ifq->rq_entries); +return smp_load_acquire (&ifq->rq_ring->tail) - ifq->cached_rq_head; } static struct io_uring_zcrx_rqe *io_zcrx_get_rqe(struct io_zcrx_ifq *ifq,
@@ -738,8 +735,7 @@ static void io_zcrx_ring_refill(struct page_pool *pp,guard (spinlock_bh)(&ifq->rq_lock); - entries =io_zcrx_rqring_entries (ifq); - entries =min_t (unsigned, entries,PP_ALLOC_CACHE_REFILL ); + entries =min (PP_ALLOC_CACHE_REFILL ,io_zcrx_rqring_entries (ifq));if (unlikely (!entries))return ;
Signature · 1 line
2.49.0
Quoted 8 lines · Pavel Begunkov
Flushing for review some of zcrx cleanups I had for a while now. This includes consolidating dma sync, optimising refilling, and using lock guards.
For a full branch with all relevant patches see
https://github.com/isilence/linux.git zcrx/for-next[...]
Applied, thanks!
[01/10] io_uring/zcrx: replace memchar_inv with is_zero
commit: 4c7956ee3605011e3c868a76aa5f5b8808380f8e
[02/10] io_uring/zcrx: use page_pool_unref_and_test()
commit: 32c02734932ac71eacdd0d799f136c0dcaf4a01c
[03/10] io_uring/zcrx: remove extra io_zcrx_drop_netdev
commit: 9f3cfd0fbe4cd2a26e43f1dddefa26a62dceb929
[04/10] io_uring/zcrx: rename dma lock
commit: 4857be683fe694be4367ae18873e8c18533cfbe0
[05/10] io_uring/zcrx: protect netdev with pp_lock
commit: 4e913a5d41c112979d67d35ffbefb5bddd3ff3a7
[06/10] io_uring/zcrx: unify allocation dma sync
commit: 5a3b2e8a4d0483073dda20c0b5fdf8f6545bb277
[07/10] io_uring/zcrx: reduce netmem scope in refill
commit: a4bb98933134b5da7cefb20a668c22ea018921c5
[08/10] io_uring/zcrx: use guards for the refill lock
commit: fdceaa88004a091461180045b084856aadfcce8d
[09/10] io_uring/zcrx: don't adjust free cache space
commit: 5f55fe9517f23692f622eb24e7f4449fdc2a8359
[10/10] io_uring/zcrx: rely on cache size truncation on refill
commit: 093964677b89452896b8566c47d9af8e0f8fd8dfBest regards,
Signature · 1 line
Jens Axboe