mm: Refactor insert_page to prepare for batched-lock insert.
Subsystems (5)
PATCH RESEND 1/3 mm: Refactor insert_page to prepare for batched-lock insert.
From: Arjun Roy <[email protected]>
Add helper methods for vm_insert_page()/insert_page() to prepare for vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.
The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
--- mm/memory.c | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-)
mm/memory.c +24 -17
@@ -1693,6 +1693,27 @@ pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,return pte_alloc_map_lock (mm, pmd, addr, ptl); } +static int validate_page_before_insert (struct page *page) +{ +if (PageAnon (page) ||PageSlab (page)) +return -EINVAL ; +flush_dcache_page (page); +return 0 ; +} + +static int insert_page_into_pte_locked (struct mm_struct *mm,pte_t *pte, +unsigned long addr,struct page *page,pgprot_t prot) +{ +if (!pte_none (*pte)) +return -EBUSY ; +/* Ok, finally just insert the thing.. */ +get_page (page); +inc_mm_counter_fast (mm,mm_counter_file (page)); +page_add_file_rmap (page, false); +set_pte_at (mm, addr, pte,mk_pte (page, prot)); +return 0 ; +} +/* * This is the old fallback for page remapping. *
@@ -1708,28 +1729,14 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,pte_t *pte;spinlock_t *ptl; - retval = -EINVAL ; -if (PageAnon (page) ||PageSlab (page)) + retval =validate_page_before_insert (page); +if (retval) goto out; retval = -ENOMEM ; -flush_dcache_page (page); pte =get_locked_pte (mm, addr, &ptl);if (!pte) goto out; - retval = -EBUSY ; -if (!pte_none (*pte)) - goto out_unlock; - -/* Ok, finally just insert the thing.. */ -get_page (page); -inc_mm_counter_fast (mm,mm_counter_file (page)); -page_add_file_rmap (page, false); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); - - retval =0 ; -pte_unmap_unlock (pte, ptl); -return retval; -out_unlock: + retval =insert_page_into_pte_locked (mm, pte, addr, page, prot);pte_unmap_unlock (pte, ptl); out:return retval;
Signature · 1 line
2.25.0.rc1.283.g88dfdc4193-goog
PATCH RESEND 2/3 mm: Add vm_insert_pages().
From: Arjun Roy <[email protected]>
Add the ability to insert multiple pages at once to a user VM with lower PTE spinlock operations.
The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
--- include/linux/mm.h | 2 + mm/memory.c | 111 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 2 deletions(-)
include/linux/mm.h +2 -0
@@ -2563,6 +2563,8 @@ struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);int remap_pfn_range (struct vm_area_struct *,unsigned long addr,unsigned long pfn,unsigned long size,pgprot_t );int vm_insert_page (struct vm_area_struct *,unsigned long addr,struct page *); +int vm_insert_pages (struct vm_area_struct *vma,unsigned long addr, +struct page **pages,unsigned long *num);int vm_insert_pfn (struct vm_area_struct *vma,unsigned long addr,unsigned long pfn); int vm_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
mm/memory.c +109 -2
135 lines not on this page · raw message
Signature · 1 line
2.25.0.rc1.283.g88dfdc4193-goog
PATCH RESEND 3/3 net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
In reply to Arjun RoyFrom: Arjun Roy <[email protected]>
Use vm_insert_pages() for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.
The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
--- net/ipv4/tcp.c | 67 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 6 deletions(-)
net/ipv4/tcp.c +61 -6
@@ -1861,14 +1861,48 @@ int tcp_mmap(struct file *file, struct socket *sock, }EXPORT_SYMBOL (tcp_mmap); +static int tcp_zerocopy_vm_insert_batch (struct vm_area_struct *vma, +struct page **pages, +unsigned long pages_to_map, +unsigned long *insert_addr, +u32 *length_with_pending, +u32 *seq, +struct tcp_zerocopy_receive *zc) +{ +unsigned long pages_remaining = pages_to_map; +int bytes_mapped; +int ret; + + ret =vm_insert_pages (vma, *insert_addr, pages, &pages_remaining); + bytes_mapped =PAGE_SIZE * (pages_to_map - pages_remaining); +/* Even if vm_insert_pages fails, it may have partially succeeded in +* mapping (some but not all of the pages). +*/ + *seq += bytes_mapped; + *insert_addr += bytes_mapped; +if (ret) { +/* But if vm_insert_pages did fail, we have to unroll some state +* we speculatively touched before. +*/ +const int bytes_not_mapped =PAGE_SIZE * pages_remaining; + *length_with_pending -= bytes_not_mapped; + zc->recv_skip_hint += bytes_not_mapped; + } +return ret; +} +static int tcp_zerocopy_receive (struct sock *sk,struct tcp_zerocopy_receive *zc) {unsigned long address = (unsigned long )zc->address;u32 length =0 , seq, offset, zap_len; +#define PAGE_BATCH_SIZE 8 +struct page *pages[PAGE_BATCH_SIZE ];const skb_frag_t *frags =NULL ;struct vm_area_struct *vma;struct sk_buff *skb =NULL ; +unsigned long pg_idx =0 ; +unsigned long curr_addr;struct tcp_sock *tp;int inq;int ret;
@@ -1901,15 +1935,26 @@ static int tcp_zerocopy_receive(struct sock *sk, zc->recv_skip_hint = zc->length; } ret =0 ; + curr_addr = address;while (length +PAGE_SIZE <= zc->length) {if (zc->recv_skip_hint <PAGE_SIZE ) { +/* If we're here, finish the current batch. */ +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, + pg_idx, + &curr_addr, + &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + }if (skb) { skb = skb->next; offset = seq -TCP_SKB_CB (skb)->seq; }else { skb =tcp_recv_skb (sk, seq, &offset); } - zc->recv_skip_hint = skb->len - offset; offset -=skb_headlen (skb);if ((int )offset <0 ||skb_has_frag_list (skb))
@@ -1933,14 +1978,24 @@ static int tcp_zerocopy_receive(struct sock *sk, zc->recv_skip_hint -= remaining;break ; } - ret =vm_insert_page (vma, address + length, -skb_frag_page (frags)); -if (ret) -break ; + pages[pg_idx] =skb_frag_page (frags); + pg_idx++; length +=PAGE_SIZE ; - seq +=PAGE_SIZE ; zc->recv_skip_hint -=PAGE_SIZE ; frags++; +if (pg_idx ==PAGE_BATCH_SIZE ) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + } + } +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, &seq, + zc); } out:up_read (¤t->mm->mmap_sem);
Signature · 1 line
2.25.0.rc1.283.g88dfdc4193-goog
Andrew Morton
maintainerRe: [PATCH resend mm,net-next 2/3] mm: Add vm_insert_pages().
In reply to Arjun RoyQuoted 6 lines · On Mon, 27 Jan 2020 18:59:57 -0800 Arjun Roy <[email protected]> wrote:
Add the ability to insert multiple pages at once to a user VM with lower PTE spinlock operations.
The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
Seems sensible, thanks. Some other vm_insert_page() callers might want to know about this, but I can't immediately spot any which appear to be high bandwidth.
Is there much point in keeping the vm_insert_page() implementation around? Replace it with
static inline int
vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
struct page *page)
{
return vm_insert_pages(vma, addr, &page, 1);
}?
Also, vm_insert_page() does
if (!page_count(page)) return -EINVAL;
and this was not carried over into vm_insert_pages(). How come?
I don't know what that test does - it was added by Linus in the original commit a145dd411eb28c83. It's only been 15 years so I'm sure he remembers ;)
Andrew Morton
maintainerQuoted 8 lines · On Mon, 27 Jan 2020 18:59:56 -0800 Arjun Roy <[email protected]> wrote:
Add helper methods for vm_insert_page()/insert_page() to prepare for
vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
I tweaked this a bit for the addition of page_has_type() to insert_page(). Please check.
From: Arjun Roy <[email protected]>
Subject: mm: Refactor insert_page to prepare for batched-lock insert.
From: Arjun Roy <[email protected]>
Add helper methods for vm_insert_page()/insert_page() to prepare for vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.
The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
---
mm/memory.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-)
mm/memory.c renamed from mm/memory.c~mm-refactor-insert_page-to-prepare-for-batched-lock-insert +24 -15
@@ -1430,6 +1430,27 @@ pte_t *__get_locked_pte(struct mm_structreturn pte_alloc_map_lock (mm, pmd, addr, ptl); } +static int validate_page_before_insert (struct page *page) +{ +if (PageAnon (page) ||PageSlab (page) ||page_has_type (page)) +return -EINVAL ; +flush_dcache_page (page); +return 0 ; +} + +static int insert_page_into_pte_locked (struct mm_struct *mm,pte_t *pte, +unsigned long addr,struct page *page,pgprot_t prot) +{ +if (!pte_none (*pte)) +return -EBUSY ; +/* Ok, finally just insert the thing.. */ +get_page (page); +inc_mm_counter_fast (mm,mm_counter_file (page)); +page_add_file_rmap (page, false); +set_pte_at (mm, addr, pte,mk_pte (page, prot)); +return 0 ; +} +/* * This is the old fallback for page remapping. *
@@ -1445,26 +1466,14 @@ static int insert_page(struct vm_area_stpte_t *pte;spinlock_t *ptl; - retval = -EINVAL ; -if (PageAnon (page) ||PageSlab (page) ||page_has_type (page)) + retval =validate_page_before_insert (page); +if (retval) goto out; retval = -ENOMEM ; -flush_dcache_page (page); pte =get_locked_pte (mm, addr, &ptl);if (!pte) goto out; - retval = -EBUSY ; -if (!pte_none (*pte)) - goto out_unlock; - -/* Ok, finally just insert the thing.. */ -get_page (page); -inc_mm_counter_fast (mm,mm_counter_file (page)); -page_add_file_rmap (page, false); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); - - retval =0 ; -out_unlock: + retval =insert_page_into_pte_locked (mm, pte, addr, page, prot);pte_unmap_unlock (pte, ptl); out:return retval;
_
Andrew Morton
maintainerRe: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
In reply to Arjun RoyQuoted 9 lines · On Mon, 27 Jan 2020 18:59:58 -0800 Arjun Roy <[email protected]> wrote:
Use
vm_insert_pages()for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
For some reason the patch causes this:
In file included from ./arch/x86/include/asm/atomic.h:5:0,
from ./include/linux/atomic.h:7,
from ./include/linux/crypto.h:15,
from ./include/crypto/hash.h:11,
from net/ipv4/tcp.c:246:
net/ipv4/tcp.c: In function ‘do_tcp_getsockopt.isra.29’:
./include/linux/compiler.h:225:31: warning: ‘tp’ may be used uninitialized in this function [-Wmaybe-uninitialized]
case 4: *(volatile __u32 *)p = *(__u32 *)res; break;
~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~
net/ipv4/tcp.c:1779:19: note: ‘tp’ was declared here
struct tcp_sock *tp;
^~It's a false positive. gcc-7.2.0
: out:
: up_read(¤t->mm->mmap_sem);
: if (length) {
: WRITE_ONCE(tp->copied_seq, seq);
but `length' is zero here.
This suppresses it:
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy-fix +2 -1
@@ -1788,6 +1788,8 @@ static int tcp_zerocopy_receive(struct ssock_rps_record_flow (sk); + tp =tcp_sk (sk); +down_read (¤t->mm->mmap_sem); ret = -EINVAL ;
@@ -1796,7 +1798,6 @@ static int tcp_zerocopy_receive(struct s goto out; zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); - tp =tcp_sk (sk); seq = tp->copied_seq; inq =tcp_inq (sk); zc->length =min_t (u32, zc->length, inq);
and I guess it's zero-cost.
Anyway, I'll sit on this lot for a while, hoping for a davem ack?
The addition of page_has_type() looks good to me, thanks!
-Arjun
Quoted 108 lines · Andrew Morton
Quoted 8 lines · On Mon, 27 Jan 2020 18:59:56 -0800 Arjun Roy <[email protected]> wrote:
Add helper methods for vm_insert_page()/insert_page() to prepare for
vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
I tweaked this a bit for the addition of
page_has_type()toinsert_page(). Please check.From: Arjun Roy <[email protected]>
Subject: mm: Refactorinsert_pageto prepare for batched-lock insert.From: Arjun Roy <[email protected]>
Add helper methods for vm_insert_page()/insert_page() to prepare for
vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
Link:
http://lkml.kernel.org/r/[email protected]Signed-off-by Arjun Roy <[email protected]>Signed-off-by Eric Dumazet <[email protected]>Signed-off-by Soheil Hassas Yeganeh <[email protected]>Signed-off-by Andrew Morton <[email protected]>---
mm/memory.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-)---
a/mm/memory.c~mm-refactor-insert_page-to-prepare-for-batched-lock-insert +++a/mm/memory.cmm/memory.c renamed from mm/memory.c~mm-refactor-insert_page-to-prepare-for-batched-lock-insert +21
-0@@ -1430,6 +1430,27 @@ pte_t *__get_locked_pte(struct mm_structreturn pte_alloc_map_lock (mm, pmd, addr, ptl); } +static int validate_page_before_insert (struct page *page) +{ +if (PageAnon (page) ||PageSlab (page) ||page_has_type (page)) +return -EINVAL ; +flush_dcache_page (page); +return 0 ; +} + +static int insert_page_into_pte_locked (struct mm_struct *mm,pte_t *pte, +unsigned long addr,struct page *page,pgprot_t prot) +{ +if (!pte_none (*pte)) +return -EBUSY ; +/* Ok, finally just insert the thing.. */ +get_page (page); +inc_mm_counter_fast (mm,mm_counter_file (page)); +page_add_file_rmap (page, false); +set_pte_at (mm, addr, pte,mk_pte (page, prot)); +return 0 ; +} +/* * This is the old fallback for page remapping. *
mm/memory.c renamed from mm/memory.c~mm-refactor-insert_page-to-prepare-for-batched-lock-insert +3
-15@@ -1445,26 +1466,14 @@ static int insert_page(struct vm_area_stpte_t *pte;spinlock_t *ptl; - retval = -EINVAL ; -if (PageAnon (page) ||PageSlab (page) ||page_has_type (page)) + retval =validate_page_before_insert (page); +if (retval) goto out; retval = -ENOMEM ; -flush_dcache_page (page); pte =get_locked_pte (mm, addr, &ptl);if (!pte) goto out; - retval = -EBUSY ; -if (!pte_none (*pte)) - goto out_unlock; - -/* Ok, finally just insert the thing.. */ -get_page (page); -inc_mm_counter_fast (mm,mm_counter_file (page)); -page_add_file_rmap (page, false); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); - - retval =0 ; -out_unlock: + retval =insert_page_into_pte_locked (mm, pte, addr, page, prot);pte_unmap_unlock (pte, ptl); out:return retval;_
The addition of page_has_type() looks good to me, thanks!
-Arjun
Quoted 104 lines · Andrew Morton
Quoted 8 lines · On Mon, 27 Jan 2020 18:59:56 -0800 Arjun Roy <[email protected]> wrote:
Add helper methods for vm_insert_page()/insert_page() to prepare for
vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
I tweaked this a bit for the addition of
page_has_type()toinsert_page(). Please check.From: Arjun Roy <[email protected]>
Subject: mm: Refactorinsert_pageto prepare for batched-lock insert.From: Arjun Roy <[email protected]>
Add helper methods for vm_insert_page()/insert_page() to prepare for
vm_insert_pages(), which batch-inserts pages to reduce spinlock operations when inserting multiple consecutive pages into the user page table.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
Signed-off-by Arjun Roy <[email protected]>Signed-off-by Eric Dumazet <[email protected]>Signed-off-by Soheil Hassas Yeganeh <[email protected]>Signed-off-by Andrew Morton <[email protected]>---
mm/memory.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-)mm/memory.c renamed from mm/memory.c~mm-refactor-insert_page-to-prepare-for-batched-lock-insert +24
-15@@ -1430,6 +1430,27 @@ pte_t *__get_locked_pte(struct mm_structreturn pte_alloc_map_lock (mm, pmd, addr, ptl); } +static int validate_page_before_insert (struct page *page) +{ +if (PageAnon (page) ||PageSlab (page) ||page_has_type (page)) +return -EINVAL ; +flush_dcache_page (page); +return 0 ; +} + +static int insert_page_into_pte_locked (struct mm_struct *mm,pte_t *pte, +unsigned long addr,struct page *page,pgprot_t prot) +{ +if (!pte_none (*pte)) +return -EBUSY ; +/* Ok, finally just insert the thing.. */ +get_page (page); +inc_mm_counter_fast (mm,mm_counter_file (page)); +page_add_file_rmap (page, false); +set_pte_at (mm, addr, pte,mk_pte (page, prot)); +return 0 ; +} +/* * This is the old fallback for page remapping. * @@ -1445,26 +1466,14 @@ static int insert_page(struct vm_area_stpte_t *pte;spinlock_t *ptl; - retval = -EINVAL ; -if (PageAnon (page) ||PageSlab (page) ||page_has_type (page)) + retval =validate_page_before_insert (page); +if (retval) goto out; retval = -ENOMEM ; -flush_dcache_page (page); pte =get_locked_pte (mm, addr, &ptl);if (!pte) goto out; - retval = -EBUSY ; -if (!pte_none (*pte)) - goto out_unlock; - -/* Ok, finally just insert the thing.. */ -get_page (page); -inc_mm_counter_fast (mm,mm_counter_file (page)); -page_add_file_rmap (page, false); -set_pte_at (mm, addr, pte,mk_pte (page, prot)); - - retval =0 ; -out_unlock: + retval =insert_page_into_pte_locked (mm, pte, addr, page, prot);pte_unmap_unlock (pte, ptl); out:return retval;_
Re: [PATCH resend mm,net-next 2/3] mm: Add vm_insert_pages().
In reply to Andrew MortonI think at least to start it probably makes sense to keep regular vm_insert_page() around - smaller stack used, less branches, if you know you just need one page - not sure if gcc would err towards smaller binary or not when compiling.
Regarding the page_count() check - as far as I can tell that's just checking to make sure that at least *someone* has a reference to the page before inserting it; in the zerocopy case we most definitely do but I guess a bad caller could call it with a bad page argument and this would guard against that.
Actually, I appear to have fat fingered it - I intended for this check to be in there but seem to have forgotten (per the comment "/* Defer page refcount checking till we're about to map that page. */" but with no actual check). So that check should go inside insert_page_in_batch_locked(), right before the validate_page_before_insert() check. I'll send an updated fixup diff shortly.
-Arjun
Quoted 36 lines · Andrew Morton
Quoted 6 lines · On Mon, 27 Jan 2020 18:59:57 -0800 Arjun Roy <[email protected]> wrote:
Add the ability to insert multiple pages at once to a user VM with lower PTE spinlock operations.
The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
Seems sensible, thanks. Some other
vm_insert_page()callers might want to know about this, but I can't immediately spot any which appear to be high bandwidth.Is there much point in keeping the
vm_insert_page()implementation around? Replace it withstatic inline int
vm_insert_page(structvm_area_struct*vma, unsigned long addr,
struct page *page){ return vm_insert_pages(vma, addr, &page, 1); }?
Also,
vm_insert_page()doesif (!page_count(page)) return -EINVAL;and this was not carried over into
vm_insert_pages(). How come?I don't know what that test does - it was added by Linus in the original commit
a145dd411eb28c83. It's only been 15 years so I'm sure he remembers ;)
Re: [PATCH resend mm,net-next 2/3] mm: Add vm_insert_pages().
In reply to Andrew MortonQuoted 7 lines · Andrew Morton
Also,
vm_insert_page()doesif (!page_count(page)) return -EINVAL;and this was not carried over into
vm_insert_pages(). How come?
Sounds like that was just a mistake.
Quoted 3 lines
I don't know what that test does - it was added by Linus in the original commit
a145dd411eb28c83. It's only been 15 years so I'm sure he remembers ;)
Oh, sure.
No, I have absolutely no memory of the details, but I think the commit message is actually the big hint: the difference between vm_insert_page() and some of the more random "insert any pdf" cases we have is exactly that:
The page you insert needs to be a nice clean kernel allocation, so you can't insert arbitrary page mappings with this, but that's not what people want.
thing. The comment above it also kind of hints at it.
We *historically* had interfaces to insert random reserved pages (for IO mappings, but also the zero page etc), but the whole point of that vm_insert_page() is that it's now an interface for drivers to insert the pages they maintain into the page tables.
But that also means that we very much didn't allow just random pages accessed by doing pfn lookups (that might not be in use at all).
Is "page_count()" a great test? No. But it's at least _a_ test of that. No reserved pages or other magic need apply.
Linus
Re: [PATCH resend mm,net-next 2/3] mm: Add vm_insert_pages().
In reply to Arjun Royinclude/linux/mm.h struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); · line 2569
Quoted 5 lines · Arjun Roy
int
vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);include/linux/mm.h +2
-0+int vm_insert_pages (struct vm_area_struct *vma,unsigned long addr, +struct page **pages,unsigned long *num);int vm_insert_pfn (struct vm_area_struct *vma,unsigned long addr,unsigned long pfn);
Sorry I didn't notice these patches earlier. I'm not thrilled about the addition of a new vm_insert_* operation; we're moving towards a vmf_insert_* API. There are almost no users left of vm_insert_page (10, at a quick count). Once they're all gone, we can switch the underlying primitives over to a vm_fault_t return type and get rid of the errno-to-vm-fault translation step that currently goes on.
So ... is this called in the fault path? Do you have a struct vm_fault around? Can you handle a vm_fault_t return value instead of an errno?
Re: [PATCH resend mm,net-next 2/3] mm: Add vm_insert_pages().
Quoted 15 lines · Matthew Wilcox
include/linux/mm.h struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); · line 2569
Quoted 5 lines · Arjun Roy
int
vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);include/linux/mm.h +2
-0+int vm_insert_pages (struct vm_area_struct *vma,unsigned long addr, +struct page **pages,unsigned long *num);int vm_insert_pfn (struct vm_area_struct *vma,unsigned long addr,unsigned long pfn);Sorry I didn't notice these patches earlier. I'm not thrilled about the addition of a new vm_insert_* operation; we're moving towards a vmf_insert_* API. There are almost no users left of
vm_insert_page(10, at a quick count). Once they're all gone, we can switch the underlying primitives over to avm_fault_treturn type and get rid of the errno-to-vm-fault translation step that currently goes on.
Quoted 2 lines
So ... is this called in the fault path? Do you have a struct
vm_faultaround? Can you handle avm_fault_treturn value instead of an errno?
This is not a page fault, really. This customer of vm_insert_page() is the TCP receive zerocopy code, which is remapping pages from the NIC into the userspace process (in lieu of sys_recvmsg()'s copy). See:tcp_zerocopy_receive() in net/ipv4/tcp.c .
I took a peek at vmf_insert_page(). I think that hides the presence of EBUSY, which would be a necessary signal for us. If that was exposed I think vm_fault_t could be fine, *but* I shall defer to Eric for actually deciding on it.
-Arjun
Re: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
In reply to Andrew MortonQuoted 59 lines · Andrew Morton
Quoted 9 lines · On Mon, 27 Jan 2020 18:59:58 -0800 Arjun Roy <[email protected]> wrote:
Use
vm_insert_pages()for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
For some reason the patch causes this:
In file included from
./arch/x86/include/asm/atomic.h:5:0,from ./include/linux/atomic.h:7, from ./include/linux/crypto.h:15, from ./include/crypto/hash.h:11, from net/ipv4/tcp.c:246: net/ipv4/tcp.c: In function ‘do_tcp_getsockopt.isra.29’: ./include/linux/compiler.h:225:31: warning: ‘tp’ may be used uninitialized in this function [-Wmaybe-uninitialized] case 4: *(volatile __u32 *)p = *(__u32 *)res; break; ~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~ net/ipv4/tcp.c:1779:19: note: ‘tp’ was declared here struct tcp_sock *tp; ^~It's a false positive. gcc-7.2.0
: out:
:up_read(¤t->mm->mmap_sem);
: if (length) {
:WRITE_ONCE(tp->copied_seq, seq);but `length' is zero here.
This suppresses it:
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy-fix +2
-1@@ -1788,6 +1788,8 @@ static int tcp_zerocopy_receive(struct ssock_rps_record_flow (sk); + tp =tcp_sk (sk); +down_read (¤t->mm->mmap_sem); ret = -EINVAL ;@@ -1796,7 +1798,6 @@ static int tcp_zerocopy_receive(struct s goto out; zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); - tp =tcp_sk (sk); seq = tp->copied_seq; inq =tcp_inq (sk); zc->length =min_t (u32, zc->length, inq);
Quoted 4 lines
and I guess it's zero-cost.
Anyway, I'll sit on this lot for a while, hoping for a davem ack?
Actually, speaking of the ack on the networking side:
I guess this patch set is a bit weird since it requires some non-trivial coordination between mm and net-next? Not sure what the normal approach is in this case.
-Arjun
Re: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
Andrew, David -
I remain a bit concerned regarding the merge process for this specific patch (0003, the net/ipv4/tcp.c change) since I have other in-flight changes for TCP receive zerocopy that I'd like to upstream for net-next - and would like to avoid weird merge issues.
So perhaps the following could work:
1. Andrew, perhaps we could remove this particular patch (0003, the net/ipv4/tcp.c change) from mm-next; that way we merge vm_insert_pages() but not the call-site within TCP, for now.
2. net-next will eventually pick vm_insert_pages() up.
3. I can modify the zerocopy code to use it at that point?
Else I'm concerned a complicated merge situation may result.
What do you all think?
Thanks,
-Arjun
Quoted 73 lines · Arjun Roy
Quoted 59 lines · Andrew Morton
Quoted 9 lines · On Mon, 27 Jan 2020 18:59:58 -0800 Arjun Roy <[email protected]> wrote:
Use
vm_insert_pages()for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.The intention of this patch-set is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
For some reason the patch causes this:
In file included from
./arch/x86/include/asm/atomic.h:5:0,from ./include/linux/atomic.h:7, from ./include/linux/crypto.h:15, from ./include/crypto/hash.h:11, from net/ipv4/tcp.c:246: net/ipv4/tcp.c: In function ‘do_tcp_getsockopt.isra.29’: ./include/linux/compiler.h:225:31: warning: ‘tp’ may be used uninitialized in this function [-Wmaybe-uninitialized] case 4: *(volatile __u32 *)p = *(__u32 *)res; break; ~~~~~~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~ net/ipv4/tcp.c:1779:19: note: ‘tp’ was declared here struct tcp_sock *tp; ^~It's a false positive. gcc-7.2.0
: out:
:up_read(¤t->mm->mmap_sem);
: if (length) {
:WRITE_ONCE(tp->copied_seq, seq);but `length' is zero here.
This suppresses it:
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy-fix +2
-1@@ -1788,6 +1788,8 @@ static int tcp_zerocopy_receive(struct ssock_rps_record_flow (sk); + tp =tcp_sk (sk); +down_read (¤t->mm->mmap_sem); ret = -EINVAL ;@@ -1796,7 +1798,6 @@ static int tcp_zerocopy_receive(struct s goto out; zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); - tp =tcp_sk (sk); seq = tp->copied_seq; inq =tcp_inq (sk); zc->length =min_t (u32, zc->length, inq);Quoted 4 lines
and I guess it's zero-cost.
Anyway, I'll sit on this lot for a while, hoping for a davem ack?
Actually, speaking of the ack on the networking side:
I guess this patch set is a bit weird since it requires some non-trivial coordination between mm and net-next? Not sure what the normal approach is in this case.
-Arjun
Andrew Morton
maintainerRe: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
Quoted 13 lines · On Fri, 21 Feb 2020 13:21:41 -0800 Arjun Roy <[email protected]> wrote:
I remain a bit concerned regarding the merge process for this specific patch (0003, the
net/ipv4/tcp.cchange) since I have other in-flight changes for TCP receive zerocopy that I'd like to upstream for net-next - and would like to avoid weird merge issues.So perhaps the following could work:
1. Andrew, perhaps we could remove this particular patch (0003, the
net/ipv4/tcp.cchange) from mm-next; that way we mergevm_insert_pages()but not the call-site within TCP, for now.
2. net-next will eventually pickvm_insert_pages()up.
3. I can modify the zerocopy code to use it at that point?
Quoted 3 lines
Else I'm concerned a complicated merge situation may result.
What do you all think?
We could do that.
For now, I'll stage the entire patch series after linux-next and shall wait and see whether things which appear in linux-next cause serious merge issues to occur. Sound OK?
Re: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
Quoted 18 lines · Andrew Morton
Quoted 15 lines · On Fri, 21 Feb 2020 13:21:41 -0800 Arjun Roy <[email protected]> wrote:
I remain a bit concerned regarding the merge process for this specific patch (0003, the
net/ipv4/tcp.cchange) since I have other in-flight changes for TCP receive zerocopy that I'd like to upstream for net-next - and would like to avoid weird merge issues.So perhaps the following could work:
1. Andrew, perhaps we could remove this particular patch (0003, the
net/ipv4/tcp.cchange) from mm-next; that way we mergevm_insert_pages()but not the call-site within TCP, for now.
2. net-next will eventually pickvm_insert_pages()up.
3. I can modify the zerocopy code to use it at that point?Else I'm concerned a complicated merge situation may result.
Quoted 7 lines
Quoted 1 line
What do you all think?
We could do that.
For now, I'll stage the entire patch series after linux-next and shall wait and see whether things which appear in linux-next cause serious merge issues to occur. Sound OK?
Sounds good for now; the conflict itself would be easy enough to fix when it does crop up.
Thanks,
-Arjun
Andrew Morton
maintainerRe: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
In reply to Arjun RoyQuoted 7 lines · On Fri, 21 Feb 2020 13:21:41 -0800 Arjun Roy <[email protected]> wrote:
I remain a bit concerned regarding the merge process for this specific patch (0003, the
net/ipv4/tcp.cchange) since I have other in-flight changes for TCP receive zerocopy that I'd like to upstream for net-next - and would like to avoid weird merge issues.So perhaps the following could work:
Quoted 7 lines
1. Andrew, perhaps we could remove this particular patch (0003, the
net/ipv4/tcp.cchange) from mm-next; that way we mergevm_insert_pages()but not the call-site within TCP, for now.
2. net-next will eventually pickvm_insert_pages()up.
3. I can modify the zerocopy code to use it at that point?Else I'm concerned a complicated merge situation may result.
The merge situation is quite clean.
I guess I'll hold off on net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy.patch (below) and shall send it to davem after Linus has merged the prerequisites.
From: Arjun Roy <[email protected]>
Subject: net-zerocopy: use vm_insert_pages() for tcp rcv zerocopy
Use vm_insert_pages() for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.
The intention of this patchset is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
---
net/ipv4/tcp.c | 70 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-)
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +63 -7
@@ -1734,14 +1734,48 @@ int tcp_mmap(struct file *file, struct s }EXPORT_SYMBOL (tcp_mmap); +static int tcp_zerocopy_vm_insert_batch (struct vm_area_struct *vma, +struct page **pages, +unsigned long pages_to_map, +unsigned long *insert_addr, +u32 *length_with_pending, +u32 *seq, +struct tcp_zerocopy_receive *zc) +{ +unsigned long pages_remaining = pages_to_map; +int bytes_mapped; +int ret; + + ret =vm_insert_pages (vma, *insert_addr, pages, &pages_remaining); + bytes_mapped =PAGE_SIZE * (pages_to_map - pages_remaining); +/* Even if vm_insert_pages fails, it may have partially succeeded in +* mapping (some but not all of the pages). +*/ + *seq += bytes_mapped; + *insert_addr += bytes_mapped; +if (ret) { +/* But if vm_insert_pages did fail, we have to unroll some state +* we speculatively touched before. +*/ +const int bytes_not_mapped =PAGE_SIZE * pages_remaining; + *length_with_pending -= bytes_not_mapped; + zc->recv_skip_hint += bytes_not_mapped; + } +return ret; +} +static int tcp_zerocopy_receive (struct sock *sk,struct tcp_zerocopy_receive *zc) {unsigned long address = (unsigned long )zc->address;u32 length =0 , seq, offset, zap_len; +#define PAGE_BATCH_SIZE 8 +struct page *pages[PAGE_BATCH_SIZE ];const skb_frag_t *frags =NULL ;struct vm_area_struct *vma;struct sk_buff *skb =NULL ; +unsigned long pg_idx =0 ; +unsigned long curr_addr;struct tcp_sock *tp;int inq;int ret;
@@ -1754,6 +1788,8 @@ static int tcp_zerocopy_receive(struct ssock_rps_record_flow (sk); + tp =tcp_sk (sk); +down_read (¤t->mm->mmap_sem); ret = -EINVAL ;
@@ -1762,7 +1798,6 @@ static int tcp_zerocopy_receive(struct s goto out; zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); - tp =tcp_sk (sk); seq = tp->copied_seq; inq =tcp_inq (sk); zc->length =min_t (u32, zc->length, inq);
@@ -1774,8 +1809,20 @@ static int tcp_zerocopy_receive(struct s zc->recv_skip_hint = zc->length; } ret =0 ; + curr_addr = address;while (length +PAGE_SIZE <= zc->length) {if (zc->recv_skip_hint <PAGE_SIZE ) { +/* If we're here, finish the current batch. */ +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, + pg_idx, + &curr_addr, + &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + }if (skb) {if (zc->recv_skip_hint >0 )break ;
@@ -1784,7 +1831,6 @@ static int tcp_zerocopy_receive(struct s }else { skb =tcp_recv_skb (sk, seq, &offset); } - zc->recv_skip_hint = skb->len - offset; offset -=skb_headlen (skb);if ((int )offset <0 ||skb_has_frag_list (skb))
@@ -1808,14 +1854,24 @@ static int tcp_zerocopy_receive(struct s zc->recv_skip_hint -= remaining;break ; } - ret =vm_insert_page (vma, address + length, -skb_frag_page (frags)); -if (ret) -break ; + pages[pg_idx] =skb_frag_page (frags); + pg_idx++; length +=PAGE_SIZE ; - seq +=PAGE_SIZE ; zc->recv_skip_hint -=PAGE_SIZE ; frags++; +if (pg_idx ==PAGE_BATCH_SIZE ) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + } + } +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, &seq, + zc); } out:up_read (¤t->mm->mmap_sem);
_
Re: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
Quoted 15 lines · Andrew Morton
Quoted 13 lines · On Fri, 21 Feb 2020 13:21:41 -0800 Arjun Roy <[email protected]> wrote:
I remain a bit concerned regarding the merge process for this specific patch (0003, the
net/ipv4/tcp.cchange) since I have other in-flight changes for TCP receive zerocopy that I'd like to upstream for net-next - and would like to avoid weird merge issues.So perhaps the following could work:
1. Andrew, perhaps we could remove this particular patch (0003, the
net/ipv4/tcp.cchange) from mm-next; that way we mergevm_insert_pages()but not the call-site within TCP, for now.
2. net-next will eventually pickvm_insert_pages()up.
3. I can modify the zerocopy code to use it at that point?
Quoted 10 lines
Quoted 1 line
Else I'm concerned a complicated merge situation may result.
The merge situation is quite clean.
I guess I'll hold off on net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy.patch (below) and shall send it to davem after Linus has merged the prerequisites.
Acknowledged, thank you!
-Arjun
Quoted 167 lines
From: Arjun Roy <[email protected]>
Subject: net-zerocopy: usevm_insert_pages()for tcp rcv zerocopyUse
vm_insert_pages()for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.The intention of this patchset is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
[[email protected]: suppress gcc-7.2.0 warning] Link:
http://lkml.kernel.org/r/[email protected]Signed-off-by Arjun Roy <[email protected]>Signed-off-by Eric Dumazet <[email protected]>Signed-off-by Soheil Hassas Yeganeh <[email protected]>Cc David Miller <[email protected]>Cc Matthew Wilcox <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc Stephen Rothwell <[email protected]>Signed-off-by Andrew Morton <[email protected]>---
net/ipv4/tcp.c | 70 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-)net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +15
-0@@ -1734,14 +1734,48 @@ int tcp_mmap(struct file *file, struct s }EXPORT_SYMBOL (tcp_mmap); +static int tcp_zerocopy_vm_insert_batch (struct vm_area_struct *vma, +struct page **pages, +unsigned long pages_to_map, +unsigned long *insert_addr, +u32 *length_with_pending, +u32 *seq, +struct tcp_zerocopy_receive *zc) +{ +unsigned long pages_remaining = pages_to_map; +int bytes_mapped; +int ret; + + ret =vm_insert_pages (vma, *insert_addr, pages, &pages_remaining); + bytes_mapped =PAGE_SIZE * (pages_to_map - pages_remaining); +/* Even if vm_insert_pages fails, it may have partially succeeded in
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +6
-0+* mapping (some but not all of the pages). +*/ + *seq += bytes_mapped; + *insert_addr += bytes_mapped; +if (ret) { +/* But if vm_insert_pages did fail, we have to unroll some state
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +15
-1+* we speculatively touched before. +*/ +const int bytes_not_mapped =PAGE_SIZE * pages_remaining; + *length_with_pending -= bytes_not_mapped; + zc->recv_skip_hint += bytes_not_mapped; + } +return ret; +} +static int tcp_zerocopy_receive (struct sock *sk,struct tcp_zerocopy_receive *zc) {unsigned long address = (unsigned long )zc->address;u32 length =0 , seq, offset, zap_len; +#define PAGE_BATCH_SIZE 8 +struct page *pages[PAGE_BATCH_SIZE ];const skb_frag_t *frags =NULL ;struct vm_area_struct *vma;struct sk_buff *skb =NULL ; +unsigned long pg_idx =0 ; +unsigned long curr_addr;struct tcp_sock *tp;int inq;int ret;@@ -1754,6 +1788,8 @@ static int tcp_zerocopy_receive(struct ssock_rps_record_flow (sk); + tp =tcp_sk (sk); +down_read (¤t->mm->mmap_sem); ret = -EINVAL ;@@ -1762,7 +1798,6 @@ static int tcp_zerocopy_receive(struct s goto out; zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); - tp =tcp_sk (sk); seq = tp->copied_seq; inq =tcp_inq (sk);zc->length = min_t(u32, zc->length, inq);net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +12
-0@@ -1774,8 +1809,20 @@ static int tcp_zerocopy_receive(struct s zc->recv_skip_hint = zc->length; } ret =0 ; + curr_addr = address;while (length +PAGE_SIZE <= zc->length) {if (zc->recv_skip_hint <PAGE_SIZE ) { +/* If we're here, finish the current batch. */ +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, + pg_idx, + &curr_addr, + &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + }if (skb) { if (zc->recv_skip_hint > 0) break;net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +15
-6@@ -1784,7 +1831,6 @@ static int tcp_zerocopy_receive(struct s }else { skb =tcp_recv_skb (sk, seq, &offset); } - zc->recv_skip_hint = skb->len - offset; offset -=skb_headlen (skb);if ((int )offset <0 ||skb_has_frag_list (skb))@@ -1808,14 +1854,24 @@ static int tcp_zerocopy_receive(struct s zc->recv_skip_hint -= remaining;break ; } - ret =vm_insert_page (vma, address + length, -skb_frag_page (frags)); -if (ret) -break ; + pages[pg_idx] =skb_frag_page (frags); + pg_idx++; length +=PAGE_SIZE ; - seq +=PAGE_SIZE ; zc->recv_skip_hint -=PAGE_SIZE ; frags++; +if (pg_idx ==PAGE_BATCH_SIZE ) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + } + } +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, &seq, + zc);} out: up_read(¤t->mm->mmap_sem);_
Re: [PATCH resend mm,net-next 3/3] net-zerocopy: Use vm_insert_pages() for tcp rcv zerocopy.
In reply to Andrew MortonQuoted 16 lines · Andrew Morton
Quoted 13 lines · On Fri, 21 Feb 2020 13:21:41 -0800 Arjun Roy <[email protected]> wrote:
I remain a bit concerned regarding the merge process for this specific patch (0003, the
net/ipv4/tcp.cchange) since I have other in-flight changes for TCP receive zerocopy that I'd like to upstream for net-next - and would like to avoid weird merge issues.So perhaps the following could work:
1. Andrew, perhaps we could remove this particular patch (0003, the
net/ipv4/tcp.cchange) from mm-next; that way we mergevm_insert_pages()but not the call-site within TCP, for now.
2. net-next will eventually pickvm_insert_pages()up.
3. I can modify the zerocopy code to use it at that point?
Quoted 8 lines
Quoted 1 line
Else I'm concerned a complicated merge situation may result.
The merge situation is quite clean.
I guess I'll hold off on net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy.patch (below) and shall send it to davem after Linus has merged the prerequisites.
Acknowledged, thank you!
(resend because gmail conveniently forgot my plain text mode preference...)
-Arjun
Quoted 157 lines
From: Arjun Roy <[email protected]>
Subject: net-zerocopy: usevm_insert_pages()for tcp rcv zerocopyUse
vm_insert_pages()for tcp receive zerocopy. Spin lock cycles (as reported by perf) drop from a couple of percentage points to a fraction of a percent. This results in a roughly 6% increase in efficiency, measured roughly as zerocopy receive count divided by CPU utilization.The intention of this patchset is to reduce atomic ops for tcp zerocopy receives, which normally hits the same spinlock multiple times consecutively.
[[email protected]: suppress gcc-7.2.0 warning]Signed-off-by Arjun Roy <[email protected]>Signed-off-by Eric Dumazet <[email protected]>Signed-off-by Soheil Hassas Yeganeh <[email protected]>Cc David Miller <[email protected]>Cc Matthew Wilcox <[email protected]>Cc Jason Gunthorpe <[email protected]>Cc Stephen Rothwell <[email protected]>Signed-off-by Andrew Morton <[email protected]>---
net/ipv4/tcp.c | 70 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-)net/ipv4/tcp.c renamed from net/ipv4/tcp.c~net-zerocopy-use-vm_insert_pages-for-tcp-rcv-zerocopy +63
-7@@ -1734,14 +1734,48 @@ int tcp_mmap(struct file *file, struct s }EXPORT_SYMBOL (tcp_mmap); +static int tcp_zerocopy_vm_insert_batch (struct vm_area_struct *vma, +struct page **pages, +unsigned long pages_to_map, +unsigned long *insert_addr, +u32 *length_with_pending, +u32 *seq, +struct tcp_zerocopy_receive *zc) +{ +unsigned long pages_remaining = pages_to_map; +int bytes_mapped; +int ret; + + ret =vm_insert_pages (vma, *insert_addr, pages, &pages_remaining); + bytes_mapped =PAGE_SIZE * (pages_to_map - pages_remaining); +/* Even if vm_insert_pages fails, it may have partially succeeded in +* mapping (some but not all of the pages). +*/ + *seq += bytes_mapped; + *insert_addr += bytes_mapped; +if (ret) { +/* But if vm_insert_pages did fail, we have to unroll some state +* we speculatively touched before. +*/ +const int bytes_not_mapped =PAGE_SIZE * pages_remaining; + *length_with_pending -= bytes_not_mapped; + zc->recv_skip_hint += bytes_not_mapped; + } +return ret; +} +static int tcp_zerocopy_receive (struct sock *sk,struct tcp_zerocopy_receive *zc) {unsigned long address = (unsigned long )zc->address;u32 length =0 , seq, offset, zap_len; +#define PAGE_BATCH_SIZE 8 +struct page *pages[PAGE_BATCH_SIZE ];const skb_frag_t *frags =NULL ;struct vm_area_struct *vma;struct sk_buff *skb =NULL ; +unsigned long pg_idx =0 ; +unsigned long curr_addr;struct tcp_sock *tp;int inq;int ret;@@ -1754,6 +1788,8 @@ static int tcp_zerocopy_receive(struct ssock_rps_record_flow (sk); + tp =tcp_sk (sk); +down_read (¤t->mm->mmap_sem); ret = -EINVAL ;@@ -1762,7 +1798,6 @@ static int tcp_zerocopy_receive(struct s goto out; zc->length = min_t(unsigned long, zc->length, vma->vm_end - address); - tp =tcp_sk (sk); seq = tp->copied_seq; inq =tcp_inq (sk); zc->length =min_t (u32, zc->length, inq);@@ -1774,8 +1809,20 @@ static int tcp_zerocopy_receive(struct s zc->recv_skip_hint = zc->length; } ret =0 ; + curr_addr = address;while (length +PAGE_SIZE <= zc->length) {if (zc->recv_skip_hint <PAGE_SIZE ) { +/* If we're here, finish the current batch. */ +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, + pg_idx, + &curr_addr, + &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + }if (skb) {if (zc->recv_skip_hint >0 )break ;@@ -1784,7 +1831,6 @@ static int tcp_zerocopy_receive(struct s }else { skb =tcp_recv_skb (sk, seq, &offset); } - zc->recv_skip_hint = skb->len - offset; offset -=skb_headlen (skb);if ((int )offset <0 ||skb_has_frag_list (skb))@@ -1808,14 +1854,24 @@ static int tcp_zerocopy_receive(struct s zc->recv_skip_hint -= remaining;break ; } - ret =vm_insert_page (vma, address + length, -skb_frag_page (frags)); -if (ret) -break ; + pages[pg_idx] =skb_frag_page (frags); + pg_idx++; length +=PAGE_SIZE ; - seq +=PAGE_SIZE ; zc->recv_skip_hint -=PAGE_SIZE ; frags++; +if (pg_idx ==PAGE_BATCH_SIZE ) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, + &seq, zc); +if (ret) + goto out; + pg_idx =0 ; + } + } +if (pg_idx) { + ret =tcp_zerocopy_vm_insert_batch (vma, pages, pg_idx, + &curr_addr, &length, &seq, + zc); } out:up_read (¤t->mm->mmap_sem);_