ring-buffer: Align meta-page to sub-buffers for improved TLB usage
Subsystems (3)
Steven Rostedt
maintainerPATCH 1/1 ring-buffer: Align meta-page to sub-buffers for improved TLB usage
git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git
ring-buffer/for-next
Head SHA1: ff74f987504fe90e88fdba8d80197202868ce342
Vincent Donnefort (1):
ring-buffer: Align meta-page to sub-buffers for improved TLB usage
----
kernel/trace/ring_buffer.c | 29 ++++++++++++++------------ tools/testing/selftests/ring-buffer/map_test.c | 14 +++++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) --------------------------- commit ff74f987504fe90e88fdba8d80197202868ce342 Author: Vincent Donnefort <[email protected]> Date: Thu May 23 10:14:45 2024 +0100
ring-buffer: Align meta-page to sub-buffers for improved TLB usage
Previously, the mapped ring-buffer layout caused misalignment between the meta-page and sub-buffers when the sub-buffer size was not a multiple of PAGE_SIZE. This prevented hardware with larger TLB entries from utilizing them effectively.
Add a padding with the zero-page between the meta-page and sub-buffers. Also update the ring-buffer map_test to verify that padding.
kernel/trace/ring_buffer.c +16 -13
@@ -6147,10 +6147,10 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer,/* install subbuf ID to kern VA translation */ cpu_buffer->subbuf_ids = subbuf_ids; - meta->meta_page_size =PAGE_SIZE ; meta->meta_struct_len =sizeof (*meta); meta->nr_subbufs = nr_subbufs; meta->subbuf_size = cpu_buffer->buffer->subbuf_size +BUF_PAGE_HDR_SIZE ; + meta->meta_page_size = meta->subbuf_size;rb_update_meta_page (cpu_buffer); }
@@ -6237,6 +6237,12 @@ static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, !(vma->vm_flags & VM_MAYSHARE))return -EPERM ; + subbuf_order = cpu_buffer->buffer->subbuf_order; + subbuf_pages =1 << subbuf_order; + +if (subbuf_order && pgoff % subbuf_pages) +return -EINVAL ; +/* * Make sure the mapping cannot become writable later. Also tell the VM * to not touch these pages (VM_DONTCOPY | VM_DONTEXPAND).
@@ -6246,11 +6252,8 @@ static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,lockdep_assert_held (&cpu_buffer->mapping_lock); - subbuf_order = cpu_buffer->buffer->subbuf_order; - subbuf_pages =1 << subbuf_order; - nr_subbufs = cpu_buffer->nr_pages +1 ;/* + reader-subbuf */ - nr_pages = ((nr_subbufs) << subbuf_order) - pgoff +1 ;/* + meta-page */ + nr_pages = ((nr_subbufs +1 ) << subbuf_order) - pgoff;/* + meta-page */ vma_pages = (vma->vm_end - vma->vm_start) >>PAGE_SHIFT ;if (!vma_pages || vma_pages > nr_pages)
@@ -6263,20 +6266,20 @@ static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer,return -ENOMEM ;if (!pgoff) { +unsigned long meta_page_padding; + pages[p++] =virt_to_page (cpu_buffer->meta_page);/* -* TODO: Align sub-buffers on their size, once -* vm_insert_pages() supports the zero-page. +* Pad with the zero-page to align the meta-page with the +* sub-buffers. */ + meta_page_padding = subbuf_pages -1 ; +while (meta_page_padding-- && p < nr_pages) + pages[p++] =ZERO_PAGE (vma->vm_start + (PAGE_SIZE * p)); }else {/* Skip the meta-page */ - pgoff--; - -if (pgoff % subbuf_pages) { - err = -EINVAL ; - goto out; - } + pgoff -= subbuf_pages; s += pgoff / subbuf_pages; }
tools/testing/selftests/ring-buffer/map_test.c +14 -0
@@ -228,6 +228,20 @@ TEST_F(map, data_mmap) data =mmap (NULL , data_len,PROT_READ ,MAP_SHARED , desc->cpu_fd, meta_len);ASSERT_EQ (data,MAP_FAILED ); + +/* Verify meta-page padding */ +if (desc->meta->meta_page_size >getpagesize ()) { +void *addr; + + data_len = desc->meta->meta_page_size; + data =mmap (NULL , data_len, +PROT_READ ,MAP_SHARED , desc->cpu_fd,0 ); +ASSERT_NE (data,MAP_FAILED ); + + addr = (void *)((unsigned long )data +getpagesize ()); +ASSERT_EQ (*((int *)addr),0 ); +munmap (data, data_len); + } }FIXTURE (snapshot) {