Use the memory pool instead of kmalloc
A message that is not in the archive (sent off-list or not yet received): [email protected]
RFC PATCH 2/2 Use the memory pool instead of kmalloc
Replace the use of kmalloc/kfree in the tracepoint probes that need dynamic allocation with the tracepoint memory pool alloc/free.
--- .../syscalls/headers/syscalls_pointers_override.h | 33 +++++++++------------- 1 file changed, 14 insertions(+), 19 deletions(-)
instrumentation/syscalls/headers/syscalls_pointers_override.h +14 -19
@@ -100,9 +100,8 @@ SC_LTTNG_TRACEPOINT_EVENT(pipe2, } \ \if (inp) { \ - tp_locvar->fds_in =kmalloc ( \ - tp_locvar->nr_ulong *sizeof (unsigned long ), \ -GFP_ATOMIC |GFP_NOWAIT ); \ + tp_locvar->fds_in =tp_mempool_alloc ( \ + tp_locvar->nr_ulong *sizeof (unsigned long )); \if (!tp_locvar->fds_in) \ goto error; \ \
@@ -113,9 +112,8 @@ SC_LTTNG_TRACEPOINT_EVENT(pipe2, goto error; \ } \if (outp) { \ - tp_locvar->fds_out =kmalloc ( \ - tp_locvar->nr_ulong *sizeof (unsigned long ), \ -GFP_ATOMIC |GFP_NOWAIT ); \ + tp_locvar->fds_out =tp_mempool_alloc ( \ + tp_locvar->nr_ulong *sizeof (unsigned long )); \if (!tp_locvar->fds_out) \ goto error; \ \
@@ -126,9 +124,8 @@ SC_LTTNG_TRACEPOINT_EVENT(pipe2, goto error; \ } \if (exp) { \ - tp_locvar->fds_ex =kmalloc ( \ - tp_locvar->nr_ulong *sizeof (unsigned long ), \ -GFP_ATOMIC |GFP_NOWAIT ); \ + tp_locvar->fds_ex =tp_mempool_alloc ( \ + tp_locvar->nr_ulong *sizeof (unsigned long )); \if (!tp_locvar->fds_ex) \ goto error; \ \
@@ -221,9 +218,9 @@ end: ; /* Label at end of compound statement. */ \ )#define LTTNG_SYSCALL_SELECT_code_post \ - kfree(tp_locvar->fds_in); \ - kfree(tp_locvar->fds_out); \ - kfree(tp_locvar->fds_ex); + tp_mempool_free(tp_locvar->fds_in); \ + tp_mempool_free(tp_locvar->fds_out); \ + tp_mempool_free(tp_locvar->fds_ex);#if defined(CONFIG_X86_32 ) || defined(CONFIG_X86_64 ) || defined(CONFIG_ARM )#define OVERRIDE_32_select
@@ -413,8 +410,7 @@ static struct lttng_type lttng_pollfd_elem = { { \int err; \ \ - tp_locvar->fds =kmalloc (tp_locvar->alloc_fds, \ -GFP_ATOMIC |GFP_NOWAIT ); \ + tp_locvar->fds =tp_mempool_alloc (tp_locvar->alloc_fds); \if (!tp_locvar->fds) \ goto error; \ err = lib_ring_buffer_copy_from_user_check_nofault( \
@@ -494,7 +490,7 @@ end: \ )#define LTTNG_SYSCALL_POLL_code_post \ - kfree(tp_locvar->fds); + tp_mempool_free(tp_locvar->fds);#if defined(CONFIG_X86_32 ) || defined(CONFIG_X86_64 ) || defined(CONFIG_ARM )#define OVERRIDE_32_poll
@@ -761,9 +757,8 @@ static struct lttng_type lttng_epoll_wait_elem = { tp_locvar->fds_length = ret; \ } \ \ - tp_locvar->events =kmalloc ( \ - maxalloc *sizeof (struct epoll_event ), \ -GFP_ATOMIC |GFP_NOWAIT ); \ + tp_locvar->events =tp_mempool_alloc ( \ + maxalloc *sizeof (struct epoll_event )); \if (!tp_locvar->events) { \ tp_locvar->fds_length =0 ; \ goto skip_code; \
@@ -805,7 +800,7 @@ static struct lttng_type lttng_epoll_wait_elem = {#define LTTNG_SYSCALL_EPOLL_WAIT_code_post \ sc_out( \ - kfree(tp_locvar->events); \ + tp_mempool_free(tp_locvar->events); \ )
Signature · 6 lines
2.7.4 _______________________________________________ lttng-dev mailing list [email protected] https://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev
Re: [RFC PATCH lttng-modules 1/2] Create a memory pool for temporary tracepoint probes storage
In reply to a message not in the archiveQuoted 9 lines · ----- On Feb 22, 2018, at 4:15 PM, Julien Desfossez [email protected]…
This memory pool is created when the lttng-tracer module is loaded. It allocates 4 buffers of 4k on each CPU. These buffers are designed to allow tracepoint probes to temporarily store data that does not fit on the stack (during the
code_preandcode_postphases). The memory is freed when the lttng-tracer module is unloaded.This removes the need for dynamic allocation during the execution of tracepoint probes, which does not behave well on
PREEMPT_RTkernel, even if theGFP_ATOMICandGFP_NOWAITflags are set.
"... event when invoked with the GFP_ATOMIC | GFP_NOWAIT flags."
Quoted 55 lines
Signed-off-by Julien Desfossez <[email protected]>---
Makefile | 3 +- lttng-abi.c | 9 ++ probes/lttng-tracepoint-event-impl.h | 1 + tp-mempool.c | 175 +++++++++++++++++++++++++++++++++++ tp-mempool.h | 48 ++++++++++ 5 files changed, 235 insertions(+), 1 deletion(-)create mode 100644
tp-mempool.c
create mode 100644tp-mempool.hMakefile +2
-1@@ -59,7 +59,8 @@ ifneq ($(KERNELRELEASE),) lttng-filter.o lttng-filter-interpreter.o \ lttng-filter-specialize.o \ lttng-filter-validator.o \ - probes/lttng-probe-user.o + probes/lttng-probe-user.o \ + tp-mempool.o ifneq ($(CONFIG_HAVE_SYSCALL_TRACEPOINTS),) lttng-tracer-objs += lttng-syscalls.olttng-abi.c +7
-0@@ -56,6 +56,7 @@#include <lttng-abi-old.h> #include <lttng-events.h> #include <lttng-tracer.h> +#include <tp-mempool.h> #include <lib/ringbuffer/frontend_types.h> /* @@ -1771,6 +1772,12 @@ int __init lttng_abi_init(void)wrapper_vmalloc_sync_all ();lttng_clock_ref (); + + ret =tp_mempool_init (); +if (ret) { + goto error; + } + lttng_proc_dentry =proc_create_data ("lttng" ,S_IRUSR |S_IWUSR ,NULL , <tng_fops,NULL );@@ -1784,6 +1791,7 @@ int __init lttng_abi_init(void)error:
lttng_clock_unref();
lttng-abi.c
Quoted 1 line
lttng-abi.c +1
-0+tp_mempool_destroy ();
we may want to unref clock after mempool destroy for symmetry.
lttng-abi.c void lttng_abi_exit(void) · line 1804
Quoted 8 lines
return ret; }lttng-abi.c +1
-0@@ -1793,4 +1801,5 @@ void lttng_abi_exit(void)lttng_clock_unref ();if (lttng_proc_dentry)remove_proc_entry ("lttng" ,NULL ); +tp_mempool_destroy ();
same.
Quoted 36 lines
lttng-abi.c +0
-0}probes/lttng-tracepoint-event-impl.h +1
-0@@ -34,6 +34,7 @@#include <wrapper/rcu.h> #include <lttng-events.h> #include <lttng-tracer-core.h> +#include <tp-mempool.h> #define __LTTNG_NULL_STRING "(null)"tp-mempool.c new file +16
-0@@ -0,0 +1,175 @@ +/* +* tp-mempool.c +* +* Copyright (C) 2018 Julien Desfossez <[email protected]> +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; only +* version 2.1 of the License. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public
tp-mempool.c line 26
Quoted 10 lines
tp-mempool.c new file +10
-0+* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <linux/slab.h> +#include <linux/percpu.h> + +#include <tp-mempool.h> + +#define BUF_SIZE 4 * 1024
missing (), but you can simply replace that with 4096
tp-mempool.c line 38
Quoted 12 lines
tp-mempool.c new file +12
-0+#define NR_BUF_PER_CPU 4 + +struct tp_buf_entry { +int cpu;/* To make sure we return the entry to the right pool. */ +char buf[BUF_SIZE ]; +struct list_head list; +}; + +struct per_cpu_buf { +struct list_head free_list;/* Free struct tp_buf_entry. */ +struct list_head allocated_list;/* Allocated struct tp_buf_entry. */ +};
What is the strategy to prevent concurrent updates here ?
In the current patchset, it's only called from syscall entry within a tracepoint probe (with preemption disabled), so there is only a single context using it per cpu. But if we end up using it in other execution contexts, it will break.
If this is an intended limitation, it should be clearly documented.
tp-mempool.c line 40
Quoted 2 lines
tp-mempool.c new file +2
-0+ +static struct per_cpu_buf __percpu *pool =NULL ;/* Per-cpu buffer. */
The " = NULL" can be removed for kernel coding style. It's zeroed anyway.
Quoted 14 lines
tp-mempool.c new file +14
-0+ +int tp_mempool_init (void ) +{ +int ret, cpu; + +/* The pool is only supposed to be allocated once. */ +if (pool) { +WARN_ON_ONCE (1 ); + ret =-1 ; + goto end; + } + + pool = alloc_percpu(struct per_cpu_buf); +if (!pool) {
tp-mempool.c line 64
Quoted 10 lines
tp-mempool.c new file +10
-0+ ret = -ENOMEM ; + goto end; + } + +for_each_possible_cpu (cpu) { +int i; +struct per_cpu_buf *cpu_buf =per_cpu_ptr (pool, cpu); + +INIT_LIST_HEAD (&cpu_buf->free_list); +INIT_LIST_HEAD (&cpu_buf->allocated_list);
We can remove the "allocated list", it's not really useful.
Also, please initialize the free_list for each possible cpu in a first pass, and then in a second pass allocate the items. Otherwise, if we have a kzalloc error, we end up calling tp_mempool_destroy which iterates on non-initialized free lists.
Quoted 30 lines
tp-mempool.c new file +30
-0+ +for (i =0 ; i <NR_BUF_PER_CPU ; i++) { +struct tp_buf_entry *entry; + + entry =kzalloc (sizeof (struct tp_buf_entry ), +GFP_KERNEL ); +if (!entry) { + ret = -ENOMEM ; + goto error_free_pool; + } + entry->cpu = cpu; +list_add_tail (&entry->list, &cpu_buf->free_list); + } + } + + ret =0 ; + goto end; + +error_free_pool: +tp_mempool_destroy (); +end: +return ret; +} + +void tp_mempool_destroy (void ) +{ +int cpu; + +if (!pool) { +return ;
tp-mempool.c line 104
Quoted 10 lines
tp-mempool.c new file +10
-0+ } + +for_each_possible_cpu (cpu) { +struct per_cpu_buf *cpu_buf =per_cpu_ptr (pool, cpu); +struct tp_buf_entry *entry, *tmp; + +list_for_each_entry_safe (entry, tmp, &cpu_buf->free_list, list) { +list_del (&entry->list); +kfree (entry); + }
If you really want to check that there is nothing allocated anymore, you can just count the number of items in the freelist here. Just reporting that something is fishy should be enough: freeing stuff from the allocated list is bogus anyway, because we don't own the allocated memory area, and another thread could still be using it.
Quoted 26 lines
tp-mempool.c new file +26
-0+if (!list_empty (&cpu_buf->allocated_list)) { +printk (KERN_WARNING "TP allocated memory pool not empty" ); +list_for_each_entry_safe (entry, tmp, + &cpu_buf->allocated_list, list) { +list_del (&entry->list); +kfree (entry); + } + } + } +free_percpu (pool); + pool =NULL ; +} + +void *tp_mempool_alloc (size_t size) +{ +void *ret; +struct tp_buf_entry *entry; +struct per_cpu_buf *cpu_buf; +int cpu =smp_processor_id (); + +if (size >BUF_SIZE ) { + ret =NULL ; + goto end; + } + + cpu_buf =per_cpu_ptr (pool, cpu);
tp-mempool.c line 140
Quoted 10 lines
tp-mempool.c new file +10
-0+if (list_empty (&cpu_buf->free_list)) { + ret =NULL ; + goto end; + } + + entry = list_first_entry(&cpu_buf->free_list, struct tp_buf_entry, list); +/* Remove the entry from the free list. */ +list_del (&entry->list); +/* Add it to the allocated list for tracking. */ +list_add_tail (&entry->list, &cpu_buf->allocated_list);
allocated list not needed.
Please memset to 0 on allocation.
Quoted 11 lines
tp-mempool.c new file +11
-0+ + ret = (void *) entry->buf; + +end: +return ret; +} + +void tp_mempool_free (void *ptr) +{ +struct tp_buf_entry *entry; +struct per_cpu_buf *cpu_buf;
tp-mempool.c line 161
Quoted 10 lines
tp-mempool.c new file +10
-0+ +if (!ptr) { + goto end; + } + + entry = container_of(ptr, struct tp_buf_entry, buf); +if (!entry) { + goto end; + } +memset (entry->buf,0 ,BUF_SIZE );
cache-wise, it's better to memset to 0 on allocation rather than free, because the memory ends up being written to right after allocation, whereas "free" often don't need to touch the memory.
tp-mempool.c line 169
Quoted 8 lines
tp-mempool.c new file +8
-0+ + cpu_buf =per_cpu_ptr (pool, entry->cpu); +if (!cpu_buf) { + goto end; + } + +/* Remove the entry from the allocated list. */ +list_del (&entry->list);
not needed.
Quoted 41 lines
tp-mempool.c new file +6
-0+/* Add it to the free list. */ +list_add_tail (&entry->list, &cpu_buf->free_list); + +end: +return ; +}tp-mempool.h new file +29
-0@@ -0,0 +1,48 @@ +#ifndef LTTNG_TP_MEMPOOL_H +#define LTTNG_TP_MEMPOOL_H + +/* +* tp-mempool.h +* +* Copyright (C) 2018 Julien Desfossez <[email protected]> +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License as published by the Free Software Foundation; only +* version 2.1 of the License. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +*/ + +#include <linux/percpu.h> + +/* +* Initialize the pool, only performed once. +* Returns 0 on success, a negative value on error. +*/
tp-mempool.h line 39
Quoted 10 lines
tp-mempool.h new file +10
-0+int tp_mempool_init (void ); + +/* +* Destroy the pool and free all the memory allocated. +*/ +void tp_mempool_destroy (void ); + +/* +* Ask for a buffer on the current cpu. +* Return a pointer to a buffer on success, NULL on error.
Concurrency requirements/guarantees should be documented right here. Allocation size limits too.
Thanks!
Mathieu
Quoted 11 lines
tp-mempool.h new file +9
-1+*/ +void *tp_mempool_alloc (size_t size); + +/* +* Release the memory reserved. +*/ +void tp_mempool_free (void *ptr); + +#endif /* LTTNG_TP_MEMPOOL_H */ --2.7.4
Signature · 7 lines
Mathieu Desnoyers EfficiOS Inc. http://www.efficios.com _______________________________________________ lttng-dev mailing list [email protected] https://lists.lttng.org/cgi-bin/mailman/listinfo/lttng-dev