Reimplementation of linux dynamic percpu memory allocator
Subsystems (2)
RFC 1/1 Reimplementation of linux dynamic percpu memory allocator
The following patch reimplements the linux dynamic percpu memory allocator so that:
1. Percpu memory dereference is faster
- One less memory reference compared to existing simple alloc_percpu
- As fast as with static percpu areas, one memory reference less actually.
2. Better memory usage
- Doesn't need a NR_CPUS pointer array for each allocation
- Interlaces objects making better utilization of memory/cachelines
3. Provides truly node local allocation
- The percpu memory with exisiting alloc_percpu does node local allocation, but the NR_CPUS place holder is not node local. This problem doesn't exist with the new implementation.
Design:
We have blocks of memory akin to slabs. Each block has (percpu blocksize) * NR_CPUS of kernel VA space allocated to it. Node local pages are allocated and mapped against the corresponding cpus' VA space. The allocator allocates memory in mulitples of a fixed currency size for an alloc request. The allocator returns address of the percpu object corresponding to cpu0. The cpu local variable for any given cpu can be obtained by simple arithmetic:obj_address + cpu_id * PCPU_BLKSIZE.
Testing:
The block allocator has undergone some userspace stress testing. The remnants of userspace testing still exists in the code as this patch is RFC.
---
include/linux/kernel.h | 2 include/linux/percpu.h | 18 - mm/Makefile | 1 mm/percpu.c | 759 +++++++++++++++++++++++++++++++++++++++++++++++++ mm/slab.c | 69 ---- 5 files changed, 769 insertions(+), 80 deletions(-)
include/linux/kernel.h +2 -0
@@ -27,6 +27,8 @@#define ARRAY_SIZE (x) (sizeof(x) / sizeof((x)[0]))#define ALIGN (x,a) (((x)+(a)-1)&~((a)-1)) +#define IS_ALIGNED (x,a) (!(((a) - 1) & (x))) +#define IS_POWEROFTWO (x) (!(((x) - 1) & (x)))#define KERN_EMERG "<0>"/* system is unusable */ #define KERN_ALERT "<1>"/* action must be taken immediately */
include/linux/percpu.h +7 -11
@@ -15,23 +15,19 @@#define get_cpu_var (var) (*({ preempt_disable(); &__get_cpu_var(var); }))#define put_cpu_var (var) preempt_enable() -#ifdef CONFIG_SMP - -struct percpu_data { -void *ptrs[NR_CPUS ]; -void *blkp; -}; +/* This is the upper bound for an object using alloc_percpu. */ +#define PCPU_BLKSIZE (PAGE_SIZE * 2) +#define PCPU_CURR_SIZE (sizeof (void *)) +#ifdef CONFIG_SMP /* * Use this to get to a cpu's version of the per-cpu object allocated using * alloc_percpu. Non-atomic access to the current CPU's version should * probably be combined with get_cpu()/put_cpu(). */ #define per_cpu_ptr (ptr, cpu) \ -({ \ - struct percpu_data *__p = (struct percpu_data *)~(unsigned long)(ptr); \ - (__typeof__(ptr))__p->ptrs[(cpu)]; \ -}) + ((__typeof__(ptr)) \ + (RELOC_HIDE(ptr, PCPU_BLKSIZE * cpu)))extern void *__alloc_percpu (size_t size,size_t align);extern void free_percpu (const void *);
@@ -56,6 +52,6 @@/* Simple wrapper for the common case: zeros memory. */ #define alloc_percpu (type) \ - ((type *)(__alloc_percpu(sizeof(type), __alignof__(type)))) + ((type *)(__alloc_percpu(ALIGN(sizeof (type), PCPU_CURR_SIZE), __alignof__(type))))#endif /* __LINUX_PERCPU_H */
mm/Makefile +1 -0
@@ -17,4 +17,5 @@ obj-$(CONFIG_NUMA) += mempolicy.o obj-$(CONFIG_SHMEM) += shmem.o obj-$(CONFIG_TINY_SHMEM) += tiny-shmem.o +obj-$(CONFIG_SMP) += percpu.o
mm/percpu.c +759 -0
759 lines not on this page · raw message
mm/slab.c +0 -69
81 lines not on this page · raw message