alloc_percpu() UP optimization
Subsystems (23)
RFC PATCH 1/1 alloc_percpu() UP optimization
Nick Piggin's read-only bind mount optimization patches gave me an idea. For those patches we have an int counter allocated with alloc_percpu(). But, to optimize the UP case, we needed a bunch of #ifdefs to store a plain 'int' instead of a percpu pointer to one.
That seems like a common enough problem that I figured we should just solve it generically. What this patch does is detect when the percpu data can fit into the pointer that we already have for it. If it can, we just use the pointer to store the data intead of dynamically allocating and pointing to it.
This requires an interface change, namely replacing alloc_percpu() with fill_percpu() which takes the address to the pointer instead of returning the newly-allocated structure and assigning it to the pointer.
I'm not super-happy with how this looks with all of the void** and so forth, but it is ugly on a pretty small scale at least. It somehow makes SMP text a wee bit smaller, but bloats UP text by ~140 bytes. (gcc 4.2.4)
This boots for me on i386 UP and SMP.
SMP before/after:
text data bss dec
4188372 1963544 5447680 11599596
4188356 1963544 5447680 11599580
UP before/after:
text data bss dec
3059610 254340 4460544 7774494
3059748 254340 4460544 7774632
---
linux-2.6.git-dave/arch/x86/kernel/acpi/cstate.c | 2 linux-2.6.git-dave/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c | 2 linux-2.6.git-dave/crypto/cryptd.c | 2 linux-2.6.git-dave/drivers/dma/dmaengine.c | 4 - linux-2.6.git-dave/drivers/infiniband/hw/ehca/ehca_irq.c | 2 linux-2.6.git-dave/drivers/net/chelsio/sge.c | 2 linux-2.6.git-dave/drivers/net/loopback.c | 2 linux-2.6.git-dave/drivers/net/veth.c | 2 linux-2.6.git-dave/fs/ext4/mballoc.c | 2 linux-2.6.git-dave/fs/nfs/client.c | 3 linux-2.6.git-dave/fs/nfs/iostat.h | 5 - linux-2.6.git-dave/fs/xfs/xfs_mount.c | 2 linux-2.6.git-dave/include/linux/genhd.h | 2 linux-2.6.git-dave/include/linux/percpu.h | 32 +++++++--- linux-2.6.git-dave/include/scsi/libfc.h | 2 linux-2.6.git-dave/kernel/kexec.c | 2 linux-2.6.git-dave/kernel/sched.c | 2 linux-2.6.git-dave/kernel/srcu.c | 2 linux-2.6.git-dave/kernel/stop_machine.c | 2 linux-2.6.git-dave/kernel/trace/blktrace.c | 6 - linux-2.6.git-dave/kernel/trace/trace_functions_graph.c | 3 linux-2.6.git-dave/kernel/workqueue.c | 4 - linux-2.6.git-dave/lib/percpu_counter.c | 2 linux-2.6.git-dave/net/core/neighbour.c | 2 linux-2.6.git-dave/net/core/sock.c | 2 linux-2.6.git-dave/net/ipv4/af_inet.c | 4 - linux-2.6.git-dave/net/ipv4/route.c | 2 linux-2.6.git-dave/net/ipv4/tcp.c | 2 linux-2.6.git-dave/net/netfilter/nf_conntrack_core.c | 2 linux-2.6.git-dave/net/netfilter/nf_conntrack_ecache.c | 2 linux-2.6.git-dave/net/xfrm/xfrm_ipcomp.c | 5 - 31 files changed, 62 insertions(+), 48 deletions(-)
include/linux/percpu.h renamed from include/linux/percpu.h~alloc-percpu-UP +24 -8
@@ -143,29 +143,45 @@ struct percpu_data {extern void *__alloc_percpu (size_t size,size_t align);extern void free_percpu (void *__pdata); +#define __fill_percpu (pptr, size, align) do { \ + *(pptr) = __alloc_percpu(size, align); \ +} while(0) +#else /* CONFIG_SMP */ -#define per_cpu_ptr (ptr, cpu) ({ (void)(cpu); (ptr); }) +static inline void *__per_cpu_ptr (void *ptr,int cpu,int size) +{ +void **pptr = ptr; +if (size <=sizeof (void *)) +return pptr; +return *pptr; +} +#define per_cpu_ptr (ptr, cpu) ((__typeof__(ptr))__per_cpu_ptr(&ptr, cpu, sizeof(*ptr))) -static inline void *__alloc_percpu (size_t size,size_t align) +static inline void __fill_percpu (void *ptr,size_t size,size_t align) { +void **pptr = (void **)ptr; +if (sizeof (ptr) >= size) { +memset (*pptr,0 , size); +return ; + }/* * Can't easily make larger alignment work with kmalloc. WARN * on it. Larger alignment should only be used for module * percpu sections on SMP for which this path isn't used. */ WARN_ON_ONCE (align >SMP_CACHE_BYTES ); -return kzalloc (size,GFP_KERNEL ); + *pptr =kzalloc (size,GFP_KERNEL ); } -static inline void free_percpu (void *p) -{ -kfree (p); -} +#define free_percpu (ptr) do { \ + if (sizeof(*ptr) > sizeof(ptr)) \ + kfree(ptr); \ +} while(0)#endif /* CONFIG_SMP */ -#define alloc_percpu (type) (type *)__alloc_percpu(sizeof(type), \ +#define fill_percpu (pptr, type) __fill_percpu(pptr, sizeof(type), \ __alignof__(type))/*
arch/x86/kernel/acpi/cstate.c renamed from arch/x86/kernel/acpi/cstate.c~alloc-percpu-UP +1 -1
@@ -156,7 +156,7 @@ static int __init ffh_cstate_init(void)if (c->x86_vendor !=X86_VENDOR_INTEL )return -1 ; - cpu_cstate_entry = alloc_percpu(struct cstate_entry); + fill_percpu(&cpu_cstate_entry, struct cstate_entry);return 0 ; }
arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c renamed from arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c~alloc-percpu-UP +1 -1
@@ -534,7 +534,7 @@ static int __init acpi_cpufreq_early_iniunsigned int i;dprintk ("acpi_cpufreq_early_init\n" ); - acpi_perf_data = alloc_percpu(struct acpi_processor_performance); + fill_percpu(&acpi_perf_data, struct acpi_processor_performance);if (!acpi_perf_data) {dprintk ("Memory allocation error for acpi_perf_data.\n" );return -ENOMEM ;
crypto/cryptd.c renamed from crypto/cryptd.c~alloc-percpu-UP +1 -1
@@ -63,7 +63,7 @@ static int cryptd_init_queue(struct crypint cpu;struct cryptd_cpu_queue *cpu_queue; - queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); + fill_percpu(&queue->cpu_queue, struct cryptd_cpu_queue);if (!queue->cpu_queue)return -ENOMEM ;for_each_possible_cpu (cpu) {
drivers/dma/dmaengine.c renamed from drivers/dma/dmaengine.c~alloc-percpu-UP +2 -2
@@ -302,7 +302,7 @@ static int __init dma_channel_table_initclear_bit (DMA_SLAVE , dma_cap_mask_all.bits);for_each_dma_cap_mask (cap, dma_cap_mask_all) { - channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent); + fill_percpu(&channel_table[cap], struct dma_chan_tbl_ent);if (!channel_table[cap]) { err = -ENOMEM ;break ;
@@ -675,7 +675,7 @@ int dma_async_device_register(struct dma/* represent channels in sysfs. Probably want devs too */ list_for_each_entry (chan, &device->channels, device_node) { rc = -ENOMEM ; - chan->local =alloc_percpu (typeof (*chan->local)); +fill_percpu (&chan->local,typeof (*chan->local));if (chan->local ==NULL ) goto err_out; chan->dev =kzalloc (sizeof (*chan->dev),GFP_KERNEL );
drivers/infiniband/hw/ehca/ehca_irq.c renamed from drivers/infiniband/hw/ehca/ehca_irq.c~alloc-percpu-UP +1 -1
@@ -904,7 +904,7 @@ int ehca_create_comp_pool(void)spin_lock_init (&pool->last_cpu_lock); pool->last_cpu =cpumask_any (cpu_online_mask); - pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task); + fill_percpu(&pool->cpu_comp_tasks, struct ehca_cpu_comp_task);if (pool->cpu_comp_tasks ==NULL ) {kfree (pool);return -EINVAL ;
drivers/net/chelsio/sge.c renamed from drivers/net/chelsio/sge.c~alloc-percpu-UP +1 -1
@@ -2090,7 +2090,7 @@ struct sge * __devinit t1_sge_create(str sge->jumbo_fl =t1_is_T1B (adapter) ?1 :0 ;for_each_port (adapter, i) { - sge->port_stats[i] = alloc_percpu(struct sge_port_stats); + fill_percpu(&sge->port_stats[i], struct sge_port_stats);if (!sge->port_stats[i]) goto nomem_port; }
drivers/net/loopback.c renamed from drivers/net/loopback.c~alloc-percpu-UP +1 -1
@@ -127,7 +127,7 @@ static int loopback_dev_init(struct net_ {struct pcpu_lstats *lstats; - lstats = alloc_percpu(struct pcpu_lstats); + fill_percpu(&lstats, struct pcpu_lstats);if (!lstats)return -ENOMEM ;
drivers/net/veth.c renamed from drivers/net/veth.c~alloc-percpu-UP +1 -1
@@ -282,7 +282,7 @@ static int veth_dev_init(struct net_devistruct veth_net_stats *stats;struct veth_priv *priv; - stats = alloc_percpu(struct veth_net_stats); + fill_percpu(&stats, struct veth_net_stats);if (stats ==NULL )return -ENOMEM ;
fs/ext4/mballoc.c renamed from fs/ext4/mballoc.c~alloc-percpu-UP +1 -1
@@ -2737,7 +2737,7 @@ int ext4_mb_init(struct super_block *sb, sbi->s_mb_history_filter =EXT4_MB_HISTORY_DEFAULT ; sbi->s_mb_group_prealloc =MB_DEFAULT_GROUP_PREALLOC ; - sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); + fill_percpu(&sbi->s_locality_groups, struct ext4_locality_group);if (sbi->s_locality_groups ==NULL ) {kfree (sbi->s_mb_offsets);kfree (sbi->s_mb_maxs);
fs/nfs/client.c renamed from fs/nfs/client.c~alloc-percpu-UP +2 -1
@@ -942,7 +942,8 @@ static struct nfs_server *nfs_alloc_servatomic_set (&server->active,0 ); - server->io_stats =nfs_alloc_iostats (); + fill_percpu(&server->io_stats, struct nfs_iostats); +if (!server->io_stats) {kfree (server);return NULL ;
fs/nfs/iostat.h renamed from fs/nfs/iostat.h~alloc-percpu-UP +0 -5
@@ -75,11 +75,6 @@ static inline void nfs_add_fscache_stats }#endif -static inline struct nfs_iostats *nfs_alloc_iostats (void ) -{ - return alloc_percpu(struct nfs_iostats); -} -static inline void nfs_free_iostats (struct nfs_iostats *stats) {if (stats !=NULL )
fs/xfs/xfs_mount.c renamed from fs/xfs/xfs_mount.c~alloc-percpu-UP +1 -1
@@ -1999,7 +1999,7 @@ xfs_icsb_init_counters(xfs_icsb_cnts_t *cntp;int i; - mp->m_sb_cnts =alloc_percpu (xfs_icsb_cnts_t); +fill_percpu (&mp->m_sb_cnts, xfs_icsb_cnts_t);if (mp->m_sb_cnts ==NULL )return -ENOMEM ;
include/linux/genhd.h renamed from include/linux/genhd.h~alloc-percpu-UP +1 -1
@@ -269,7 +269,7 @@ static inline void part_stat_set_all(strstatic inline int init_part_stats (struct hd_struct *part) { - part->dkstats = alloc_percpu(struct disk_stats); + fill_percpu(&part->dkstats, struct disk_stats);if (!part->dkstats)return 0 ;return 1 ;
include/scsi/libfc.h renamed from include/scsi/libfc.h~alloc-percpu-UP +1 -1
@@ -722,7 +722,7 @@ static inline void fc_lport_state_enter(static inline int fc_lport_init_stats (struct fc_lport *lp) {/* allocate per cpu stats block */ - lp->dev_stats = alloc_percpu(struct fcoe_dev_stats); + fill_percpu(&lp->dev_stats, struct fcoe_dev_stats);if (!lp->dev_stats)return -ENOMEM ;return 0 ;
kernel/kexec.c renamed from kernel/kexec.c~alloc-percpu-UP +1 -1
@@ -1139,7 +1139,7 @@ void crash_save_cpu(struct pt_regs *regsstatic int __init crash_notes_memory_init (void ) {/* Allocate memory for saving cpu registers. */ - crash_notes =alloc_percpu (note_buf_t); +fill_percpu (&crash_notes, note_buf_t);if (!crash_notes) {printk ("Kexec: Memory allocation for saving cpu register" " states failed\n" );
kernel/sched.c renamed from kernel/sched.c~alloc-percpu-UP +1 -1
@@ -10021,7 +10021,7 @@ static struct cgroup_subsys_state *cpuacif (!ca) goto out; - ca->cpuusage =alloc_percpu (u64); +fill_percpu (&ca->cpuusage, u64);if (!ca->cpuusage) goto out_free_ca;
kernel/srcu.c renamed from kernel/srcu.c~alloc-percpu-UP +1 -1
@@ -46,7 +46,7 @@ int init_srcu_struct(struct srcu_struct { sp->completed =0 ;mutex_init (&sp->mutex); - sp->per_cpu_ref = alloc_percpu(struct srcu_struct_array); + fill_percpu(&sp->per_cpu_ref, struct srcu_struct_array);return (sp->per_cpu_ref ?0 : -ENOMEM ); }
kernel/stop_machine.c renamed from kernel/stop_machine.c~alloc-percpu-UP +1 -1
@@ -120,7 +120,7 @@ int stop_machine_create(void) stop_machine_wq =create_rt_workqueue ("kstop" );if (!stop_machine_wq) goto err_out; - stop_machine_work = alloc_percpu(struct work_struct); + fill_percpu(&stop_machine_work, struct work_struct);if (!stop_machine_work) goto err_out; done:
kernel/trace/blktrace.c renamed from kernel/trace/blktrace.c~alloc-percpu-UP +3 -3
@@ -432,11 +432,11 @@ int do_blk_trace_setup(struct request_qureturn -ENOMEM ; ret = -ENOMEM ; - bt->sequence = alloc_percpu(unsigned long); + fill_percpu(&bt->sequence, unsigned long);if (!bt->sequence) goto err; - bt->msg_data =__alloc_percpu (BLK_TN_MAX_MSG , __alignof__(char )); +__fill_percpu (&bt->msg_data,BLK_TN_MAX_MSG , __alignof__(char ));if (!bt->msg_data) goto err;
@@ -1294,7 +1294,7 @@ static int blk_trace_setup_queue(structif (!bt)return -ENOMEM ; - bt->msg_data =__alloc_percpu (BLK_TN_MAX_MSG , __alignof__(char )); +__fill_percpu (&bt->msg_data,BLK_TN_MAX_MSG , __alignof__(char ));if (!bt->msg_data) goto free_bt;
kernel/trace/trace_functions_graph.c renamed from kernel/trace/trace_functions_graph.c~alloc-percpu-UP +2 -1
@@ -844,9 +844,10 @@ static void print_graph_headers(struct sstatic void graph_trace_open (struct trace_iterator *iter) {/* pid and depth on the last trace processed */ - struct fgraph_data *data = alloc_percpu(struct fgraph_data); +struct fgraph_data *data;int cpu; + fill_percpu(&data, struct fgraph_data);if (!data)pr_warning ("function graph tracer: not enough memory\n" );else
kernel/workqueue.c renamed from kernel/workqueue.c~alloc-percpu-UP +2 -2
@@ -669,7 +669,7 @@ int schedule_on_each_cpu(work_func_t funint cpu;struct work_struct *works; - works = alloc_percpu(struct work_struct); + fill_percpu(&works, struct work_struct);if (!works)return -ENOMEM ;
@@ -808,7 +808,7 @@ struct workqueue_struct *__create_workquif (!wq)return NULL ; - wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct); + fill_percpu(&wq->cpu_wq, struct cpu_workqueue_struct);if (!wq->cpu_wq) {kfree (wq);return NULL ;
lib/percpu_counter.c renamed from lib/percpu_counter.c~alloc-percpu-UP +1 -1
@@ -72,7 +72,7 @@ int __percpu_counter_init(struct percpu_spin_lock_init (&fbc->lock);lockdep_set_class (&fbc->lock, key); fbc->count = amount; - fbc->counters =alloc_percpu (s32); +fill_percpu (&fbc->counters, s32);if (!fbc->counters)return -ENOMEM ;#ifdef CONFIG_HOTPLUG_CPU
net/core/neighbour.c renamed from net/core/neighbour.c~alloc-percpu-UP +1 -1
@@ -1411,7 +1411,7 @@ void neigh_table_init_no_netlink(structkmem_cache_create (tbl->id, tbl->entry_size,0 ,SLAB_HWCACHE_ALIGN |SLAB_PANIC ,NULL ); - tbl->stats = alloc_percpu(struct neigh_statistics); + fill_percpu(&tbl->stats, struct neigh_statistics);if (!tbl->stats)panic ("cannot create neighbour cache statistics" );
net/core/sock.c renamed from net/core/sock.c~alloc-percpu-UP +1 -1
@@ -2025,7 +2025,7 @@ EXPORT_SYMBOL_GPL(sock_prot_inuse_get);static int sock_inuse_init_net (struct net *net) { - net->core.inuse = alloc_percpu(struct prot_inuse); + fill_percpu(&net->core.inuse, struct prot_inuse);return net->core.inuse ?0 : -ENOMEM ; }
net/ipv4/af_inet.c renamed from net/ipv4/af_inet.c~alloc-percpu-UP +2 -2
@@ -1375,10 +1375,10 @@ EXPORT_SYMBOL_GPL(snmp_fold_field);int snmp_mib_init (void *ptr[2 ],size_t mibsize) {BUG_ON (ptr ==NULL ); - ptr[0 ] =__alloc_percpu (mibsize, __alignof__(unsigned long long )); +__fill_percpu (&ptr[0 ], mibsize, __alignof__(unsigned long long ));if (!ptr[0 ]) goto err0; - ptr[1 ] =__alloc_percpu (mibsize, __alignof__(unsigned long long )); +__fill_percpu (&ptr[1 ], mibsize, __alignof__(unsigned long long ));if (!ptr[1 ]) goto err1;return 0 ;
net/ipv4/route.c renamed from net/ipv4/route.c~alloc-percpu-UP +1 -1
@@ -3377,7 +3377,7 @@ int __init ip_rt_init(void)int rc =0 ;#ifdef CONFIG_NET_CLS_ROUTE - ip_rt_acct =__alloc_percpu (256 *sizeof (struct ip_rt_acct ), __alignof__(struct ip_rt_acct )); +__fill_percpu (&ip_rt_acct,256 *sizeof (struct ip_rt_acct ), __alignof__(struct ip_rt_acct ));if (!ip_rt_acct)panic ("IP: failed to allocate ip_rt_acct\n" );#endif
net/ipv4/tcp.c renamed from net/ipv4/tcp.c~alloc-percpu-UP +1 -1
@@ -2646,7 +2646,7 @@ static struct tcp_md5sig_pool **__tcp_alint cpu;struct tcp_md5sig_pool **pool; - pool = alloc_percpu(struct tcp_md5sig_pool *); + fill_percpu(&pool, struct tcp_md5sig_pool *);if (!pool)return NULL ;
net/netfilter/nf_conntrack_core.c renamed from net/netfilter/nf_conntrack_core.c~alloc-percpu-UP +1 -1
@@ -1226,7 +1226,7 @@ static int nf_conntrack_init_net(structatomic_set (&net->ct.count,0 );INIT_HLIST_NULLS_HEAD (&net->ct.unconfirmed,0 ); - net->ct.stat = alloc_percpu(struct ip_conntrack_stat); + fill_percpu(&net->ct.stat, struct ip_conntrack_stat);if (!net->ct.stat) { ret = -ENOMEM ; goto err_stat;
net/netfilter/nf_conntrack_ecache.c renamed from net/netfilter/nf_conntrack_ecache.c~alloc-percpu-UP +1 -1
@@ -100,7 +100,7 @@ void nf_ct_event_cache_flush(struct netint nf_conntrack_ecache_init (struct net *net) { - net->ct.ecache = alloc_percpu(struct nf_conntrack_ecache); + fill_percpu(&net->ct.ecache, struct nf_conntrack_ecache);if (!net->ct.ecache)return -ENOMEM ;return 0 ;
net/xfrm/xfrm_ipcomp.c renamed from net/xfrm/xfrm_ipcomp.c~alloc-percpu-UP +3 -2
@@ -223,7 +223,7 @@ static void **ipcomp_alloc_scratches(voiif (ipcomp_scratch_users++)return ipcomp_scratches; - scratches = alloc_percpu(void *); + fill_percpu(&scratches, void *);if (!scratches)return NULL ;
@@ -296,9 +296,10 @@ static struct crypto_comp **ipcomp_allocINIT_LIST_HEAD (&pos->list);list_add (&pos->list, &ipcomp_tfms_list); - pos->tfms = tfms = alloc_percpu(struct crypto_comp *); + fill_percpu(&tfms, struct crypto_comp *);if (!tfms) goto error; + pos->tfms = tfms;for_each_possible_cpu (cpu) { struct crypto_comp *tfm = crypto_alloc_comp(alg_name, 0,
_