bitops: Ensure the compile time HWEIGHT is only used for such
Subsystems (7)
Commit-ID: fce877e3a429940a986e085a41e8b57f2d922e36 Gitweb: http://git.kernel.org/tip/fce877e3a429940a986e085a41e8b57f2d922e36 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 29 Jan 2010 13:25:12 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Thu, 4 Feb 2010 09:59:50 +0100
bitops: Ensure the compile time HWEIGHT is only used for such
Avoid accidental misuse by failing to compile things
LKML-Reference: <new-submission>
--- arch/x86/kernel/cpu/perf_event.c | 10 +++++++--- include/linux/bitops.h | 33 ++++++++++++++++++++++----------- 2 files changed, 29 insertions(+), 14 deletions(-)
arch/x86/kernel/cpu/perf_event.c +7 -3
@@ -93,13 +93,16 @@ struct cpu_hw_events {struct perf_event *event_list[X86_PMC_IDX_MAX ];/* in enabled order */ }; -#define EVENT_CONSTRAINT (c, n, m) { \ +#define __EVENT_CONSTRAINT (c, n, m, w) {\ { .idxmsk64[0] = (n) }, \ .code = (c), \ .cmask = (m), \ - .weight = HWEIGHT64((u64)(n)), \ + .weight = (w), \ } +#define EVENT_CONSTRAINT (c, n, m) \ + __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n)) +#define INTEL_EVENT_CONSTRAINT (c, n) \ EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVTSEL_MASK)
@@ -2622,7 +2625,8 @@ void __init init_hw_perf_events(void)register_die_notifier (&perf_event_nmi_notifier); unconstrained = (struct event_constraint ) -EVENT_CONSTRAINT (0 , (1ULL << x86_pmu.num_events) -1 ,0 ); +__EVENT_CONSTRAINT (0 , (1ULL << x86_pmu.num_events) -1 , +0 , x86_pmu.num_events);pr_info ("... version: %d\n" , x86_pmu.version);pr_info ("... bit width: %d\n" , x86_pmu.event_bits);
include/linux/bitops.h +22 -11
@@ -45,19 +45,30 @@ static inline unsigned long hweight_long(unsigned long w)return sizeof (w) ==4 ?hweight32 (w) :hweight64 (w); } -#define HWEIGHT8 (w) \ - ( (!!((w) & (1ULL << 0))) + \ - (!!((w) & (1ULL << 1))) + \ - (!!((w) & (1ULL << 2))) + \ - (!!((w) & (1ULL << 3))) + \ - (!!((w) & (1ULL << 4))) + \ - (!!((w) & (1ULL << 5))) + \ - (!!((w) & (1ULL << 6))) + \ +/* +* Clearly slow versions of the hweightN() functions, their benefit is +* of course compile time evaluation of constant arguments. +*/ +#define HWEIGHT8 (w) \ + ( BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) + \ + (!!((w) & (1ULL << 0))) + \ + (!!((w) & (1ULL << 1))) + \ + (!!((w) & (1ULL << 2))) + \ + (!!((w) & (1ULL << 3))) + \ + (!!((w) & (1ULL << 4))) + \ + (!!((w) & (1ULL << 5))) + \ + (!!((w) & (1ULL << 6))) + \ (!!((w) & (1ULL << 7))) ) -#define HWEIGHT16 (w) (HWEIGHT8(w) + HWEIGHT8(w >> 8)) -#define HWEIGHT32 (w) (HWEIGHT16(w) + HWEIGHT16(w >> 16)) -#define HWEIGHT64 (w) (HWEIGHT32(w) + HWEIGHT32(w >> 32)) +#define HWEIGHT16 (w) (HWEIGHT8(w) + HWEIGHT8((w) >> 8)) +#define HWEIGHT32 (w) (HWEIGHT16(w) + HWEIGHT16((w) >> 16)) +#define HWEIGHT64 (w) (HWEIGHT32(w) + HWEIGHT32((w) >> 32)) + +/* +* Type invariant version that simply casts things to the +* largest type. +*/ +#define HWEIGHT (w) HWEIGHT64((u64)(w))/** * rol32 - rotate a 32-bit value left
Peter Zijlstra
maintainerPATCH 1/1 bitops: Optimize hweight() by making use of compile-time evaluation
Hi Andrew,
Ingo suggested you take this patch through -mm since it touches !x86 arch bits.
It should apply nicely once fce877e3a429940a986e085a41e8b57f2d922e36 makes in into -next.
---
Subject: bitops: Optimize hweight() by making use of compile-time evaluation
From: Peter Zijlstra <[email protected]>
Date: Mon Feb 01 15:03:07 CET 2010
Rename the extisting runtime hweight() implementations to __arch_hweight(), rename the compile-time versions to __const_hweight() and then have hweight() pick between them.
LKML-Reference: <1265028224.24455.154.camel@laptop>
--- arch/alpha/include/asm/bitops.h | 18 ++++++------ arch/ia64/include/asm/bitops.h | 11 ++++--- arch/sparc/include/asm/bitops_64.h | 11 ++++--- include/asm-generic/bitops/arch_hweight.h | 11 +++++++ include/asm-generic/bitops/const_hweight.h | 42 +++++++++++++++++++++++++++++ include/asm-generic/bitops/hweight.h | 8 +---- include/linux/bitops.h | 25 ----------------- lib/hweight.c | 19 ++++++------- 8 files changed, 87 insertions(+), 58 deletions(-)
arch/alpha/include/asm/bitops.h +10 -8
@@ -405,29 +405,31 @@ static inline int fls(int x)#if defined(CONFIG_ALPHA_EV6 ) && defined(CONFIG_ALPHA_EV67 )/* Whee. EV67 can calculate it directly. */ -static inline unsigned long hweight64 (unsigned long w) +static inline unsigned long __arch_hweight64 (unsigned long w) {return __kernel_ctpop (w); } -static inline unsigned int hweight32 (unsigned int w) +static inline unsigned int __arch_weight32 (unsigned int w) { -return hweight64 (w); +return __arch_hweight64 (w); } -static inline unsigned int hweight16 (unsigned int w) +static inline unsigned int __arch_hweight16 (unsigned int w) { -return hweight64 (w &0xffff ); +return __arch_hweight64 (w &0xffff ); } -static inline unsigned int hweight8 (unsigned int w) +static inline unsigned int __arch_hweight8 (unsigned int w) { -return hweight64 (w &0xff ); +return __arch_hweight64 (w &0xff ); }#else -#include <asm-generic/bitops/hweight.h> +#include <asm-generic/bitops/arch_hweight.h> #endif +#include <asm-generic/bitops/const_hweight.h> +#endif /* __KERNEL__ */ #include <asm-generic/bitops/find.h>
arch/ia64/include/asm/bitops.h +6 -5
@@ -437,17 +437,18 @@ __fls (unsigned long x)* hweightN: returns the hamming weight (i.e. the number * of bits set) of a N-bit word */ -static __inline__unsigned long -hweight64 (unsigned long x) +static __inline__unsigned long __arch_hweight64 (unsigned long x) {unsigned long result; result =ia64_popcnt (x);return result; } -#define hweight32 (x) (unsigned int) hweight64((x) & 0xfffffffful) -#define hweight16 (x) (unsigned int) hweight64((x) & 0xfffful) -#define hweight8 (x) (unsigned int) hweight64((x) & 0xfful) +#define __arch_hweight32 (x) ((unsigned int) __arch_hweight64((x) & 0xfffffffful)) +#define __arch_hweight16 (x) ((unsigned int) __arch_hweight64((x) & 0xfffful)) +#define __arch_hweight8 (x) ((unsigned int) __arch_hweight64((x) & 0xfful)) + +#include <asm-generic/bitops/const_hweight.h> #endif /* __KERNEL__ */
arch/sparc/include/asm/bitops_64.h +6 -5
@@ -44,7 +44,7 @@ extern void change_bit(unsigned long nr,#ifdef ULTRA_HAS_POPULATION_COUNT -static inline unsigned int hweight64 (unsigned long w) +static inline unsigned int __arch_hweight64 (unsigned long w) {unsigned int res;
@@ -52,7 +52,7 @@ static inline unsigned int hweight64(unsreturn res; } -static inline unsigned int hweight32 (unsigned int w) +static inline unsigned int __arch_hweight32 (unsigned int w) {unsigned int res;
@@ -60,7 +60,7 @@ static inline unsigned int hweight32(unsreturn res; } -static inline unsigned int hweight16 (unsigned int w) +static inline unsigned int __arch_hweight16 (unsigned int w) {unsigned int res;
@@ -68,7 +68,7 @@ static inline unsigned int hweight16(unsreturn res; } -static inline unsigned int hweight8 (unsigned int w) +static inline unsigned int __arch_hweight8 (unsigned int w) {unsigned int res;
@@ -78,9 +78,10 @@ static inline unsigned int hweight8(unsi#else -#include <asm-generic/bitops/hweight.h> +#include <asm-generic/bitops/arch_hweight.h> #endif +#include <asm-generic/bitops/const_hweight.h> #include <asm-generic/bitops/lock.h> #endif /* __KERNEL__ */
include/asm-generic/bitops/arch_hweight.h new file +11 -0
@@ -0,0 +1,11 @@ +#ifndef _ASM_GENERIC_BITOPS_ARCH_HWEIGHT_H_ +#define _ASM_GENERIC_BITOPS_ARCH_HWEIGHT_H_ + +#include <asm/types.h> + +extern unsigned int __arch_hweight32 (unsigned int w); +extern unsigned int __arch_hweight16 (unsigned int w); +extern unsigned int __arch_hweight8 (unsigned int w); +extern unsigned long __arch_hweight64 (__u64 w); + +#endif /* _ASM_GENERIC_BITOPS_HWEIGHT_H_ */
include/asm-generic/bitops/const_hweight.h new file +42 -0
42 lines not on this page · raw message
include/asm-generic/bitops/hweight.h +2 -6
@@ -1,11 +1,7 @@#ifndef _ASM_GENERIC_BITOPS_HWEIGHT_H_#define _ASM_GENERIC_BITOPS_HWEIGHT_H_ -#include <asm/types.h> - -extern unsigned int hweight32 (unsigned int w); -extern unsigned int hweight16 (unsigned int w); -extern unsigned int hweight8 (unsigned int w); -extern unsigned long hweight64 (__u64 w); +#include <asm-generic/bitops/arch_hweight.h> +#include <asm-generic/bitops/const_hweight.h> #endif /* _ASM_GENERIC_BITOPS_HWEIGHT_H_ */
include/linux/bitops.h +0 -25
@@ -45,31 +45,6 @@ static inline unsigned long hweight_longreturn sizeof (w) ==4 ?hweight32 (w) :hweight64 (w); } -/* -* Clearly slow versions of the hweightN() functions, their benefit is -* of course compile time evaluation of constant arguments. -*/ -#define HWEIGHT8 (w) \ - ( BUILD_BUG_ON_ZERO(!__builtin_constant_p(w)) + \ - (!!((w) & (1ULL << 0))) + \ - (!!((w) & (1ULL << 1))) + \ - (!!((w) & (1ULL << 2))) + \ - (!!((w) & (1ULL << 3))) + \ - (!!((w) & (1ULL << 4))) + \ - (!!((w) & (1ULL << 5))) + \ - (!!((w) & (1ULL << 6))) + \ - (!!((w) & (1ULL << 7))) ) - -#define HWEIGHT16 (w) (HWEIGHT8(w) + HWEIGHT8((w) >> 8)) -#define HWEIGHT32 (w) (HWEIGHT16(w) + HWEIGHT16((w) >> 16)) -#define HWEIGHT64 (w) (HWEIGHT32(w) + HWEIGHT32((w) >> 32)) - -/* -* Type invariant version that simply casts things to the -* largest type. -*/ -#define HWEIGHT (w) HWEIGHT64((u64)(w)) -/** * rol32 - rotate a 32-bit value left * @word: value to rotate
lib/hweight.c +10 -9
@@ -9,7 +9,7 @@* The Hamming Weight of a number is the total number of bits set in it. */ -unsigned int hweight32 (unsigned int w) +unsigned int __arch_hweight32 (unsigned int w) {#ifdef ARCH_HAS_FAST_MULTIPLIER w -= (w >>1 ) &0x55555555 ;
@@ -24,29 +24,30 @@ unsigned int hweight32(unsigned int w)return (res + (res >>16 )) &0x000000FF ;#endif } -EXPORT_SYMBOL (hweight32); +EXPORT_SYMBOL (__arch_hweight32); -unsigned int hweight16 (unsigned int w) +unsigned int __arch_hweight16 (unsigned int w) {unsigned int res = w - ((w >>1 ) &0x5555 ); res = (res &0x3333 ) + ((res >>2 ) &0x3333 ); res = (res + (res >>4 )) &0x0F0F ;return (res + (res >>8 )) &0x00FF ; } -EXPORT_SYMBOL (hweight16); +EXPORT_SYMBOL (__arch_hweight16); -unsigned int hweight8 (unsigned int w) +unsigned int __arch_hweight8 (unsigned int w) {unsigned int res = w - ((w >>1 ) &0x55 ); res = (res &0x33 ) + ((res >>2 ) &0x33 );return (res + (res >>4 )) &0x0F ; } -EXPORT_SYMBOL (hweight8); +EXPORT_SYMBOL (__arch_hweight8); -unsigned long hweight64 (__u64 w) +unsigned long __arch_hweight64 (__u64 w) {#if BITS_PER_LONG ==32 -return hweight32 ((unsigned int )(w >>32 )) +hweight32 ((unsigned int )w); +return __arch_hweight32 ((unsigned int )(w >>32 )) + +__arch_hweight32 ((unsigned int )w);#elif BITS_PER_LONG ==64 #ifdef ARCH_HAS_FAST_MULTIPLIER w -= (w >>1 ) &0x5555555555555555ul ;
@@ -63,4 +64,4 @@ unsigned long hweight64(__u64 w)#endif #endif } -EXPORT_SYMBOL (hweight64); +EXPORT_SYMBOL (__arch_hweight64);