Immediates Values (real variables)

Index: linux-2.6-lttng/kernel/module.c
===================================================================
@@ -65,8 +65,8 @@
 static DEFINE_SPINLOCK(modlist_lock);
 
 /* List of modules, protected by module_mutex AND modlist_lock */
-static DEFINE_MUTEX(module_mutex);
-static LIST_HEAD(modules);
+DEFINE_MUTEX(module_mutex);
+LIST_HEAD(modules);
 static DECLARE_MUTEX(notify_mutex);
 
 static BLOCKING_NOTIFIER_HEAD(module_notify_list);

Index: linux-2.6-lttng/include/linux/module.h
===================================================================
@@ -60,6 +60,10 @@
 	struct kobject *drivers_dir;
 };
 
+/* Protects the list of modules. */
+extern struct mutex module_mutex;
+extern struct list_head modules;
+
 /* These are either module local, or the kernel's dummy ones. */
 extern int init_module(void);
 extern void cleanup_module(void);

Index: linux-2.6-lttng/include/linux/immediate.h
===================================================================
@@ -0,0 +1,53 @@
+#ifndef _LINUX_IMMEDIATE_H
+#define _LINUX_IMMEDIATE_H
+
+/*
+ * Immediate values, can be updated at runtime and save cache lines.
+ *
+ * (C) Copyright 2007 Mathieu Desnoyers <[email protected]>
+ *
+ * This file is released under the GPLv2.
+ * See the file COPYING for more details.
+ */
+
+#ifdef __KERNEL__
+
+#ifdef CONFIG_IMMEDIATE
+#include <asm/immediate.h>
+#else
+#include <asm-generic/immediate.h>
+#endif
+
+/*
+ * Always access this type with the provided functions.
+ */
+#define DEFINE_IMMEDIATE_TYPE(type, name) \
+	typedef struct { type value; } name
+
+/*
+ * Standard pre-defined immediate types.
+ */
+DEFINE_IMMEDIATE_TYPE(char, immediate_char_t);
+DEFINE_IMMEDIATE_TYPE(short, immediate_short_t);
+DEFINE_IMMEDIATE_TYPE(int, immediate_int_t);
+DEFINE_IMMEDIATE_TYPE(long, immediate_long_t);
+DEFINE_IMMEDIATE_TYPE(void*, immediate_void_ptr_t);
+
+/*
+ * Use this macro to initialize an immediate value to an initial static value.
+ */
+#define IMMEDIATE_INIT(i)		{ (i) }
+
+/*
+ * Force a data read of the immediate value instead of the immediate value
+ * based mechanism. Useful for __init and __exit section data read.
+ */
+#define _immediate_read(var)		(var)->value
+
+/*
+ * Force the use of a normal if () statement depending on an immediate value.
+ */
+#define _immediate_if(var)		if (_immediate_read(var))
+
+#endif /* __KERNEL__ */
+#endif

Index: linux-2.6-lttng/include/asm-generic/vmlinux.lds.h
===================================================================
@@ -122,6 +122,13 @@
 		VMLINUX_SYMBOL(__stop___kcrctab_gpl_future) = .;	\
 	}								\
 									\
+	/* Immediate values: pointers */				\
+	__immediate : AT(ADDR(__immediate) - LOAD_OFFSET) {		\
+		VMLINUX_SYMBOL(__start___immediate) = .;		\
+		*(__immediate)						\
+		VMLINUX_SYMBOL(__stop___immediate) = .;			\
+	}								\
+									\
 	/* Kernel symbol table: strings */				\
         __ksymtab_strings : AT(ADDR(__ksymtab_strings) - LOAD_OFFSET) {	\
 		*(__ksymtab_strings)					\

Index: linux-2.6-lttng/include/linux/module.h
===================================================================
@@ -15,6 +15,7 @@
 #include <linux/stringify.h>
 #include <linux/kobject.h>
 #include <linux/moduleparam.h>
+#include <linux/immediate.h>
 #include <asm/local.h>
 
 #include <asm/module.h>

@@ -374,6 +375,11 @@
 	/* The command line arguments (may be mangled).  People like
 	   keeping pointers to this stuff */
 	char *args;
+
+#ifdef CONFIG_IMMEDIATE
+	const struct __immediate *immediate;
+	unsigned int num_immediate;
+#endif
 };
 #ifndef MODULE_ARCH_INIT
 #define MODULE_ARCH_INIT {}

Index: linux-2.6-lttng/kernel/module.c
===================================================================
@@ -32,6 +32,7 @@
 #include <linux/cpu.h>
 #include <linux/moduleparam.h>
 #include <linux/errno.h>
+#include <linux/immediate.h>
 #include <linux/err.h>
 #include <linux/vermagic.h>
 #include <linux/notifier.h>

@@ -1623,6 +1624,7 @@
 	unsigned int unusedcrcindex;
 	unsigned int unusedgplindex;
 	unsigned int unusedgplcrcindex;
+	unsigned int immediateindex = 0;
 	struct module *mod;
 	long err = 0;
 	void *percpu = NULL, *ptr = NULL; /* Stops spurious gcc warning */

@@ -1719,6 +1721,9 @@
 #ifdef ARCH_UNWIND_SECTION_NAME
 	unwindex = find_sec(hdr, sechdrs, secstrings, ARCH_UNWIND_SECTION_NAME);
 #endif
+#ifdef CONFIG_IMMEDIATE
+	immediateindex = find_sec(hdr, sechdrs, secstrings, "__immediate");
+#endif
 
 	/* Don't keep modinfo section */
 	sechdrs[infoindex].sh_flags &= ~(unsigned long)SHF_ALLOC;

@@ -1729,6 +1734,8 @@
 #endif
 	if (unwindex)
 		sechdrs[unwindex].sh_flags |= SHF_ALLOC;
+	if (immediateindex)
+		sechdrs[immediateindex].sh_flags |= SHF_ALLOC;
 
 	/* Check module struct version now, before we try to use module. */
 	if (!check_modstruct_version(sechdrs, versindex, mod)) {

@@ -1869,6 +1876,13 @@
 	mod->gpl_future_syms = (void *)sechdrs[gplfutureindex].sh_addr;
 	if (gplfuturecrcindex)
 		mod->gpl_future_crcs = (void *)sechdrs[gplfuturecrcindex].sh_addr;
+#ifdef CONFIG_IMMEDIATE
+	if (immediateindex) {
+		mod->immediate = (void *)sechdrs[immediateindex].sh_addr;
+		mod->num_immediate =
+			sechdrs[immediateindex].sh_size / sizeof(*mod->immediate);
+	}
+#endif
 
 	mod->unused_syms = (void *)sechdrs[unusedindex].sh_addr;
 	if (unusedcrcindex)

@@ -1934,6 +1948,8 @@
         }
 #endif
 
+	module_immediate_setup(mod);
+
 	err = module_finalize(hdr, sechdrs, mod);
 	if (err < 0)
 		goto cleanup;

119 lines not on this page · raw message

Index: linux-2.6-lttng/init/main.c
===================================================================
@@ -56,6 +56,7 @@
 #include <linux/pid_namespace.h>
 #include <linux/device.h>
 #include <linux/kthread.h>
+#include <linux/immediate.h>
 
 #include <asm/io.h>
 #include <asm/bugs.h>

@@ -521,6 +522,7 @@
 	unwind_init();
 	lockdep_init();
 	container_init_early();
+	immediate_update_early();
 
 	local_irq_disable();
 	early_boot_irqs_off();

Index: linux-2.6-lttng/include/linux/immediate.h
===================================================================
@@ -10,12 +10,23 @@
  * See the file COPYING for more details.
  */
 
-#ifdef __KERNEL__
-
 #ifdef CONFIG_IMMEDIATE
 #include <asm/immediate.h>
 #else
-#include <asm-generic/immediate.h>
+/*
+ * Generic immediate values: a simple, standard, memory load.
+ */
+
+struct module;
+
+#define immediate_read(var)		_immediate_read(var)
+#define immediate_set(var, i)		((var)->value = (i))
+#define _immediate_set(var, i)		immediate_set(var, i)
+#define immediate_set_early(var, i)	immediate_set(var, i)
+#define immediate_if(var)		if (immediate_read(var))
+
+static inline void module_immediate_setup(struct module *mod) { }
+static inline void immediate_update_early(void) { }
 #endif
 
 /*

@@ -49,5 +60,4 @@ DEFINE_IMMEDIATE_TYPE(void*, immediate_v
  */
 #define _immediate_if(var)		if (_immediate_read(var))
 
-#endif /* __KERNEL__ */
 #endif

Index: linux-2.6-lttng/kernel/immediate.c
===================================================================
@@ -27,13 +27,13 @@ extern const struct __immediate __stop__
  * modules_mutex nests inside immediate_mutex. immediate_mutex protects builtin
  * immediates and module immediates.
  */
-DEFINE_MUTEX(immediate_mutex);
+static DEFINE_MUTEX(immediate_mutex);
 
 /*
  * Sets a range of immediates to a enabled state : set the enable bit.
  */
-static void _immediate_update_range(
-	const struct __immediate *begin, const struct __immediate *end)
+static inline void _immediate_update_range(const struct __immediate *begin,
+		const struct __immediate *end)
 {
 	const struct __immediate *iter;
 	int ret;

@@ -63,11 +63,10 @@ static void _immediate_update_range(
  */
 void module_immediate_setup(struct module *mod)
 {
-	_immediate_update_range(mod->immediate, mod->immediate+mod->num_immediate);
+	_immediate_update_range(mod->immediate,
+				mod->immediate+mod->num_immediate);
 }
-#endif
 
-#ifdef CONFIG_MODULES
 /*
  * immediate mutex nests inside the modules mutex.
  */

@@ -103,8 +102,8 @@ EXPORT_SYMBOL_GPL(immediate_update);
  * Update the immediate values to the state of the variables they refer to. It
  * is done before SMP is active, at the very beginning of start_kernel().
  */
-void __init immediate_update_early_range(
-	const struct __immediate *begin, const struct __immediate *end)
+void __init immediate_update_early_range(const struct __immediate *begin,
+		const struct __immediate *end)
 {
 	const struct __immediate *iter;
 

Index: linux-2.6-lttng/include/linux/immediate.h
===================================================================
@@ -19,18 +19,64 @@
 
 struct module;
 
+/**
+ * immediate_read - read immediate variable
+ * @var: pointer of type immediate_*_t
+ *
+ * Reads the value of @var.
+ */
 #define immediate_read(var)		_immediate_read(var)
+
+/**
+ * immediate_set - set immediate variable (with locking)
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var, taking the module_mutex if required by
+ * the architecture.
+ */
 #define immediate_set(var, i)		((var)->value = (i))
+
+/**
+ * _immediate_set - set immediate variable (without locking)
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var. Must be called with module_mutex held.
+ */
 #define _immediate_set(var, i)		immediate_set(var, i)
+
+/**
+ * immediate_set_early - set immediate variable at early boot
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var. Should be used for early boot updates.
+ */
 #define immediate_set_early(var, i)	immediate_set(var, i)
+
+/**
+ * immediate_if - if () statement depending on an immediate value
+ * @var: pointer of type immediate_*_t
+ *
+ * Use as an if () statement depending on an immediate value.
+ */
 #define immediate_if(var)		if (immediate_read(var))
 
+/*
+ * Internal update functions.
+ */
 static inline void module_immediate_setup(struct module *mod) { }
 static inline void immediate_update_early(void) { }
 #endif
 
-/*
- * Always access this type with the provided functions.
+/**
+ * DEFINE_IMMEDIATE_TYPE - Define an immediate type
+ * @type: type that the immediate should hold
+ * @name: name of the immediate type
+ *
+ * Define new immediate types. Naming scheme is immediate_*_t.
+ * Always access these types with the provided functions.
  */
 #define DEFINE_IMMEDIATE_TYPE(type, name) \
 	typedef struct { type value; } name

@@ -44,18 +90,28 @@ DEFINE_IMMEDIATE_TYPE(int, immediate_int
 DEFINE_IMMEDIATE_TYPE(long, immediate_long_t);
 DEFINE_IMMEDIATE_TYPE(void*, immediate_void_ptr_t);
 
-/*
- * Use this macro to initialize an immediate value to an initial static value.
+/**
+ * IMMEDIATE_INIT - Static initialization of an immediate variable
+ * @i: required value
+ *
+ * Use this macro to initialize an immediate value to an initial static
+ * value.
  */
 #define IMMEDIATE_INIT(i)		{ (i) }
 
-/*
+/**
+ * _immediate_read - Read immediate value with standard memory load.
+ * @var: pointer of type immediate_*_t
+ *
  * Force a data read of the immediate value instead of the immediate value
  * based mechanism. Useful for __init and __exit section data read.
  */
 #define _immediate_read(var)		(var)->value
 
 /*
+ * _immediate_if - if () statement depending on immediate value (memory load)
+ * @var: pointer of type immediate_*_t
+ *
  * Force the use of a normal if () statement depending on an immediate value.
  */
 #define _immediate_if(var)		if (_immediate_read(var))

Index: linux-2.6-lttng/kernel/immediate.c
===================================================================
@@ -54,7 +54,10 @@ static inline void _immediate_update_ran
 }
 
 #ifdef CONFIG_MODULES
-/*
+/**
+ * module_immediate_setup - Update immediate values in a module
+ * @mod: pointer to the struct module
+ *
  * Setup the immediate according to the variable upon which it depends.  Called
  * by load_module with module_mutex held. This mutex protects against concurrent
  * modifications to modules'immediates. Therefore, since

@@ -89,6 +92,12 @@ static inline void immediate_update_modu
 static inline void immediate_update_modules(int lock) { }
 #endif
 
+/**
+ * immediate_update - update all immediate values in the kernel
+ * @lock: should a module_mutex be taken ?
+ *
+ * Iterate on the kernel core and modules to update the immediate values.
+ */
 void immediate_update(int lock)
 {
 	/* Core kernel immediates */

@@ -98,11 +107,7 @@ void immediate_update(int lock)
 }
 EXPORT_SYMBOL_GPL(immediate_update);
 
-/*
- * Update the immediate values to the state of the variables they refer to. It
- * is done before SMP is active, at the very beginning of start_kernel().
- */
-void __init immediate_update_early_range(const struct __immediate *begin,
+static void __init immediate_update_early_range(const struct __immediate *begin,
 		const struct __immediate *end)
 {
 	const struct __immediate *iter;

@@ -111,7 +116,12 @@ void __init immediate_update_early_range
 		arch_immediate_update_early(iter);
 }
 
-
+/**
+ * immediate_update_early - Update immediate values at boot time
+ *
+ * Update the immediate values to the state of the variables they refer to. It
+ * is done before SMP is active, at the very beginning of start_kernel().
+ */
 void __init immediate_update_early(void)
 {
 	immediate_update_early_range(__start___immediate, __stop___immediate);

Index: linux-2.6-lttng/kernel/marker.c
===================================================================
@@ -30,11 +30,21 @@ extern struct __mark_marker __stop___mar
 
 /*
  * module_mutex nests inside markers_mutex. Markers mutex protects the builtin
- * and module markers, and the hash table.
+ * and module markers, the hash table and deferred_sync.
  */
 DEFINE_MUTEX(markers_mutex);
 
 /*
+ * Marker deferred synchronization.
+ * Upon marker probe_unregister, we delay call to synchronize_sched() to
+ * accelerate mass unregistration (only when there is no more reference to a
+ * give module do we call synchronize_sched()). However, we need to make sure
+ * every critical region have ended before we re-arm a marker that has been
+ * unregistered and then registered back with a different probe data.
+ */
+static int deferred_sync;
+
+/*
  * Marker hash table, containing the active markers.
  * Protected by module_mutex.
  */

@@ -303,8 +313,10 @@ static inline void __marker_update_probe
 			__stop___markers, probe_module, &refcount);
 	/* Markers in modules. */
 	__marker_update_probes_modules(probe_module, &refcount);
-	if (probe_module && refcount == 0)
+	if (probe_module && refcount == 0) {
 		synchronize_sched();
+		deferred_sync = 0;
+	}
 }
 
 #ifdef CONFIG_MODULES

@@ -353,6 +365,10 @@ int marker_probe_register(const char *na
 		ret = -EBUSY;
 		goto end;
 	}
+	if (deferred_sync) {
+		synchronize_sched();
+		deferred_sync = 0;
+	}
 	ret = _add_marker(name, format, probe, pdata);
 	if (ret)
 		goto end;

@@ -384,6 +400,7 @@ void *marker_probe_unregister(const char
 	/* In what module is the probe handler ? */
 	probe_module = __module_text_address((unsigned long)entry->probe);
 	pdata = _remove_marker(name);
+	deferred_sync = 1;
 	_marker_update_probes(probe_module);
 end:
 	mutex_unlock(&markers_mutex);

@@ -425,6 +442,7 @@ iter_end:
 	/* In what module is the probe handler ? */
 	probe_module = __module_text_address((unsigned long)entry->probe);
 	pdata = _remove_marker(entry->name);
+	deferred_sync = 1;
 	_marker_update_probes(probe_module);
 end:
 	mutex_unlock(&markers_mutex);

Index: linux-2.6-lttng/include/asm-generic/immediate.h
===================================================================
@@ -0,0 +1,19 @@
+#ifndef _ASM_GENERIC_IMMEDIATE_H
+#define _ASM_GENERIC_IMMEDIATE_H
+
+/*
+ * Generic immediate values: a simple, standard, memory load.
+ */
+
+struct module;
+
+#define immediate_read(var)		_immediate_read(var)
+#define immediate_set(var, i)		((var)->value = (i))
+#define _immediate_set(var, i)		immediate_set(var, i)
+#define immediate_set_early(var, i)	immediate_set(var, i)
+#define immediate_if(var)		if (immediate_read(var))
+
+static inline void module_immediate_setup(struct module *mod) { }
+static inline void immediate_update_early(void) { }
+
+#endif /* _ASM_GENERIC_IMMEDIATE_H */

Index: linux-2.6-lttng/include/asm-alpha/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-arm/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-arm26/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-cris/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-frv/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-h8300/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-ia64/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-m32r/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-m68k/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-m68knommu/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-mips/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-parisc/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-ppc/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-s390/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-sh/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-sh64/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-sparc/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-sparc64/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-um/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-v850/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-x86_64/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-xtensa/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-i386/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-powerpc/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-blackfin/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/linux/kprobes.h
===================================================================
@@ -36,9 +36,9 @@
 #include <linux/spinlock.h>
 #include <linux/rcupdate.h>
 
-#ifdef CONFIG_KPROBES
 #include <asm/kprobes.h>
 
+#ifdef CONFIG_KPROBES
 /* kprobe_status settings */
 #define KPROBE_HIT_ACTIVE	0x00000001
 #define KPROBE_HIT_SS		0x00000002

Index: linux-2.6-lttng/include/asm-i386/kdebug.h
===================================================================
@@ -6,6 +6,7 @@
  * from x86_64 architecture.
  */
 #include <linux/notifier.h>
+#include <asm/ptrace.h>
 
 struct pt_regs;
 

@@ -31,4 +32,13 @@ enum die_val {
 	DIE_PAGE_FAULT_NO_CONTEXT,
 };
 
+/* trap3/1 are intr gates for kprobes.  So, restore the status of IF,
+ * if necessary, before executing the original int3/1 (trap) handler.
+ */
+static inline void restore_interrupts(struct pt_regs *regs)
+{
+	if (regs->eflags & IF_MASK)
+		local_irq_enable();
+}
+
 #endif

Index: linux-2.6-lttng/include/asm-i386/kprobes.h
===================================================================
@@ -80,15 +80,6 @@ struct kprobe_ctlblk {
 	struct prev_kprobe prev_kprobe;
 };
 
-/* trap3/1 are intr gates for kprobes.  So, restore the status of IF,
- * if necessary, before executing the original int3/1 (trap) handler.
- */
-static inline void restore_interrupts(struct pt_regs *regs)
-{
-	if (regs->eflags & IF_MASK)
-		local_irq_enable();
-}
-
 extern int kprobe_exceptions_notify(struct notifier_block *self,
 				    unsigned long val, void *data);
 extern int kernel_text_is_ro;

Index: linux-2.6-lttng/include/asm-i386/immediate.h
===================================================================
@@ -1 +1,117 @@
-#include <asm-generic/immediate.h>
+#ifndef _ASM_I386_IMMEDIATE_H
+#define _ASM_I386_IMMEDIATE_H
+
+/*
+ * Immediate values. i386 architecture optimizations.
+ *
+ * (C) Copyright 2006 Mathieu Desnoyers <[email protected]>
+ *
+ * This file is released under the GPLv2.
+ * See the file COPYING for more details.
+ */
+
+struct module;
+
+struct __immediate {
+	long var;		/* Pointer to the identifier variable of the
+				 * immediate value
+				 */
+	long immediate;		/*
+				 * Pointer to the memory location of the
+				 * immediate value within the instruction.
+				 */
+	long size;		/* Type size. */
+};
+
+/*
+ * Optimized version of the immediate.
+ * Make sure the 2 and 4 bytes update will be atomic by aligning the immediate
+ * value.
+ * 2 bytes (short) uses a 66H prefix.
+ * If size is bigger than 4 bytes, fall back on a memory read.
+ * Do not use in __init and __exit functions. Use _immediate_read() instead.
+ */
+#define immediate_read(var)						\
+	({								\
+		__typeof__((var)->value) value;				\
+		switch (sizeof(value)) {				\
+		case 1:							\
+			asm (	".section __immediate, \"a\", @progbits;\n\t" \
+					".long %1, (0f)+1, 1;\n\t"	\
+					".previous;\n\t"		\
+					"0:\n\t"			\
+					"mov %2,%0;\n\t"		\
+				: "=r" (value)				\
+				: "m" ((var)->value),			\
+				  "i" (0));				\
+			break;						\
+		case 2:							\
+			asm (	".section __immediate, \"a\", @progbits;\n\t" \
+					".long %1, (0f)+2, 2;\n\t"	\
+					".previous;\n\t"		\
+					"1:\n\t"			\
+					".align 2;\n\t"			\
+					"0:\n\t"			\
+					"mov %2,%0;\n\t"		\
+				: "=r" (value)				\
+				: "m" ((var)->value),			\
+				  "i" (0));				\
+			break;						\
+		case 4:							\
+			asm (	".section __immediate, \"a\", @progbits;\n\t" \
+					".long %1, (0f)+1, 4;\n\t"	\
+					".previous;\n\t"		\
+					"1:\n\t"			\
+					".org (1b)+(3-((1b)%%4)), 0x90;\n\t" \
+					"0:\n\t"			\
+					"mov %2,%0;\n\t"		\
+				: "=r" (value)				\
+				: "m" ((var)->value),			\
+				  "i" (0));				\
+			break;						\
+		default:value = (var)->value;				\
+			break;						\
+		};							\
+		value;							\
+	})
+
+/*
+ * Update immediate value, can take module mutex.
+ */
+#define immediate_set(var, i) \
+	(var)->value = (i); \
+	immediate_update(1);
+
+/*
+ * Update immediate value. Module mutex must already be taken.
+ */
+#define _immediate_set(var, i) \
+	(var)->value = (i); \
+	immediate_update(0);
+
+/*
+ * Update immediate value at early boot.
+ */
+#define immediate_set_early(var, i) \
+	(var)->value = (i); \
+	immediate_update_early();
+
+/*
+ * Branch depending on an immediate value. Could eventually be optimized further
+ * by improving gcc to give the ability to patch a jump instruction instead of
+ * the value it depends on.
+ * Do not use in __init and __exit functions. Use _immediate_if() instead.
+ */
+#define immediate_if(var)	if (unlikely(immediate_read(var)))
+
+/*
+ * Used internally.
+ */
+
+extern void immediate_update(int lock);
+extern void module_immediate_setup(struct module *mod);
+extern void immediate_update_early(void);
+extern int arch_immediate_update(const struct __immediate *immediate);
+extern void arch_immediate_update_early(const struct __immediate *immediate);
+
+#endif /* _ASM_I386_IMMEDIATE_H */

295 lines not on this page · raw message

Index: linux-2.6-lttng/arch/i386/kernel/traps.c
===================================================================
@@ -629,7 +629,7 @@
 }
 
 DO_VM86_ERROR_INFO( 0, SIGFPE,  "divide error", divide_error, FPE_INTDIV, regs->eip)
-#ifndef CONFIG_KPROBES
+#if !defined(CONFIG_KPROBES) && !defined(CONFIG_IMMEDIATE)
 DO_VM86_ERROR( 3, SIGTRAP, "int3", int3)
 #endif
 DO_VM86_ERROR( 4, SIGSEGV, "overflow", overflow)

@@ -856,14 +856,14 @@
 	nmi_exit();
 }
 
-#ifdef CONFIG_KPROBES
+#if defined(CONFIG_KPROBES) || defined(CONFIG_IMMEDIATE)
 fastcall void __kprobes do_int3(struct pt_regs *regs, long error_code)
 {
 	if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP)
 			== NOTIFY_STOP)
 		return;
-	/* This is an interrupt gate, because kprobes wants interrupts
-	disabled.  Normal trap handlers don't. */
+	/* This is an interrupt gate, because kprobes and immediate values wants
+	 * interrupts disabled. Normal trap handlers don't. */
 	restore_interrupts(regs);
 	do_trap(3, SIGTRAP, "int3", 1, regs, error_code, NULL);
 }

Index: linux-2.6-lttng/include/asm-i386/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-i386/immediate.h
===================================================================
@@ -23,13 +23,16 @@ struct __immediate {
 	long size;		/* Type size. */
 };
 
-/*
+/**
+ * immediate_read - read immediate variable
+ * @var: pointer of type immediate_*_t
+ *
+ * Reads the value of @var.
  * Optimized version of the immediate.
- * Make sure the 2 and 4 bytes update will be atomic by aligning the immediate
- * value.
- * 2 bytes (short) uses a 66H prefix.
- * If size is bigger than 4 bytes, fall back on a memory read.
  * Do not use in __init and __exit functions. Use _immediate_read() instead.
+ * Makes sure the 2 and 4 bytes update will be atomic by aligning the immediate
+ * value. 2 bytes (short) uses a 66H prefix. If size is bigger than 4 bytes,
+ * fall back on a memory read.
  */
 #define immediate_read(var)						\
 	({								\

@@ -75,39 +78,56 @@ struct __immediate {
 		value;							\
 	})
 
-/*
- * Update immediate value, can take module mutex.
+
+/**
+ * immediate_set - set immediate variable (with locking)
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var, taking the module_mutex if required by
+ * the architecture.
  */
 #define immediate_set(var, i) \
 	(var)->value = (i); \
 	immediate_update(1);
 
-/*
- * Update immediate value. Module mutex must already be taken.
+/**
+ * _immediate_set - set immediate variable (without locking)
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var. Must be called with module_mutex held.
  */
 #define _immediate_set(var, i) \
 	(var)->value = (i); \
 	immediate_update(0);
 
-/*
- * Update immediate value at early boot.
+/**
+ * immediate_set_early - set immediate variable at early boot
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var. Should be used for early boot updates.
  */
 #define immediate_set_early(var, i) \
 	(var)->value = (i); \
 	immediate_update_early();
 
-/*
+/**
+ * immediate_if - if () statement depending on an immediate value
+ * @var: pointer of type immediate_*_t
+ *
+ * Use as an if () statement depending on an immediate value.
+ * Do not use in __init and __exit functions. Use _immediate_if() instead.
  * Branch depending on an immediate value. Could eventually be optimized further
  * by improving gcc to give the ability to patch a jump instruction instead of
  * the value it depends on.
- * Do not use in __init and __exit functions. Use _immediate_if() instead.
  */
 #define immediate_if(var)	if (unlikely(immediate_read(var)))
 
 /*
- * Used internally.
+ * Internal update functions.
  */
-
 extern void immediate_update(int lock);
 extern void module_immediate_setup(struct module *mod);
 extern void immediate_update_early(void);

Index: linux-2.6-lttng/arch/i386/kernel/immediate.c
===================================================================
@@ -170,8 +170,12 @@ static struct notifier_block immediate_n
 	.priority = 0x7fffffff,	/* we need to be notified first */
 };
 
-/*
- * Must be called with immediate_mutex held.
+
+/**
+ * arch_immediate_update - update one immediate value
+ * @immediate: pointer of type const struct __immediate to update
+ *
+ * Update one immediate value. Must be called with immediate_mutex held.
  */
 __kprobes int arch_immediate_update(const struct __immediate *immediate)
 {

@@ -266,8 +270,11 @@ __kprobes int arch_immediate_update(cons
 	return 0;
 }
 
-/*
- * Very early initialization of the in-core immediate values.
+/**
+ * arch_immediate_update_early - update one immediate value at boot time
+ * @immediate: pointer of type const struct __immediate to update
+ *
+ * Update one immediate value at boot time.
  */
 void __init arch_immediate_update_early(const struct __immediate *immediate)
 {

Index: linux-2.6-lttng/include/asm-powerpc/immediate.h
===================================================================
@@ -1 +1,106 @@
-#include <asm-generic/immediate.h>
+#ifndef _ASM_POWERPC_IMMEDIATE_H
+#define _ASM_POWERPC_IMMEDIATE_H
+
+/*
+ * Immediate values. PowerPC architecture optimizations.
+ *
+ * (C) Copyright 2006 Mathieu Desnoyers <[email protected]>
+ *
+ * This file is released under the GPLv2.
+ * See the file COPYING for more details.
+ */
+
+#include <asm/asm-compat.h>
+
+struct module;
+
+struct __immediate {
+	long var;		/* Identifier variable of the immediate value */
+	long immediate;		/*
+				 * Pointer to the memory location that holds
+				 * the immediate value within the load immediate
+				 * instruction.
+				 */
+	long size;		/* Type size. */
+};
+
+/*
+ * Optimized version of the immediate.
+ * Make sure the 2 bytes update will be atomic by aligning the immediate
+ * value.
+ * Use a normal memory read for the 4 bytes immediate because there is no way to
+ * atomically update it without using a seqlock read side, which would cost more
+ * in term of total i-cache and d-cache space than a simple memory read.
+ * Do not use in __init and __exit functions. Use _immediate_read() instead.
+ */
+#define immediate_read(var)						\
+	({								\
+		__typeof__((var)->value) value;				\
+		switch (sizeof(value)) {				\
+		case 1:							\
+			asm (	".section __immediate, \"a\", @progbits;\n\t" \
+					PPC_LONG "%1, ((0f)+3), 1;\n\t"	\
+					".previous;\n\t"		\
+					"0:\n\t"			\
+					"li %0,%2;\n\t"			\
+				: "=r" (value)				\
+				: "i" (&(var)->value),			\
+				  "i" (0));				\
+			break;						\
+		case 2:							\
+			asm (	".section __immediate, \"a\", @progbits;\n\t" \
+					PPC_LONG "%1, ((0f)+2), 2;\n\t"	\
+					".previous;\n\t"		\
+					".align 2\n\t"			\
+					"0:\n\t"			\
+					"li %0,%2;\n\t"			\
+				: "=r" (value)				\
+				: "i" (&(var)->value),			\
+				  "i" (0));				\
+			break;						\
+		default:						\
+			value = (var)->value;				\
+			break;						\
+		};							\
+		value;							\
+	})
+
+/*
+ * Update immediate value, can take module mutex.
+ */
+#define immediate_set(var, i) \
+	(var)->value = (i); \
+	immediate_update(1);
+
+/*
+ * Update immediate value. Module mutex must already be taken.
+ */
+#define _immediate_set(var, i) \
+	(var)->value = (i); \
+	immediate_update(0);
+
+/*
+ * Update immediate value at early boot.
+ */
+#define immediate_set_early(var, i) \
+	(var)->value = (i); \
+	immediate_update_early();
+
+/*
+ * Branch depending on an immediate value. Could eventually be optimized further
+ * by improving gcc to give the ability to patch a jump instruction instead of
+ * the value it depends on.
+ * Do not use in __init and __exit functions. Use _immediate_if() instead.
+ */
+#define immediate_if(var)	if (unlikely(immediate_read(var)))
+
+/*
+ * Used internally.
+ */
+extern void immediate_update(int lock);
+extern void module_immediate_setup(struct module *mod);
+extern void immediate_update_early(void);
+extern int arch_immediate_update(const struct __immediate *immediate);
+extern void arch_immediate_update_early(const struct __immediate *immediate);
+
+#endif /* _ASM_POWERPC_IMMEDIATE_H */

98 lines not on this page · raw message

Index: linux-2.6-lttng/include/asm-powerpc/immediate.h
===================================================================
@@ -0,0 +1 @@
+#include <asm-generic/immediate.h>

Index: linux-2.6-lttng/include/asm-powerpc/immediate.h
===================================================================
@@ -24,14 +24,18 @@ struct __immediate {
 	long size;		/* Type size. */
 };
 
-/*
+/**
+ * immediate_read - read immediate variable
+ * @var: pointer of type immediate_*_t
+ *
+ * Reads the value of @var.
  * Optimized version of the immediate.
- * Make sure the 2 bytes update will be atomic by aligning the immediate
- * value.
- * Use a normal memory read for the 4 bytes immediate because there is no way to
- * atomically update it without using a seqlock read side, which would cost more
- * in term of total i-cache and d-cache space than a simple memory read.
  * Do not use in __init and __exit functions. Use _immediate_read() instead.
+ * Makes sure the 2 bytes update will be atomic by aligning the immediate
+ * value. Use a normal memory read for the 4 bytes immediate because there is no
+ * way to atomically update it without using a seqlock read side, which would
+ * cost more in term of total i-cache and d-cache space than a simple memory
+ * read.
  */
 #define immediate_read(var)						\
 	({								\

@@ -65,37 +69,54 @@ struct __immediate {
 		value;							\
 	})
 
-/*
- * Update immediate value, can take module mutex.
+/**
+ * immediate_set - set immediate variable (with locking)
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var, taking the module_mutex if required by
+ * the architecture.
  */
 #define immediate_set(var, i) \
 	(var)->value = (i); \
 	immediate_update(1);
 
-/*
- * Update immediate value. Module mutex must already be taken.
+/**
+ * _immediate_set - set immediate variable (without locking)
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var. Must be called with module_mutex held.
  */
 #define _immediate_set(var, i) \
 	(var)->value = (i); \
 	immediate_update(0);
 
-/*
- * Update immediate value at early boot.
+/**
+ * immediate_set_early - set immediate variable at early boot
+ * @var: pointer of type immediate_*_t
+ * @i: required value
+ *
+ * Sets the value of @var. Should be used for early boot updates.
  */
 #define immediate_set_early(var, i) \
 	(var)->value = (i); \
 	immediate_update_early();
 
-/*
+/**
+ * immediate_if - if () statement depending on an immediate value
+ * @var: pointer of type immediate_*_t
+ *
+ * Use as an if () statement depending on an immediate value.
+ * Do not use in __init and __exit functions. Use _immediate_if() instead.
  * Branch depending on an immediate value. Could eventually be optimized further
  * by improving gcc to give the ability to patch a jump instruction instead of
  * the value it depends on.
- * Do not use in __init and __exit functions. Use _immediate_if() instead.
  */
 #define immediate_if(var)	if (unlikely(immediate_read(var)))
 
 /*
- * Used internally.
+ * Internal update functions.
  */
 extern void immediate_update(int lock);
 extern void module_immediate_setup(struct module *mod);

Index: linux-2.6-lttng/arch/powerpc/kernel/immediate.c
===================================================================
@@ -13,8 +13,11 @@
 
 #define LI_OPCODE_LEN	2
 
-/*
- * The immediate value are aligned.
+/**
+ * arch_immediate_update - update one immediate value
+ * @immediate: pointer of type const struct __immediate to update
+ *
+ * Update one immediate value. Must be called with immediate_mutex held.
  */
 int arch_immediate_update(const struct __immediate *immediate)
 {

@@ -68,8 +71,11 @@ int arch_immediate_update(const struct _
 	return 0;
 }
 
-/*
- * Very early initialization of the in-core immediate values.
+/**
+ * arch_immediate_update_early - update one immediate value at boot time
+ * @immediate: pointer of type const struct __immediate to update
+ *
+ * Update one immediate value at boot time.
  * We can use flush_icache_range, since the cpu identification has been done in
  * the early_init stage.
  */

@@ -95,4 +101,3 @@ void __init arch_immediate_update_early(
 	flush_icache_range((unsigned long)immediate->immediate,
 				immediate->size);
 }
-