jump label v10

index 415b1a9..3c7e9ab 100644
@@ -163,7 +163,8 @@
 	BRANCH_PROFILE()						\
 	TRACE_PRINTKS()							\
 	FTRACE_EVENTS()							\
-	TRACE_SYSCALLS()
+	TRACE_SYSCALLS()						\
+	JUMP_TABLE()							\
 
 /*
  * Data section helpers

@@ -202,6 +203,7 @@
 		*(__vermagic)		/* Kernel version magic */	\
 		*(__markers_strings)	/* Markers: strings */		\
 		*(__tracepoints_strings)/* Tracepoints: strings */	\
+		*(__jump_strings)/* Jump: strings */	\
 	}								\
 									\
 	.rodata1          : AT(ADDR(.rodata1) - LOAD_OFFSET) {		\

@@ -553,6 +555,12 @@
 #define BUG_TABLE
 #endif
 
+#define JUMP_TABLE()							\
+	. = ALIGN(8);							\
+		VMLINUX_SYMBOL(__start___jump_table) = .;		\
+		*(__jump_table)						\
+		VMLINUX_SYMBOL(__stop___jump_table) = .;		\
+
 #ifdef CONFIG_PM_TRACE
 #define TRACEDATA							\
 	. = ALIGN(4);							\

56 lines not on this page · raw message

179 lines not on this page · raw message

index bc6abb7..963f5f9 100644
@@ -4,6 +4,7 @@
 #include <linux/types.h>
 #include <linux/stddef.h>
 #include <linux/stringify.h>
+#include <linux/jump_label.h>
 #include <asm/asm.h>
 
 /*

@@ -160,6 +161,8 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
 #define __parainstructions_end	NULL
 #endif
 
+extern void *text_poke_early(void *addr, const void *opcode, size_t len);
+
 /*
  * Clear and restore the kernel write-protection flag on the local CPU.
  * Allows the kernel to edit read-only pages.

@@ -180,4 +183,15 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
 extern void *text_poke(void *addr, const void *opcode, size_t len);
 extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
 
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
+#define IDEAL_NOP_SIZE_5 5
+extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
+extern int arch_init_ideal_nop5(void);
+#else
+static inline int arch_init_ideal_nop5(void)
+{
+	return 0;
+}
+#endif
+
 #endif /* _ASM_X86_ALTERNATIVE_H */

87 lines not on this page · raw message

84 lines not on this page · raw message

index b008e78..cb28d82 100644
@@ -49,6 +49,7 @@
 #include <asm/pci-direct.h>
 #include <linux/init_ohci1394_dma.h>
 #include <linux/kvm_para.h>
+#include <linux/jump_label.h>
 
 #include <linux/errno.h>
 #include <linux/kernel.h>

@@ -1069,6 +1070,8 @@ void __init setup_arch(char **cmdline_p)
 	x86_init.oem.banner();
 
 	mcheck_init();
+
+	arch_init_ideal_nop5();
 }
 
 #ifdef CONFIG_X86_32

index 9ac29c0..fc2b181 100644
@@ -11,6 +11,8 @@ enum jump_label_type {
 	JUMP_LABEL_DISABLE
 };
 
+struct module;
+
 #ifdef HAVE_JUMP_LABEL
 
 extern struct jump_entry __start___jump_table[];

@@ -23,8 +25,9 @@ extern struct jump_entry __stop___jump_table[];
 
 extern void arch_jump_label_transform(struct jump_entry *entry,
 				 enum jump_label_type type);
-extern const u8 *arch_jump_label_get_nop(void);
 extern void jump_label_update(const char *name, enum jump_label_type type);
+extern void jump_label_apply_nops(struct module *mod);
+extern void arch_jump_label_text_poke_early(jump_label_t addr);
 
 #define enable_jump_label(name) \
 	jump_label_update(name, JUMP_LABEL_ENABLE);

@@ -52,6 +55,11 @@ static inline int disable_jump_label(const char *name)
 	return 0;
 }
 
+static inline int jump_label_apply_nops(struct module *mod)
+{
+	return 0;
+}
+
 #endif
 
 #endif

index ecca4f4..ebe9145 100644
@@ -11,6 +11,7 @@
 #include <linux/list.h>
 #include <linux/jhash.h>
 #include <linux/slab.h>
+#include <linux/err.h>
 
 #ifdef HAVE_JUMP_LABEL
 

@@ -185,10 +186,18 @@ void jump_label_update(const char *name, enum jump_label_type type)
 static __init int init_jump_label(void)
 {
 	int ret;
+	struct jump_entry *iter_start = __start___jump_table;
+	struct jump_entry *iter_stop = __stop___jump_table;
+	struct jump_entry *iter;
 
 	mutex_lock(&jump_label_mutex);
 	ret = build_jump_label_hashtable(__start___jump_table,
 					 __stop___jump_table);
+	iter = iter_start;
+	while (iter < iter_stop) {
+		arch_jump_label_text_poke_early(iter->code);
+		iter++;
+	}
 	mutex_unlock(&jump_label_mutex);
 	return ret;
 }

@@ -299,6 +308,29 @@ static int jump_label_module_notify(struct notifier_block *self, unsigned long v
 	return ret;
 }
 
+/***
+ * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
+ * @mod: module to patch
+ *
+ * Allow for run-time selection of the optimal nops. Before the module
+ * loads patch these with arch_get_jump_label_nop(), which is specified by
+ * the arch specific jump label code.
+ */
+void jump_label_apply_nops(struct module *mod)
+{
+	struct jump_entry *iter;
+
+	/* if the module doesn't have jump label entries, just return */
+	if (!mod->num_jump_entries)
+		return;
+
+	iter = mod->jump_entries;
+	while (iter < mod->jump_entries + mod->num_jump_entries) {
+		arch_jump_label_text_poke_early(iter->code);
+		iter++;
+	}
+}
+
 struct notifier_block jump_label_module_nb = {
 	.notifier_call = jump_label_module_notify,
 	.priority = 0,

index 0d88ce9..1a1cc47 100644
@@ -2749,20 +2749,9 @@ extern unsigned long __stop_mcount_loc[];
 
 void __init ftrace_init(void)
 {
-	unsigned long count, addr, flags;
+	unsigned long count;
 	int ret;
 
-	/* Keep the ftrace pointer to the stub */
-	addr = (unsigned long)ftrace_stub;
-
-	local_irq_save(flags);
-	ftrace_dyn_arch_init(&addr);
-	local_irq_restore(flags);
-
-	/* ftrace_dyn_arch_init places the return code in addr */
-	if (addr)
-		goto failed;
-
 	count = __stop_mcount_loc - __start_mcount_loc;
 
 	ret = ftrace_dyn_table_alloc(count);

index 1bfb6cf..b1a4208 100644
@@ -1202,7 +1202,8 @@ static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
 	}
 	/* Check whether the address range is reserved */
 	if (ftrace_text_reserved(src, src + len - 1) ||
-	    alternatives_text_reserved(src, src + len - 1))
+	    alternatives_text_reserved(src, src + len - 1) ||
+	    jump_label_text_reserved(src, src + len - 1))
 		return -EBUSY;
 
 	return len;

index fc2b181..d3d00b6 100644
@@ -25,9 +25,10 @@ extern struct jump_entry __stop___jump_table[];
 
 extern void arch_jump_label_transform(struct jump_entry *entry,
 				 enum jump_label_type type);
+extern void arch_jump_label_text_poke_early(jump_label_t addr);
 extern void jump_label_update(const char *name, enum jump_label_type type);
+extern int jump_label_text_reserved(void *start, void *end);
 extern void jump_label_apply_nops(struct module *mod);
-extern void arch_jump_label_text_poke_early(jump_label_t addr);
 
 #define enable_jump_label(name) \
 	jump_label_update(name, JUMP_LABEL_ENABLE);

@@ -60,6 +61,11 @@ static inline int jump_label_apply_nops(struct module *mod)
 	return 0;
 }
 
+static inline int jump_label_text_reserved(void *start, void *end)
+{
+	return 0;
+}
+
 #endif
 
 #endif

89 lines not on this page · raw message

index 282035f..2f9bea6 100644
@@ -1141,7 +1141,8 @@ int __kprobes register_kprobe(struct kprobe *p)
 	preempt_disable();
 	if (!kernel_text_address((unsigned long) p->addr) ||
 	    in_kprobes_functions((unsigned long) p->addr) ||
-	    ftrace_text_reserved(p->addr, p->addr)) {
+	    ftrace_text_reserved(p->addr, p->addr) ||
+	    jump_label_text_reserved(p->addr, p->addr)) {
 		preempt_enable();
 		return -EINVAL;
 	}

index a7cc379..209b379 100644
@@ -263,6 +263,11 @@ int __init trace_workqueue_early_init(void)
 {
 	int ret, cpu;
 
+	for_each_possible_cpu(cpu) {
+		spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
+		INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
+	}
+
 	ret = register_trace_workqueue_insertion(probe_workqueue_insertion, NULL);
 	if (ret)
 		goto out;

@@ -279,11 +284,6 @@ int __init trace_workqueue_early_init(void)
 	if (ret)
 		goto no_creation;
 
-	for_each_possible_cpu(cpu) {
-		spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
-		INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
-	}
-
 	return 0;
 
 no_creation:

index 10fb17f..faef97e 100644
@@ -12,6 +12,7 @@
 #include <linux/jhash.h>
 #include <linux/slab.h>
 #include <linux/err.h>
+#include <linux/sort.h>
 
 #ifdef HAVE_JUMP_LABEL
 

@@ -38,33 +39,27 @@ struct jump_label_module_entry {
 	struct module *mod;
 };
 
-static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
+static int jump_label_cmp(const void *a, const void *b)
 {
-	struct jump_entry tmp;
+	const struct jump_entry *jea = a;
+	const struct jump_entry *jeb = b;
 
-	tmp = *next;
-	*next = *previous;
-	*previous = tmp;
+	if (jea->key < jeb->key)
+		return -1;
+
+	if (jea->key > jeb->key)
+		return 1;
+
+	return 0;
 }
 
 static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
 {
-	int swapped = 0;
-	struct jump_entry *iter;
-	struct jump_entry *iter_next;
-
-	do {
-		swapped = 0;
-		iter = start;
-		iter_next = start;
-		iter_next++;
-		for (; iter_next < stop; iter++, iter_next++) {
-			if (iter->key > iter_next->key) {
-				swap_jump_label_entries(iter, iter_next);
-				swapped = 1;
-			}
-		}
-	} while (swapped == 1);
+	unsigned long size;
+
+	size = (((unsigned long)stop - (unsigned long)start)
+					/ sizeof(struct jump_entry));
+	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
 }
 
 static struct jump_label_entry *get_jump_label_entry(jump_label_t key)

index 103d1b6..a4a90b6 100644
@@ -17,6 +17,7 @@
 #include <linux/errno.h>
 #include <linux/types.h>
 #include <linux/rcupdate.h>
+#include <linux/jump_label.h>
 
 struct module;
 struct tracepoint;

@@ -145,7 +146,9 @@ static inline void tracepoint_update_probe_range(struct tracepoint *begin,
 	extern struct tracepoint __tracepoint_##name;			\
 	static inline void trace_##name(proto)				\
 	{								\
-		if (unlikely(__tracepoint_##name.state))		\
+		JUMP_LABEL(&__tracepoint_##name.state, do_trace);	\
+		return;							\
+do_trace:								\
 			__DO_TRACE(&__tracepoint_##name,		\
 				TP_PROTO(data_proto),			\
 				TP_ARGS(data_args));			\

index c77f3ec..d6073a5 100644
@@ -25,6 +25,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
+#include <linux/jump_label.h>
 
 extern struct tracepoint __start___tracepoints[];
 extern struct tracepoint __stop___tracepoints[];

@@ -263,7 +264,13 @@ static void set_tracepoint(struct tracepoint_entry **entry,
 	 * is used.
 	 */
 	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
-	elem->state = active;
+	if (!elem->state && active) {
+		enable_jump_label(&elem->state);
+		elem->state = active;
+	} else if (elem->state && !active) {
+		disable_jump_label(&elem->state);
+		elem->state = active;
+	}
 }
 
 /*

@@ -277,7 +284,10 @@ static void disable_tracepoint(struct tracepoint *elem)
 	if (elem->unregfunc && elem->state)
 		elem->unregfunc();
 
-	elem->state = 0;
+	if (elem->state) {
+		disable_jump_label(&elem->state);
+		elem->state = 0;
+	}
 	rcu_assign_pointer(elem->funcs, NULL);
 }
 

index 3c7e9ab..8679ec7 100644
@@ -163,8 +163,7 @@
 	BRANCH_PROFILE()						\
 	TRACE_PRINTKS()							\
 	FTRACE_EVENTS()							\
-	TRACE_SYSCALLS()						\
-	JUMP_TABLE()							\
+	TRACE_SYSCALLS()
 
 /*
  * Data section helpers

@@ -203,7 +202,6 @@
 		*(__vermagic)		/* Kernel version magic */	\
 		*(__markers_strings)	/* Markers: strings */		\
 		*(__tracepoints_strings)/* Tracepoints: strings */	\
-		*(__jump_strings)/* Jump: strings */	\
 	}								\
 									\
 	.rodata1          : AT(ADDR(.rodata1) - LOAD_OFFSET) {		\

@@ -212,6 +210,8 @@
 									\
 	BUG_TABLE							\
 									\
+	JUMP_TABLE							\
+									\
 	/* PCI quirks */						\
 	.pci_fixup        : AT(ADDR(.pci_fixup) - LOAD_OFFSET) {	\
 		VMLINUX_SYMBOL(__start_pci_fixups_early) = .;		\

@@ -555,11 +555,13 @@
 #define BUG_TABLE
 #endif
 
-#define JUMP_TABLE()							\
+#define JUMP_TABLE							\
 	. = ALIGN(8);							\
+	__jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) {		\
 		VMLINUX_SYMBOL(__start___jump_table) = .;		\
 		*(__jump_table)						\
 		VMLINUX_SYMBOL(__stop___jump_table) = .;		\
+	}
 
 #ifdef CONFIG_PM_TRACE
 #define TRACEDATA							\

index d3d00b6..c9e17b0 100644
@@ -18,43 +18,36 @@ struct module;
 extern struct jump_entry __start___jump_table[];
 extern struct jump_entry __stop___jump_table[];
 
-#define DEFINE_JUMP_LABEL(name)						\
-	const char __jlstrtab_##name[]					\
-	__used __attribute__((section("__jump_strings")))  = #name;	\
-	EXPORT_SYMBOL_GPL(__jlstrtab_##name);
-
 extern void arch_jump_label_transform(struct jump_entry *entry,
 				 enum jump_label_type type);
 extern void arch_jump_label_text_poke_early(jump_label_t addr);
-extern void jump_label_update(const char *name, enum jump_label_type type);
+extern void jump_label_update(unsigned long key, enum jump_label_type type);
 extern int jump_label_text_reserved(void *start, void *end);
 extern void jump_label_apply_nops(struct module *mod);
 
-#define enable_jump_label(name) \
-	jump_label_update(name, JUMP_LABEL_ENABLE);
+#define enable_jump_label(key) \
+	jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
 
-#define disable_jump_label(name) \
-	jump_label_update(name, JUMP_LABEL_DISABLE);
+#define disable_jump_label(key) \
+	jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
 
 #else
 
-#define DEFINE_JUMP_LABEL(name)
-
-#define JUMP_LABEL(tag, label, cond)		\
+#define JUMP_LABEL(key, label)			\
 do {						\
-	if (unlikely(cond))			\
+	if (unlikely(*key))			\
 		goto label;			\
 } while (0)
 
-static inline int enable_jump_label(const char *name)
-{
-	return 0;
-}
+#define enable_jump_label(cond_var)	\
+do {					\
+	*(cond_var) = 1;		\
+} while (0)
 
-static inline int disable_jump_label(const char *name)
-{
-	return 0;
-}
+#define disable_jump_label(cond_var)	\
+do {					\
+	*(cond_var) = 0;		\
+} while (0)
 
 static inline int jump_label_apply_nops(struct module *mod)
 {

index c1e59bb..10fb17f 100644
@@ -28,7 +28,7 @@ struct jump_label_entry {
 	int nr_entries;
 	/* hang modules off here */
 	struct hlist_head modules;
-	const char *name;
+	unsigned long key;
 };
 
 struct jump_label_module_entry {

@@ -59,8 +59,7 @@ static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry
 		iter_next = start;
 		iter_next++;
 		for (; iter_next < stop; iter++, iter_next++) {
-			if (strcmp((char *)iter->name,
-						(char *)iter_next->name) > 0) {
+			if (iter->key > iter_next->key) {
 				swap_jump_label_entries(iter, iter_next);
 				swapped = 1;
 			}

@@ -68,29 +67,28 @@ static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry
 	} while (swapped == 1);
 }
 
-static struct jump_label_entry *get_jump_label_entry(const char *name)
+static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
 {
 	struct hlist_head *head;
 	struct hlist_node *node;
 	struct jump_label_entry *e;
-	u32 hash = jhash(name, strlen(name), 0);
+	u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
 
 	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
 	hlist_for_each_entry(e, node, head, hlist) {
-		if (!strcmp((char *)name, (char *)e->name))
+		if (key == e->key)
 			return e;
 	}
 	return NULL;
 }
 
-static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_entries, struct jump_entry *table)
+static struct jump_label_entry *add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
 {
 	struct hlist_head *head;
 	struct jump_label_entry *e;
-	size_t name_len;
 	u32 hash;
 
-	e = get_jump_label_entry(name);
+	e = get_jump_label_entry(key);
 	if (e)
 		return ERR_PTR(-EEXIST);
 

@@ -98,10 +96,9 @@ static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_en
 	if (!e)
 		return ERR_PTR(-ENOMEM);
 
-	name_len = strlen(name) + 1;
-	hash = jhash(name, name_len-1, 0);
+	hash = jhash((void *)&key, sizeof(jump_label_t), 0);
 	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
-	e->name = name;
+	e->key = key;
 	e->table = table;
 	e->nr_entries = nr_entries;
 	INIT_HLIST_HEAD(&(e->modules));

@@ -118,17 +115,16 @@ static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entr
 	sort_jump_label_entries(start, stop);
 	iter = start;
 	while (iter < stop) {
-		entry = get_jump_label_entry((char *)iter->name);
+		entry = get_jump_label_entry(iter->key);
 		if (!entry) {
 			iter_begin = iter;
 			count = 0;
 			while ((iter < stop) &&
-				(strcmp((char *)iter->name,
-					(char *)iter_begin->name) == 0)) {
+				(iter->key == iter_begin->key)) {
 				iter++;
 				count++;
 			}
-			entry = add_jump_label_entry((char *)iter_begin->name,
+			entry = add_jump_label_entry(iter_begin->key,
 							count, iter_begin);
 			if (IS_ERR(entry))
 				return PTR_ERR(entry);

@@ -150,7 +146,7 @@ static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entr
  *
  */
 
-void jump_label_update(const char *name, enum jump_label_type type)
+void jump_label_update(unsigned long key, enum jump_label_type type)
 {
 	struct jump_entry *iter;
 	struct jump_label_entry *entry;

@@ -159,7 +155,7 @@ void jump_label_update(const char *name, enum jump_label_type type)
 	int count;
 
 	mutex_lock(&jump_label_mutex);
-	entry = get_jump_label_entry(name);
+	entry = get_jump_label_entry((jump_label_t)key);
 	if (entry) {
 		count = entry->nr_entries;
 		iter = entry->table;

@@ -317,16 +313,16 @@ static int add_jump_label_module(struct module *mod)
 				mod->jump_entries + mod->num_jump_entries);
 	iter = mod->jump_entries;
 	while (iter < mod->jump_entries + mod->num_jump_entries) {
-		entry = get_jump_label_entry((char *)iter->name);
+		entry = get_jump_label_entry(iter->key);
 		iter_begin = iter;
 		count = 0;
 		while ((iter < mod->jump_entries + mod->num_jump_entries) &&
-			(strcmp((char *)iter->name, (char *)iter_begin->name) == 0)) {
+			(iter->key == iter_begin->key)) {
 				iter++;
 				count++;
 		}
 		if (!entry) {
-			entry = add_jump_label_entry((char *)iter_begin->name, 0, NULL);
+			entry = add_jump_label_entry(iter_begin->key, 0, NULL);
 			if (IS_ERR(entry))
 				return PTR_ERR(entry);
 		}

new file mode 100644
index 0000000..62e66d7
@@ -0,0 +1,32 @@
+#ifndef _ASM_SPARC_JUMP_LABEL_H
+#define _ASM_SPARC_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/system.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+#define JUMP_LABEL(key, label)					\
+	do {							\
+		asm goto("1:\n\t"				\
+			 "nop\n\t"				\
+			 "nop\n\t"				\
+			 ".pushsection __jump_table,  \"a\"\n\t"\
+			 ".word 1b, %l[" #label "], %c0\n\t"	\
+			 ".popsection \n\t"			\
+			 : :  "i" (key) :  : label);\
+	} while (0)
+
+#endif /* __KERNEL__ */
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+	jump_label_t code;
+	jump_label_t target;
+	jump_label_t key;
+};
+
+#endif

new file mode 100644
index 0000000..10cfbef
@@ -0,0 +1,39 @@
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/cpu.h>
+
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
+{
+	u32 val;
+	u32 *insn = (u32 *) (unsigned long) entry->code;
+
+	if (type == JUMP_LABEL_ENABLE) {
+		s32 off = (s32)entry->target - (s32)entry->code;
+		val = 0x40000000 | ((u32) off >> 2);
+	} else {
+		val = 0x01000000;
+	}
+
+	get_online_cpus();
+	mutex_lock(&text_mutex);
+	*insn = val;
+	flushi(insn);
+	mutex_unlock(&text_mutex);
+	put_online_cpus();
+}
+
+void arch_jump_label_text_poke_early(jump_label_t addr)
+{
+	u32 *insn_p = (u32 *) (unsigned long) addr;
+
+	*insn_p = 0x01000000;
+	flushi(insn_p);
+}
+
+#endif

index f848aad..ee3c7dd 100644
@@ -18,6 +18,9 @@
 #include <asm/spitfire.h>
 
 #ifdef CONFIG_SPARC64
+
+#include <linux/jump_label.h>
+
 static void *module_map(unsigned long size)
 {
 	struct vm_struct *area;

@@ -227,6 +230,9 @@ int module_finalize(const Elf_Ehdr *hdr,
 		    const Elf_Shdr *sechdrs,
 		    struct module *me)
 {
+	/* make jump label nops */
+	jump_label_apply_nops(me);
+
 	/* Cheetah's I-cache is fully coherent.  */
 	if (tlb_type == spitfire) {
 		unsigned long va;

index 52c0da4..bef3cda 100644
@@ -1,6 +1,8 @@
 #ifndef _DYNAMIC_DEBUG_H
 #define _DYNAMIC_DEBUG_H
 
+#include <linux/jump_label.h>
+
 /* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
  * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
  * use independent hash functions, to reduce the chance of false positives.

@@ -22,8 +24,6 @@ struct _ddebug {
 	const char *function;
 	const char *filename;
 	const char *format;
-	char primary_hash;
-	char secondary_hash;
 	unsigned int lineno:24;
 	/*
  	 * The flags field controls the behaviour at the callsite.

@@ -33,6 +33,7 @@ struct _ddebug {
 #define _DPRINTK_FLAGS_PRINT   (1<<0)  /* printk() a message using the format */
 #define _DPRINTK_FLAGS_DEFAULT 0
 	unsigned int flags:8;
+	char enabled;
 } __attribute__((aligned(8)));
 
 

@@ -42,33 +43,35 @@ int ddebug_add_module(struct _ddebug *tab, unsigned int n,
 #if defined(CONFIG_DYNAMIC_DEBUG)
 extern int ddebug_remove_module(const char *mod_name);
 
-#define __dynamic_dbg_enabled(dd)  ({	     \
-	int __ret = 0;							     \
-	if (unlikely((dynamic_debug_enabled & (1LL << DEBUG_HASH)) &&	     \
-			(dynamic_debug_enabled2 & (1LL << DEBUG_HASH2))))   \
-				if (unlikely(dd.flags))			     \
-					__ret = 1;			     \
-	__ret; })
-
 #define dynamic_pr_debug(fmt, ...) do {					\
+	__label__ do_printk;						\
+	__label__ out;							\
 	static struct _ddebug descriptor				\
 	__used								\
 	__attribute__((section("__verbose"), aligned(8))) =		\
-	{ KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH,	\
-		DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT };	\
-	if (__dynamic_dbg_enabled(descriptor))				\
-		printk(KERN_DEBUG pr_fmt(fmt),	##__VA_ARGS__);		\
+	{ KBUILD_MODNAME, __func__, __FILE__, fmt, __LINE__,		\
+		_DPRINTK_FLAGS_DEFAULT };				\
+	JUMP_LABEL(&descriptor.enabled, do_printk);			\
+	goto out;							\
+do_printk:								\
+	printk(KERN_DEBUG pr_fmt(fmt),	##__VA_ARGS__);			\
+out:	;								\
 	} while (0)
 
 
 #define dynamic_dev_dbg(dev, fmt, ...) do {				\
+	__label__ do_printk;						\
+	__label__ out;							\
 	static struct _ddebug descriptor				\
 	__used								\
 	__attribute__((section("__verbose"), aligned(8))) =		\
-	{ KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH,	\
-		DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT };	\
-	if (__dynamic_dbg_enabled(descriptor))				\
-		dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__);	\
+	{ KBUILD_MODNAME, __func__, __FILE__, fmt, __LINE__,		\
+		_DPRINTK_FLAGS_DEFAULT };				\
+	JUMP_LABEL(&descriptor.enabled, do_printk);			\
+	goto out;							\
+do_printk:								\
+	dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__);		\
+out:	;								\
 	} while (0)
 
 #else

65 lines not on this page · raw message

64 lines not on this page · raw message

47 lines not on this page · raw message

49 lines not on this page · raw message

index e0bc186..5399f58 100644
@@ -239,6 +239,9 @@ int module_finalize(const Elf_Ehdr *hdr,
 		apply_paravirt(pseg, pseg + para->sh_size);
 	}
 
+	/* make jump label nops */
+	jump_label_apply_nops(me);
+
 	return module_bug_finalize(hdr, sechdrs, me);
 }
 

index 846b72a..9ac29c0 100644
@@ -18,7 +18,8 @@ extern struct jump_entry __stop___jump_table[];
 
 #define DEFINE_JUMP_LABEL(name)						\
 	const char __jlstrtab_##name[]					\
-	__used __attribute__((section("__jump_strings")))  = #name;
+	__used __attribute__((section("__jump_strings")))  = #name;	\
+	EXPORT_SYMBOL_GPL(__jlstrtab_##name);
 
 extern void arch_jump_label_transform(struct jump_entry *entry,
 				 enum jump_label_type type);

index 8a6b9fd..403ac26 100644
@@ -350,7 +350,10 @@ struct module
 	struct tracepoint *tracepoints;
 	unsigned int num_tracepoints;
 #endif
-
+#ifdef HAVE_JUMP_LABEL
+	struct jump_entry *jump_entries;
+	unsigned int num_jump_entries;
+#endif
 #ifdef CONFIG_TRACING
 	const char **trace_bprintk_fmt_start;
 	unsigned int num_trace_bprintk_fmt;

152 lines not on this page · raw message

index 5d2d281..695a131 100644
@@ -55,6 +55,7 @@
 #include <linux/async.h>
 #include <linux/percpu.h>
 #include <linux/kmemleak.h>
+#include <linux/jump_label.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/module.h>

@@ -2413,6 +2414,12 @@ static noinline struct module *load_module(void __user *umod,
 					sizeof(*mod->tracepoints),
 					&mod->num_tracepoints);
 #endif
+#ifdef HAVE_JUMP_LABEL
+	mod->jump_entries = section_objs(hdr, sechdrs, secstrings,
+					"__jump_table",
+					sizeof(*mod->jump_entries),
+					&mod->num_jump_entries);
+#endif
 #ifdef CONFIG_EVENT_TRACING
 	mod->trace_events = section_objs(hdr, sechdrs, secstrings,
 					 "_ftrace_events",

index 2f9bea6..a53bff3 100644
@@ -47,6 +47,7 @@
 #include <linux/memory.h>
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
+#include <linux/jump_label.h>
 
 #include <asm-generic/sections.h>
 #include <asm/cacheflush.h>

new file mode 100644
index 0000000..62e66d7
@@ -0,0 +1,32 @@
+#ifndef _ASM_SPARC_JUMP_LABEL_H
+#define _ASM_SPARC_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/system.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+#define JUMP_LABEL(key, label)					\
+	do {							\
+		asm goto("1:\n\t"				\
+			 "nop\n\t"				\
+			 "nop\n\t"				\
+			 ".pushsection __jump_table,  \"a\"\n\t"\
+			 ".word 1b, %l[" #label "], %c0\n\t"	\
+			 ".popsection \n\t"			\
+			 : :  "i" (key) :  : label);\
+	} while (0)
+
+#endif /* __KERNEL__ */
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+	jump_label_t code;
+	jump_label_t target;
+	jump_label_t key;
+};
+
+#endif

46 lines not on this page · raw message

index f848aad..ee3c7dd 100644
@@ -18,6 +18,9 @@
 #include <asm/spitfire.h>
 
 #ifdef CONFIG_SPARC64
+
+#include <linux/jump_label.h>
+
 static void *module_map(unsigned long size)
 {
 	struct vm_struct *area;

@@ -227,6 +230,9 @@ int module_finalize(const Elf_Ehdr *hdr,
 		    const Elf_Shdr *sechdrs,
 		    struct module *me)
 {
+	/* make jump label nops */
+	jump_label_apply_nops(me);
+
 	/* Cheetah's I-cache is fully coherent.  */
 	if (tlb_type == spitfire) {
 		unsigned long va;