Skip to contentSkip to conversation

riscv: Add jump-label implementation

PATCH In review Emil Renner Berthing 2 participants 6 messages Jul 7, 2020 – Jul 8, 2020

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 riscv: Add jump-label implementation

Add jump-label implementation based on the ARM64 version.

Tested on the HiFive Unleashed board.

---

Changes since RFC:
- Use RISCV_PTR and RISCV_LGPTR macros to match struct jump_table also in 32bit kernels.
- Remove unneeded branch ? 1 : 0, thanks Björn
- Fix \n\n instead of \n\t mistake

 .../core/jump-labels/arch-support.txt         |  2 +-
 arch/riscv/Kconfig                            |  2 +
 arch/riscv/include/asm/jump_label.h           | 59 +++++++++++++++++++
 arch/riscv/kernel/Makefile                    |  2 +
 arch/riscv/kernel/jump_label.c                | 44 ++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/jump_label.h
 create mode 100644 arch/riscv/kernel/jump_label.c
Documentation/features/core/jump-labels/arch-support.txt +1 -1
index 632a1c7aefa2..760243d18ed7 100644
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
arch/riscv/Kconfig +2 -0
index fd639937e251..d2f5c53fdc19 100644
@@ -46,6 +46,8 @@ config RISCV
 	select GENERIC_TIME_VSYSCALL if MMU && 64BIT
 	select HANDLE_DOMAIN_IRQ
 	select HAVE_ARCH_AUDITSYSCALL
+	select HAVE_ARCH_JUMP_LABEL
+	select HAVE_ARCH_JUMP_LABEL_RELATIVE
 	select HAVE_ARCH_KASAN if MMU && 64BIT
 	select HAVE_ARCH_KGDB
 	select HAVE_ARCH_KGDB_QXFER_PKT
arch/riscv/include/asm/jump_label.h new file +59 -0

59 lines not on this page · raw message

arch/riscv/kernel/Makefile +2 -0
index b355cf485671..a5287ab9f7f2 100644
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)	+= cpu-hotplug.o
 obj-$(CONFIG_KGDB)		+= kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)	+= jump_label.o
+
 clean:
arch/riscv/kernel/jump_label.c new file +44 -0

44 lines not on this page · raw message

Signature · 1 line
2.27.0
Reviewed-by on v1 1/1
#1 lore raw
Quoted 3 lines · Emil Renner Berthing

Add jump-label implementation based on the ARM64 version.

Thanks for working on this!

Quoted 139 lines

Tested on the HiFive Unleashed board.

---

Changes since RFC:
- Use RISCV_PTR and RISCV_LGPTR macros to match struct jump_table also in 32bit kernels.
- Remove unneeded branch ? 1 : 0, thanks Björn
- Fix \n\n instead of \n\t mistake

 .../core/jump-labels/arch-support.txt         |  2 +-
 arch/riscv/Kconfig                            |  2 +
 arch/riscv/include/asm/jump_label.h           | 59 +++++++++++++++++++
 arch/riscv/kernel/Makefile                    |  2 +
 arch/riscv/kernel/jump_label.c                | 44 ++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/jump_label.h
 create mode 100644 arch/riscv/kernel/jump_label.c
Documentation/features/core/jump-labels/arch-support.txt +1 -1
index 632a1c7aefa2..760243d18ed7 100644
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
arch/riscv/Kconfig +2 -0
index fd639937e251..d2f5c53fdc19 100644
@@ -46,6 +46,8 @@ config RISCV
        select GENERIC_TIME_VSYSCALL if MMU && 64BIT
        select HANDLE_DOMAIN_IRQ
        select HAVE_ARCH_AUDITSYSCALL
+       select HAVE_ARCH_JUMP_LABEL
+       select HAVE_ARCH_JUMP_LABEL_RELATIVE
        select HAVE_ARCH_KASAN if MMU && 64BIT
        select HAVE_ARCH_KGDB
        select HAVE_ARCH_KGDB_QXFER_PKT
arch/riscv/include/asm/jump_label.h new file +59 -0
new file mode 100644
index 000000000000..d5fb342bfccf
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+                                              bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     nop                                     \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+                                                   bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     jal             zero, %l[label]         \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+#endif  /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */
arch/riscv/kernel/Makefile +2 -0
index b355cf485671..a5287ab9f7f2 100644
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)      += cpu-hotplug.o
 obj-$(CONFIG_KGDB)             += kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
+
 clean:
arch/riscv/kernel/jump_label.c new file +11 -0
new file mode 100644
index 000000000000..55b2d742efe1
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013

arch/riscv/kernel/jump_label.c line 21

Quoted 10 lines
arch/riscv/kernel/jump_label.c new file +10 -0
+#define RISCV_INSN_JAL 0x0000006f
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       void *addr = (void *)jump_entry_code(entry);
+       u32 insn;
+
+       if (type == JUMP_LABEL_JMP) {
+               u32 offset = jump_entry_target(entry) - jump_entry_code(entry);

The 20b jump offset of JAL is enough, but to catch future changes that might potentially break this, I would add a WARN to catch if offset

Quoted 1 line

20b (or overkill fallback to JALR -- don't do that.).

Somewhat related to that; The UNIX specification for RISC-V requires support for RVC. Would it make sense to use the C.-instructions C.J and C.NOP? Really a nit, or more of a question. I'm fine with this 32b version!

arch/riscv/kernel/jump_label.c line 29

Quoted 8 lines
arch/riscv/kernel/jump_label.c new file +8 -0
+
+               insn = RISCV_INSN_JAL |
+                       ((offset & GENMASK(19, 12)) << (12 - 12)) |
+                       ((offset & GENMASK(11, 11)) << (20 - 11)) |
+                       ((offset & GENMASK(10,  1)) << (21 -  1)) |
+                       ((offset & GENMASK(20, 20)) << (31 - 20));
+       } else
+               insn = RISCV_INSN_NOP;

Since the if-statement needs {}, the else-clause should have one too.

Feel free to add:

Cheers,
Björn

Quoted 18 lines
arch/riscv/kernel/jump_label.c new file +15 -1
+
+       patch_text_nosync(addr, &insn, sizeof(insn));
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       /*
+        * We use the same instructions in the arch_static_branch and
+        * arch_static_branch_jump inline functions, so there's no
+        * need to patch them up here.
+        * The core will call arch_jump_label_transform  when those
+        * instructions need to be replaced.
+        */
+}
--

2.27.0

#3 lore raw
In reply to Emil Renner Berthing
Quoted 41 lines · Emil Renner Berthing

Add jump-label implementation based on the ARM64 version.

Tested on the HiFive Unleashed board.

---

Changes since RFC:
- Use RISCV_PTR and RISCV_LGPTR macros to match struct jump_table also in 32bit kernels.
- Remove unneeded branch ? 1 : 0, thanks Björn
- Fix \n\n instead of \n\t mistake

 .../core/jump-labels/arch-support.txt         |  2 +-
 arch/riscv/Kconfig                            |  2 +
 arch/riscv/include/asm/jump_label.h           | 59 +++++++++++++++++++
 arch/riscv/kernel/Makefile                    |  2 +
 arch/riscv/kernel/jump_label.c                | 44 ++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/jump_label.h
 create mode 100644 arch/riscv/kernel/jump_label.c
Documentation/features/core/jump-labels/arch-support.txt +1 -1
index 632a1c7aefa2..760243d18ed7 100644
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
arch/riscv/Kconfig +0 -0
index fd639937e251..d2f5c53fdc19 100644

arch/riscv/Kconfig config RISCV · line 50

Quoted 5 lines
arch/riscv/Kconfig +2 -0
@@ -46,6 +46,8 @@ config RISCV
        select GENERIC_TIME_VSYSCALL if MMU && 64BIT
        select HANDLE_DOMAIN_IRQ
        select HAVE_ARCH_AUDITSYSCALL
+       select HAVE_ARCH_JUMP_LABEL
+       select HAVE_ARCH_JUMP_LABEL_RELATIVE

Missed one thing in the last reply; Please update defconfig with CONFIG_JUMP_LABEL=y.

Björn

Quoted 132 lines
arch/riscv/Kconfig +0 -0
        select HAVE_ARCH_KASAN if MMU && 64BIT
        select HAVE_ARCH_KGDB
        select HAVE_ARCH_KGDB_QXFER_PKT
arch/riscv/include/asm/jump_label.h new file +59 -0
new file mode 100644
index 000000000000..d5fb342bfccf
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+                                              bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     nop                                     \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+                                                   bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     jal             zero, %l[label]         \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+#endif  /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */
arch/riscv/kernel/Makefile +2 -0
index b355cf485671..a5287ab9f7f2 100644
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)      += cpu-hotplug.o
 obj-$(CONFIG_KGDB)             += kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
+
 clean:
arch/riscv/kernel/jump_label.c new file +44 -1
new file mode 100644
index 000000000000..55b2d742efe1
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013
+#define RISCV_INSN_JAL 0x0000006f
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       void *addr = (void *)jump_entry_code(entry);
+       u32 insn;
+
+       if (type == JUMP_LABEL_JMP) {
+               u32 offset = jump_entry_target(entry) - jump_entry_code(entry);
+
+               insn = RISCV_INSN_JAL |
+                       ((offset & GENMASK(19, 12)) << (12 - 12)) |
+                       ((offset & GENMASK(11, 11)) << (20 - 11)) |
+                       ((offset & GENMASK(10,  1)) << (21 -  1)) |
+                       ((offset & GENMASK(20, 20)) << (31 - 20));
+       } else
+               insn = RISCV_INSN_NOP;
+
+       patch_text_nosync(addr, &insn, sizeof(insn));
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       /*
+        * We use the same instructions in the arch_static_branch and
+        * arch_static_branch_jump inline functions, so there's no
+        * need to patch them up here.
+        * The core will call arch_jump_label_transform  when those
+        * instructions need to be replaced.
+        */
+}
--

2.27.0

#4 lore raw
In reply to Emil Renner Berthing
Quoted 5 lines · Emil Renner Berthing

Add jump-label implementation based on the ARM64 version.

Tested on the HiFive Unleashed board.

I took your patch for a spin on qemu. The boot selftest (CONFIG_STATIC_KEYS_SELFTEST=y) passes, but the module test (CONFIG_TEST_STATIC_KEYS=m) does not.

When I run the in "test tools/testing/selftests/static_keys" (this simply loads the test_static_key_base.ko and test_static_keys.ko modules) I get:

[  134.090464] test_static_keys: Unknown relocation type 36

36 is the relocation type R_RISCV_ADD64, which is not handled in arch/riscv/kernel/module.c.

If you dump the relocation entries for test_static_keys.ko (riscv64-linux-gnu-objdump -r test_static_keys.ko), you'll see that:

RELOCATION RECORDS FOR [__jump_table]:

OFFSET           TYPE              VALUE
0000000000000000 R_RISCV_ADD32     .L1^B1
0000000000000000 R_RISCV_SUB32     .L0
0000000000000004 R_RISCV_ADD32     .L3
0000000000000004 R_RISCV_SUB32     .L0
0000000000000008 R_RISCV_ADD64     old_true_key+0x0000000000000001
0000000000000008 R_RISCV_SUB64     .L0

...

It would be great if you could add a patch for that as well (separate, same series). R_RISCV_ADD64 *and* R_RISCV_SUB64 are currently unhandled by module.c.

Cheers,
Björn

Quoted 173 lines

---

Changes since RFC:
- Use RISCV_PTR and RISCV_LGPTR macros to match struct jump_table also in 32bit kernels.
- Remove unneeded branch ? 1 : 0, thanks Björn
- Fix \n\n instead of \n\t mistake

 .../core/jump-labels/arch-support.txt         |  2 +-
 arch/riscv/Kconfig                            |  2 +
 arch/riscv/include/asm/jump_label.h           | 59 +++++++++++++++++++
 arch/riscv/kernel/Makefile                    |  2 +
 arch/riscv/kernel/jump_label.c                | 44 ++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/jump_label.h
 create mode 100644 arch/riscv/kernel/jump_label.c
Documentation/features/core/jump-labels/arch-support.txt +1 -1
index 632a1c7aefa2..760243d18ed7 100644
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
arch/riscv/Kconfig +2 -0
index fd639937e251..d2f5c53fdc19 100644
@@ -46,6 +46,8 @@ config RISCV
        select GENERIC_TIME_VSYSCALL if MMU && 64BIT
        select HANDLE_DOMAIN_IRQ
        select HAVE_ARCH_AUDITSYSCALL
+       select HAVE_ARCH_JUMP_LABEL
+       select HAVE_ARCH_JUMP_LABEL_RELATIVE
        select HAVE_ARCH_KASAN if MMU && 64BIT
        select HAVE_ARCH_KGDB
        select HAVE_ARCH_KGDB_QXFER_PKT
arch/riscv/include/asm/jump_label.h new file +59 -0
new file mode 100644
index 000000000000..d5fb342bfccf
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+                                              bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     nop                                     \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+                                                   bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     jal             zero, %l[label]         \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+#endif  /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */
arch/riscv/kernel/Makefile +2 -0
index b355cf485671..a5287ab9f7f2 100644
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)      += cpu-hotplug.o
 obj-$(CONFIG_KGDB)             += kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
+
 clean:
arch/riscv/kernel/jump_label.c new file +44 -1
new file mode 100644
index 000000000000..55b2d742efe1
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013
+#define RISCV_INSN_JAL 0x0000006f
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       void *addr = (void *)jump_entry_code(entry);
+       u32 insn;
+
+       if (type == JUMP_LABEL_JMP) {
+               u32 offset = jump_entry_target(entry) - jump_entry_code(entry);
+
+               insn = RISCV_INSN_JAL |
+                       ((offset & GENMASK(19, 12)) << (12 - 12)) |
+                       ((offset & GENMASK(11, 11)) << (20 - 11)) |
+                       ((offset & GENMASK(10,  1)) << (21 -  1)) |
+                       ((offset & GENMASK(20, 20)) << (31 - 20));
+       } else
+               insn = RISCV_INSN_NOP;
+
+       patch_text_nosync(addr, &insn, sizeof(insn));
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       /*
+        * We use the same instructions in the arch_static_branch and
+        * arch_static_branch_jump inline functions, so there's no
+        * need to patch them up here.
+        * The core will call arch_jump_label_transform  when those
+        * instructions need to be replaced.
+        */
+}
--

2.27.0

#2 lore raw
In reply to Björn Töpel
Quoted 151 lines · Björn Töpel
Quoted 3 lines · Emil Renner Berthing

Add jump-label implementation based on the ARM64 version.

Thanks for working on this!

Quoted 144 lines

Tested on the HiFive Unleashed board.

---

Changes since RFC:
- Use RISCV_PTR and RISCV_LGPTR macros to match struct jump_table also in 32bit kernels.
- Remove unneeded branch ? 1 : 0, thanks Björn
- Fix \n\n instead of \n\t mistake

 .../core/jump-labels/arch-support.txt         |  2 +-
 arch/riscv/Kconfig                            |  2 +
 arch/riscv/include/asm/jump_label.h           | 59 +++++++++++++++++++
 arch/riscv/kernel/Makefile                    |  2 +
 arch/riscv/kernel/jump_label.c                | 44 ++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/jump_label.h
 create mode 100644 arch/riscv/kernel/jump_label.c
Documentation/features/core/jump-labels/arch-support.txt +1 -1
index 632a1c7aefa2..760243d18ed7 100644
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
arch/riscv/Kconfig +2 -0
index fd639937e251..d2f5c53fdc19 100644
@@ -46,6 +46,8 @@ config RISCV
        select GENERIC_TIME_VSYSCALL if MMU && 64BIT
        select HANDLE_DOMAIN_IRQ
        select HAVE_ARCH_AUDITSYSCALL
+       select HAVE_ARCH_JUMP_LABEL
+       select HAVE_ARCH_JUMP_LABEL_RELATIVE
        select HAVE_ARCH_KASAN if MMU && 64BIT
        select HAVE_ARCH_KGDB
        select HAVE_ARCH_KGDB_QXFER_PKT
arch/riscv/include/asm/jump_label.h new file +59 -0
new file mode 100644
index 000000000000..d5fb342bfccf
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+                                              bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     nop                                     \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+                                                   bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     jal             zero, %l[label]         \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+#endif  /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */
arch/riscv/kernel/Makefile +2 -0
index b355cf485671..a5287ab9f7f2 100644
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)      += cpu-hotplug.o
 obj-$(CONFIG_KGDB)             += kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
+
 clean:
arch/riscv/kernel/jump_label.c new file +16 -0
new file mode 100644
index 000000000000..55b2d742efe1
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013
+#define RISCV_INSN_JAL 0x0000006f
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
Quoted 9 lines

arch/riscv/kernel/jump_label.c line 21

Quoted 5 lines
arch/riscv/kernel/jump_label.c new file +5 -0
+       void *addr = (void *)jump_entry_code(entry);
+       u32 insn;
+
+       if (type == JUMP_LABEL_JMP) {
+               u32 offset = jump_entry_target(entry) - jump_entry_code(entry);

The 20b jump offset of JAL is enough, but to catch future changes that might potentially break this, I would add a WARN to catch if offset

Quoted 1 line

20b (or overkill fallback to JALR -- don't do that.).

You're right. Not all architectures have warnings, but all arch's but parisc and xtensa supports offsets > 20b. I'll add a WARN.

Quoted 4 lines

Somewhat related to that; The UNIX specification for RISC-V requires support for RVC. Would it make sense to use the C.-instructions C.J and C.NOP? Really a nit, or more of a question. I'm fine with this 32b version!

That would definitely be possible, but then we only get 11b of offset available. That's even lower than the 17b of parisc and xtensa. That would probably need a CONFIG flag unless someone can come up with a way to estimate the offset at compile time.

Alternatively I can't see a way that two compressed instructions could combine to a jump with more than the 20b of offset, and I can't imagine two compressed nops would be better than one 32b nop, so unless you're willing to live with 11b offsets I think we're better off with the 32b instructions.

Quoted 10 lines

arch/riscv/kernel/jump_label.c line 29

Quoted 8 lines
arch/riscv/kernel/jump_label.c new file +8 -0
+
+               insn = RISCV_INSN_JAL |
+                       ((offset & GENMASK(19, 12)) << (12 - 12)) |
+                       ((offset & GENMASK(11, 11)) << (20 - 11)) |
+                       ((offset & GENMASK(10,  1)) << (21 -  1)) |
+                       ((offset & GENMASK(20, 20)) << (31 - 20));
+       } else
+               insn = RISCV_INSN_NOP;

Since the if-statement needs {}, the else-clause should have one too.

Will fix.

Quoted 2 lines

Feel free to add:

Thanks!

Quoted 20 lines

Cheers,
Björn

Quoted 18 lines
arch/riscv/kernel/jump_label.c new file +15 -1
+
+       patch_text_nosync(addr, &insn, sizeof(insn));
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       /*
+        * We use the same instructions in the arch_static_branch and
+        * arch_static_branch_jump inline functions, so there's no
+        * need to patch them up here.
+        * The core will call arch_jump_label_transform  when those
+        * instructions need to be replaced.
+        */
+}
--

2.27.0

#5 lore raw
In reply to Björn Töpel
Quoted 32 lines · Björn Töpel
Quoted 5 lines · Emil Renner Berthing

Add jump-label implementation based on the ARM64 version.

Tested on the HiFive Unleashed board.

I took your patch for a spin on qemu. The boot selftest (CONFIG_STATIC_KEYS_SELFTEST=y) passes, but the module test (CONFIG_TEST_STATIC_KEYS=m) does not.

When I run the in "test tools/testing/selftests/static_keys" (this simply loads the test_static_key_base.ko and test_static_keys.ko modules) I get:

[  134.090464] test_static_keys: Unknown relocation type 36

36 is the relocation type R_RISCV_ADD64, which is not handled in arch/riscv/kernel/module.c.

If you dump the relocation entries for test_static_keys.ko (riscv64-linux-gnu-objdump -r test_static_keys.ko), you'll see that:

RELOCATION RECORDS FOR [__jump_table]:

OFFSET           TYPE              VALUE
0000000000000000 R_RISCV_ADD32     .L1^B1
0000000000000000 R_RISCV_SUB32     .L0
0000000000000004 R_RISCV_ADD32     .L3
0000000000000004 R_RISCV_SUB32     .L0
0000000000000008 R_RISCV_ADD64     old_true_key+0x0000000000000001
0000000000000008 R_RISCV_SUB64     .L0
Quoted 5 lines

...

It would be great if you could add a patch for that as well (separate, same series). R_RISCV_ADD64 *and* R_RISCV_SUB64 are currently unhandled by module.c.

On it! I'll also add the CONFIG_JUMP_LABEL to defconfig to the next version.

Quoted 177 lines

Cheers,
Björn

Quoted 173 lines

---

Changes since RFC:
- Use RISCV_PTR and RISCV_LGPTR macros to match struct jump_table also in 32bit kernels.
- Remove unneeded branch ? 1 : 0, thanks Björn
- Fix \n\n instead of \n\t mistake

 .../core/jump-labels/arch-support.txt         |  2 +-
 arch/riscv/Kconfig                            |  2 +
 arch/riscv/include/asm/jump_label.h           | 59 +++++++++++++++++++
 arch/riscv/kernel/Makefile                    |  2 +
 arch/riscv/kernel/jump_label.c                | 44 ++++++++++++++
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 arch/riscv/include/asm/jump_label.h
 create mode 100644 arch/riscv/kernel/jump_label.c
Documentation/features/core/jump-labels/arch-support.txt +1 -1
index 632a1c7aefa2..760243d18ed7 100644
@@ -23,7 +23,7 @@
     |    openrisc: | TODO |
     |      parisc: |  ok  |
     |     powerpc: |  ok  |
-    |       riscv: | TODO |
+    |       riscv: |  ok  |
     |        s390: |  ok  |
     |          sh: | TODO |
     |       sparc: |  ok  |
arch/riscv/Kconfig +2 -0
index fd639937e251..d2f5c53fdc19 100644
@@ -46,6 +46,8 @@ config RISCV
        select GENERIC_TIME_VSYSCALL if MMU && 64BIT
        select HANDLE_DOMAIN_IRQ
        select HAVE_ARCH_AUDITSYSCALL
+       select HAVE_ARCH_JUMP_LABEL
+       select HAVE_ARCH_JUMP_LABEL_RELATIVE
        select HAVE_ARCH_KASAN if MMU && 64BIT
        select HAVE_ARCH_KGDB
        select HAVE_ARCH_KGDB_QXFER_PKT
arch/riscv/include/asm/jump_label.h new file +59 -0
new file mode 100644
index 000000000000..d5fb342bfccf
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/include/asm/jump_label.h
+ */
+#ifndef __ASM_JUMP_LABEL_H
+#define __ASM_JUMP_LABEL_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+static __always_inline bool arch_static_branch(struct static_key *key,
+                                              bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     nop                                     \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key,
+                                                   bool branch)
+{
+       asm_volatile_goto(
+               "       .option push                            \n\t"
+               "       .option norelax                         \n\t"
+               "       .option norvc                           \n\t"
+               "1:     jal             zero, %l[label]         \n\t"
+               "       .option pop                             \n\t"
+               "       .pushsection    __jump_table, \"aw\"    \n\t"
+               "       .align          " RISCV_LGPTR "         \n\t"
+               "       .long           1b - ., %l[label] - .   \n\t"
+               "       " RISCV_PTR "   %0 - .                  \n\t"
+               "       .popsection                             \n\t"
+               :  :  "i"(&((char *)key)[branch]) :  : label);
+
+       return false;
+label:
+       return true;
+}
+
+#endif  /* __ASSEMBLY__ */
+#endif /* __ASM_JUMP_LABEL_H */
arch/riscv/kernel/Makefile +2 -0
index b355cf485671..a5287ab9f7f2 100644
@@ -53,4 +53,6 @@ endif
 obj-$(CONFIG_HOTPLUG_CPU)      += cpu-hotplug.o
 obj-$(CONFIG_KGDB)             += kgdb.o
 
+obj-$(CONFIG_JUMP_LABEL)       += jump_label.o
+
 clean:
arch/riscv/kernel/jump_label.c new file +44 -1
new file mode 100644
index 000000000000..55b2d742efe1
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 Emil Renner Berthing
+ *
+ * Based on arch/arm64/kernel/jump_label.c
+ */
+#include <linux/kernel.h>
+#include <linux/jump_label.h>
+#include <asm/patch.h>
+
+#define RISCV_INSN_NOP 0x00000013
+#define RISCV_INSN_JAL 0x0000006f
+
+void arch_jump_label_transform(struct jump_entry *entry,
+                              enum jump_label_type type)
+{
+       void *addr = (void *)jump_entry_code(entry);
+       u32 insn;
+
+       if (type == JUMP_LABEL_JMP) {
+               u32 offset = jump_entry_target(entry) - jump_entry_code(entry);
+
+               insn = RISCV_INSN_JAL |
+                       ((offset & GENMASK(19, 12)) << (12 - 12)) |
+                       ((offset & GENMASK(11, 11)) << (20 - 11)) |
+                       ((offset & GENMASK(10,  1)) << (21 -  1)) |
+                       ((offset & GENMASK(20, 20)) << (31 - 20));
+       } else
+               insn = RISCV_INSN_NOP;
+
+       patch_text_nosync(addr, &insn, sizeof(insn));
+}
+
+void arch_jump_label_transform_static(struct jump_entry *entry,
+                                     enum jump_label_type type)
+{
+       /*
+        * We use the same instructions in the arch_static_branch and
+        * arch_static_branch_jump inline functions, so there's no
+        * need to patch them up here.
+        * The core will call arch_jump_label_transform  when those
+        * instructions need to be replaced.
+        */
+}
--

2.27.0