use static key to optimize pgtable_l4_enabled
v3 cover use static key to optimize pgtable_l4_enabled
The pgtable_l4|[l5]_enabled check sits at hot code path, performance is impacted a lot. Since pgtable_l4|[l5]_enabled isn't changed after boot, so static key can be used to solve the issue[1].
An unified way static key was introduced in [2], but it's only targets riscv isa extension. We dunno whether SV48 and SV57 will be considered as isa extension, so the unified solution isn't used for pgtable_l4[l5]_enabled.
patch1 fix a NULL pointer deference if static key is used a bit earlier. patch2 uses the static key to optimize pgtable_l4|[l5]_enabled.
[1] http://lists.infradead.org/pipermail/linux-riscv/2021-December/011164.html
[2] https://lore.kernel.org/linux-riscv/[email protected]/T/#t
Since v2:
- move the W=1 warning fix to a separate patch
- move the unified way to use static key to a new patch series.
Since v1:
- Add a W=1 warning fix
- Fix W=1 error
- Based on v5.18-rcN, since SV57 support is added, so convert pgtable_l5_enabled as well.
Jisheng Zhang (2):
riscv: move sbi_init() earlier before jump_label_init()
riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
arch/riscv/include/asm/pgalloc.h | 16 ++++---- arch/riscv/include/asm/pgtable-32.h | 3 ++ arch/riscv/include/asm/pgtable-64.h | 59 +++++++++++++++++--------- arch/riscv/include/asm/pgtable.h | 5 +-- arch/riscv/kernel/cpu.c | 4 +- arch/riscv/kernel/setup.c | 2 +- arch/riscv/mm/init.c | 64 ++++++++++++++++++----------- arch/riscv/mm/kasan_init.c | 16 ++++---- 8 files changed, 103 insertions(+), 66 deletions(-)
Signature · 1 line
2.34.1
PATCH v3 1/2 riscv: move sbi_init() earlier before jump_label_init()
We call jump_label_init() in setup_arch() is to use static key mechanism earlier, but riscv jump label relies on the sbi functions, If we enable static key before sbi_init(), the code path looks like:
static_branch_enable()
..
arch_jump_label_transform()
patch_text_nosync()
flush_icache_range()
flush_icache_all()
sbi_remote_fence_i() for CONFIG_RISCV_SBI case
__sbi_rfence()Since sbi isn't initialized, so NULL deference! Here is a typical panic log:
[ 0.000000] Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 [ 0.000000] Oops [#1] [ 0.000000] Modules linked in: [ 0.000000] CPU: 0 PID: 0 Comm: swapper Not tainted 5.18.0-rc7+ #79 [ 0.000000] Hardware name: riscv-virtio,qemu (DT) [ 0.000000] epc : 0x0 [ 0.000000] ra : sbi_remote_fence_i+0x1e/0x26 [ 0.000000] epc : 0000000000000000 ra : ffffffff80005826 sp : ffffffff80c03d50 [ 0.000000] gp : ffffffff80ca6178 tp : ffffffff80c0ad80 t0 : 6200000000000000 [ 0.000000] t1 : 0000000000000000 t2 : 62203a6b746e6972 s0 : ffffffff80c03d60 [ 0.000000] s1 : ffffffff80001af6 a0 : 0000000000000000 a1 : 0000000000000000 [ 0.000000] a2 : 0000000000000000 a3 : 0000000000000000 a4 : 0000000000000000 [ 0.000000] a5 : 0000000000000000 a6 : 0000000000000000 a7 : 0000000000080200 [ 0.000000] s2 : ffffffff808b3e48 s3 : ffffffff808bf698 s4 : ffffffff80cb2818 [ 0.000000] s5 : 0000000000000001 s6 : ffffffff80c9c345 s7 : ffffffff80895aa0 [ 0.000000] s8 : 0000000000000001 s9 : 000000000000007f s10: 0000000000000000 [ 0.000000] s11: 0000000000000000 t3 : ffffffff80824d08 t4 : 0000000000000022 [ 0.000000] t5 : 000000000000003d t6 : 0000000000000000 [ 0.000000] status: 0000000000000100 badaddr: 0000000000000000 cause: 000000000000000c [ 0.000000] ---[ end trace 0000000000000000 ]--- [ 0.000000] Kernel panic - not syncing: Attempted to kill the idle task! [ 0.000000] ---[ end Kernel panic - not syncing: Attempted to kill the idle task! ]---
Fix this issue by moving sbi_init() earlier before jump_label_init()
--- arch/riscv/kernel/setup.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/riscv/kernel/setup.c +1 -1
@@ -268,6 +268,7 @@ void __init setup_arch(char **cmdline_p) *cmdline_p = boot_command_line;early_ioremap_setup (); +sbi_init ();jump_label_init ();parse_early_param ();
@@ -284,7 +285,6 @@ void __init setup_arch(char **cmdline_p)misc_mem_init ();init_resources (); -sbi_init ();#ifdef CONFIG_KASAN kasan_init ();
Signature · 7 lines
2.34.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
PATCH v3 2/2 riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
In reply to Jisheng ZhangOn a specific HW platform, pgtable_l4|[l5]_enabled won't change after boot, and the check sits at hot code path, this characteristic makes it suitable for optimization with static key.
_pgtable_l4|[l5]_enabled is used very early during boot, even is used with MMU off, so the static key mechanism isn't ready. For this case, we use another static key _pgtable_lx_ready to indicate whether we have finalised pgtable_l4|[l5]_enabled or not, then fall back to _pgtable_l4|[l5]_enabled_early bool.
--- arch/riscv/include/asm/pgalloc.h | 16 ++++---- arch/riscv/include/asm/pgtable-32.h | 3 ++ arch/riscv/include/asm/pgtable-64.h | 59 +++++++++++++++++--------- arch/riscv/include/asm/pgtable.h | 5 +-- arch/riscv/kernel/cpu.c | 4 +- arch/riscv/mm/init.c | 64 ++++++++++++++++++----------- arch/riscv/mm/kasan_init.c | 16 ++++---- 7 files changed, 102 insertions(+), 65 deletions(-)
arch/riscv/include/asm/pgalloc.h +8 -8
@@ -41,7 +41,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)static inline void p4d_populate (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud);set_p4d (p4d,__p4d ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
@@ -51,7 +51,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)static inline void p4d_populate_safe (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud); set_p4d_safe(p4d,
@@ -61,7 +61,7 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d);set_pgd (pgd,__pgd ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));
@@ -71,7 +71,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)static inline void pgd_populate_safe (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d); set_pgd_safe(pgd,
@@ -82,7 +82,7 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,#define pud_alloc_one pud_alloc_onestatic inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return __pud_alloc_one (mm, addr);return NULL ;
@@ -91,7 +91,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)#define pud_free pud_freestatic inline void pud_free (struct mm_struct *mm,pud_t *pud) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())__pud_free (mm, pud); }
@@ -100,7 +100,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)#define p4d_alloc_one p4d_alloc_onestatic inline p4d_t *p4d_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {gfp_t gfp =GFP_PGTABLE_USER ;if (mm == &init_mm)
@@ -120,7 +120,7 @@ static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)#define p4d_free p4d_freestatic inline void p4d_free (struct mm_struct *mm,p4d_t *p4d) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())__p4d_free (mm, p4d); }
arch/riscv/include/asm/pgtable-32.h +3 -0
@@ -16,4 +16,7 @@#define MAX_POSSIBLE_PHYSMEM_BITS 34 +#define pgtable_l5_enabled () 0 +#define pgtable_l4_enabled () 0 +#endif /* _ASM_RISCV_PGTABLE_32_H */
arch/riscv/include/asm/pgtable-64.h +39 -20
166 lines not on this page · raw message
arch/riscv/include/asm/pgtable.h +2 -3
@@ -63,8 +63,8 @@* position vmemmap directly below the VMALLOC region. */ #ifdef CONFIG_64BIT -#define VA_BITS (pgtable_l5_enabled ? \ - 57 : (pgtable_l4_enabled ? 48 : 39)) +#define VA_BITS (pgtable_l5_enabled() ? \ + 57 : (pgtable_l4_enabled() ? 48 : 39))#else #define VA_BITS 32#endif
@@ -738,7 +738,6 @@ extern uintptr_t _dtb_early_pa;#define dtb_early_pa _dtb_early_pa#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; -extern bool pgtable_l4_enabled;void paging_init (void );void misc_mem_init (void );
arch/riscv/kernel/cpu.c +2 -2
@@ -141,9 +141,9 @@ static void print_mmu(struct seq_file *f)#if defined(CONFIG_32BIT )strncpy (sv_type,"sv32" ,5 );#elif defined(CONFIG_64BIT ) -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())strncpy (sv_type,"sv57" ,5 ); -else if (pgtable_l4_enabled) +else if (pgtable_l4_enabled ())strncpy (sv_type,"sv48" ,5 );else strncpy (sv_type,"sv39" ,5 );
arch/riscv/mm/init.c +40 -24
130 lines not on this page · raw message
arch/riscv/mm/kasan_init.c +8 -8
@@ -209,15 +209,15 @@ static void __init kasan_populate_p4d(pgd_t *pgd,set_pgd (pgd,pfn_pgd (PFN_DOWN (__pa (base_p4d)),PAGE_TABLE )); } -#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \ +#define kasan_early_shadow_pgd_next (pgtable_l5_enabled() ? \ (uintptr_t)kasan_early_shadow_p4d : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ (uintptr_t)kasan_early_shadow_pud : \ (uintptr_t)kasan_early_shadow_pmd))#define kasan_populate_pgd_next (pgdp, vaddr, next, early) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_populate_p4d(pgdp, vaddr, next, early) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_populate_pud(pgdp, vaddr, next, early) : \ kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))
@@ -274,7 +274,7 @@ asmlinkage void __init kasan_early_init(void) (__pa ((uintptr_t )kasan_early_shadow_pte)), PAGE_TABLE)); -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {for (i =0 ; i <PTRS_PER_PUD ; ++i) set_pud(kasan_early_shadow_pud + i, pfn_pud(PFN_DOWN
@@ -282,7 +282,7 @@ asmlinkage void __init kasan_early_init(void) PAGE_TABLE)); } -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {for (i =0 ; i <PTRS_PER_P4D ; ++i) set_p4d(kasan_early_shadow_p4d + i, pfn_p4d(PFN_DOWN
@@ -393,9 +393,9 @@ static void __init kasan_shallow_populate_p4d(pgd_t *pgdp, }#define kasan_shallow_populate_pgd_next (pgdp, vaddr, next) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_shallow_populate_p4d(pgdp, vaddr, next) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_shallow_populate_pud(pgdp, vaddr, next) : \ kasan_shallow_populate_pmd(pgdp, vaddr, next)))
Signature · 7 lines
2.34.1 _______________________________________________ linux-riscv mailing list [email protected] http://lists.infradead.org/mailman/listinfo/linux-riscv
kernel test robot
build botRe: [PATCH v3 2/2] riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
I love your patch! Yet something to improve:
Hi Jisheng,
I love your patch! Yet something to improve:
[auto build test ERROR on linus/master]
[also build test ERROR on v5.18-rc7]
[cannot apply to next-20220519]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/intel-lab-lkp/linux/commits/Jisheng-Zhang/use-static-key-to-optimize-pgtable_l4_enabled/20220520-001459 base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git f993aed406eaf968ba3867a76bb46c95336a33d0 config: riscv-buildonly-randconfig-r003-20220519 (https://download.01.org/0day-ci/archive/20220520/[email protected]/config) compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project e00cbbec06c08dc616a0d52a20f678b8fbd4e304)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# install riscv cross compiling tool for clang build
# apt-get install binutils-riscv64-linux-gnu
# https://github.com/intel-lab-lkp/linux/commit/d052c69ebaf48ac2925d6f9fa033d9e394da1074
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Jisheng-Zhang/use-static-key-to-optimize-pgtable_l4_enabled/20220520-001459 git checkout d052c69ebaf48ac2925d6f9fa033d9e394da1074
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=riscv SHELL=/bin/bash drivers/tty/ net/ceph/
If you fix the issue, kindly add following tag as appropriate
All error/warnings (new ones prefixed by >>):
In file included from drivers/tty/tty_io.c:73:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:11:
In file included from include/linux/uaccess.h:11:
In file included from arch/riscv/include/asm/uaccess.h:12:
In file included from arch/riscv/include/asm/pgtable.h:112:
>> arch/riscv/include/asm/pgtable-64.h:19:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
arch/riscv/include/asm/pgtable-64.h:27:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
In file included from drivers/tty/tty_io.c:75:
In file included from include/linux/interrupt.h:11:
In file included from include/linux/hardirq.h:11:
In file included from ./arch/riscv/include/generated/asm/hardirq.h:1:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:464:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __raw_readb(PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:477:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:37:51: note: expanded from macro '__le16_to_cpu'
#define __le16_to_cpu(x) ((__force __u16)(__le16)(x))
^
In file included from drivers/tty/tty_io.c:75:
In file included from include/linux/interrupt.h:11:
In file included from include/linux/hardirq.h:11:
In file included from ./arch/riscv/include/generated/asm/hardirq.h:1:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:490:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:35:51: note: expanded from macro '__le32_to_cpu'
#define __le32_to_cpu(x) ((__force __u32)(__le32)(x))
^
In file included from drivers/tty/tty_io.c:75:
In file included from include/linux/interrupt.h:11:
In file included from include/linux/hardirq.h:11:
In file included from ./arch/riscv/include/generated/asm/hardirq.h:1:
In file included from include/asm-generic/hardirq.h:17:
In file included from include/linux/irq.h:20:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:501:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writeb(value, PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:511:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:521:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:1024:55: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
return (port > MMIO_UPPER_LIMIT) ? NULL : PCI_IOBASE + port;
~~~~~~~~~~ ^ 7 warnings and 2 errors generated.
--
In file included from drivers/tty/tty_ioctl.c:11:
In file included from include/uapi/linux/termios.h:6:
In file included from ./arch/riscv/include/generated/uapi/asm/termios.h:1:
In file included from include/asm-generic/termios.h:6:
In file included from include/linux/uaccess.h:11:
In file included from arch/riscv/include/asm/uaccess.h:12:
In file included from arch/riscv/include/asm/pgtable.h:112:
>> arch/riscv/include/asm/pgtable-64.h:19:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
arch/riscv/include/asm/pgtable-64.h:27:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:97:11: warning: array index 3 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set->sig[3] | set->sig[2] |
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:97:25: warning: array index 2 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set->sig[3] | set->sig[2] |
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:98:4: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
set->sig[1] | set->sig[0]) == 0;
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:100:11: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set->sig[1] | set->sig[0]) == 0;
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:113:11: warning: array index 3 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set1->sig[3] == set2->sig[3]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:113:27: warning: array index 3 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set1->sig[3] == set2->sig[3]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:114:5: warning: array index 2 is past the end of the array (which contains 1 element) [-Warray-bounds]
(set1->sig[2] == set2->sig[2]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:114:21: warning: array index 2 is past the end of the array (which contains 1 element) [-Warray-bounds]
(set1->sig[2] == set2->sig[2]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:115:5: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
(set1->sig[1] == set2->sig[1]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:115:21: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
(set1->sig[1] == set2->sig[1]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:118:11: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set1->sig[1] == set2->sig[1]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/tty_ioctl.c:13:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:118:27: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set1->sig[1] == set2->sig[1]) &&
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];--
In file included from drivers/tty/tty_port.c:8:
In file included from include/linux/tty.h:5:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:9:
In file included from include/linux/sched/task.h:11:
In file included from include/linux/uaccess.h:11:
In file included from arch/riscv/include/asm/uaccess.h:12:
In file included from arch/riscv/include/asm/pgtable.h:112:
>> arch/riscv/include/asm/pgtable-64.h:19:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
arch/riscv/include/asm/pgtable-64.h:27:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
In file included from drivers/tty/tty_port.c:8:
In file included from include/linux/tty.h:12:
In file included from include/linux/tty_port.h:5:
In file included from include/linux/kfifo.h:42:
In file included from include/linux/scatterlist.h:9:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:464:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __raw_readb(PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:477:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:37:51: note: expanded from macro '__le16_to_cpu'
#define __le16_to_cpu(x) ((__force __u16)(__le16)(x))
^
In file included from drivers/tty/tty_port.c:8:
In file included from include/linux/tty.h:12:
In file included from include/linux/tty_port.h:5:
In file included from include/linux/kfifo.h:42:
In file included from include/linux/scatterlist.h:9:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:490:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:35:51: note: expanded from macro '__le32_to_cpu'
#define __le32_to_cpu(x) ((__force __u32)(__le32)(x))
^
In file included from drivers/tty/tty_port.c:8:
In file included from include/linux/tty.h:12:
In file included from include/linux/tty_port.h:5:
In file included from include/linux/kfifo.h:42:
In file included from include/linux/scatterlist.h:9:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:501:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writeb(value, PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:511:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:521:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:1024:55: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
return (port > MMIO_UPPER_LIMIT) ? NULL : PCI_IOBASE + port;
~~~~~~~~~~ ^
>> drivers/tty/tty_port.c:245:2: warning: implicit conversion from 'unsigned long' to 'unsigned int' changes value from 18446744073709551615 to 4294967295 [-Wconstant-conversion]
INIT_KFIFO(port->xmit_fifo);
^~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/kfifo.h:130:69: note: expanded from macro 'INIT_KFIFO'
__kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
~ ~~~~~~~~~~~~~~~~~~~~~~~^~~ 8 warnings and 2 errors generated.
--
In file included from drivers/tty/serial/earlycon.c:16:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:15:
In file included from include/linux/pgtable.h:6:
In file included from arch/riscv/include/asm/pgtable.h:112:
>> arch/riscv/include/asm/pgtable-64.h:19:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
arch/riscv/include/asm/pgtable-64.h:27:6: error: call to undeclared function 'static_branch_likely'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
if (static_branch_likely(&_pgtable_lx_ready))
^
In file included from drivers/tty/serial/earlycon.c:16:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:464:31: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __raw_readb(PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:477:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le16_to_cpu((__le16 __force)__raw_readw(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:37:51: note: expanded from macro '__le16_to_cpu'
#define __le16_to_cpu(x) ((__force __u16)(__le16)(x))
^
In file included from drivers/tty/serial/earlycon.c:16:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:490:61: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
val = __le32_to_cpu((__le32 __force)__raw_readl(PCI_IOBASE + addr));
~~~~~~~~~~ ^
include/uapi/linux/byteorder/little_endian.h:35:51: note: expanded from macro '__le32_to_cpu'
#define __le32_to_cpu(x) ((__force __u32)(__le32)(x))
^
In file included from drivers/tty/serial/earlycon.c:16:
In file included from include/linux/io.h:13:
In file included from arch/riscv/include/asm/io.h:136:
include/asm-generic/io.h:501:33: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writeb(value, PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:511:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writew((u16 __force)cpu_to_le16(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:521:59: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
__raw_writel((u32 __force)cpu_to_le32(value), PCI_IOBASE + addr);
~~~~~~~~~~ ^
include/asm-generic/io.h:1024:55: warning: performing pointer arithmetic on a null pointer has undefined behavior [-Wnull-pointer-arithmetic]
return (port > MMIO_UPPER_LIMIT) ? NULL : PCI_IOBASE + port;
~~~~~~~~~~ ^
In file included from drivers/tty/serial/earlycon.c:17:
In file included from include/linux/serial_core.h:13:
In file included from include/linux/interrupt.h:21:
In file included from arch/riscv/include/asm/sections.h:9:
In file included from include/linux/mm.h:700:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:97:11: warning: array index 3 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set->sig[3] | set->sig[2] |
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/serial/earlycon.c:17:
In file included from include/linux/serial_core.h:13:
In file included from include/linux/interrupt.h:21:
In file included from arch/riscv/include/asm/sections.h:9:
In file included from include/linux/mm.h:700:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:97:25: warning: array index 2 is past the end of the array (which contains 1 element) [-Warray-bounds]
return (set->sig[3] | set->sig[2] |
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^
In file included from drivers/tty/serial/earlycon.c:17:
In file included from include/linux/serial_core.h:13:
In file included from include/linux/interrupt.h:21:
In file included from arch/riscv/include/asm/sections.h:9:
In file included from include/linux/mm.h:700:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:98:4: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
set->sig[1] | set->sig[0]) == 0;
^ ~
include/uapi/asm-generic/signal.h:62:2: note: array 'sig' declared here
unsigned long sig[_NSIG_WORDS];
^ In file included from drivers/tty/serial/earlycon.c:17:
In file included from include/linux/serial_core.h:13:
In file included from include/linux/interrupt.h:21:
In file included from arch/riscv/include/asm/sections.h:9:
In file included from include/linux/mm.h:700:
In file included from include/linux/huge_mm.h:8:
In file included from include/linux/fs.h:33:
In file included from include/linux/percpu-rwsem.h:7:
In file included from include/linux/rcuwait.h:6:
In file included from include/linux/sched/signal.h:6:
include/linux/signal.h:100:11: warning: array index 1 is past the end of the array (which contains 1 element) [-Warray-bounds]
..
vim +/static_branch_likely +19 arch/riscv/include/asm/pgtable-64.h
16
17 static __always_inline bool pgtable_l5_enabled(void)
18 {
> 19 if (static_branch_likely(&_pgtable_lx_ready))
20 return static_branch_likely(&_pgtable_l5_enabled);
21 else
22 return _pgtable_l5_enabled_early;
23 }
24 Signature · 2 lines
0-DAY CI Kernel Test Service https://01.org/lkp
kernel test robot
build botRe: [PATCH v3 2/2] riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
In reply to Jisheng ZhangI love your patch! Yet something to improve:
Hi Jisheng,
I love your patch! Yet something to improve:
[auto build test ERROR on linus/master]
[also build test ERROR on v5.18-rc7]
[cannot apply to next-20220519]
[If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/intel-lab-lkp/linux/commits/Jisheng-Zhang/use-static-key-to-optimize-pgtable_l4_enabled/20220520-001459 base: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git f993aed406eaf968ba3867a76bb46c95336a33d0 config: riscv-allnoconfig (https://download.01.org/0day-ci/archive/20220520/[email protected]/config) compiler: riscv64-linux-gcc (GCC) 11.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/d052c69ebaf48ac2925d6f9fa033d9e394da1074
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Jisheng-Zhang/use-static-key-to-optimize-pgtable_l4_enabled/20220520-001459 git checkout d052c69ebaf48ac2925d6f9fa033d9e394da1074
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-11.3.0 make.cross W=1 O=build_dir ARCH=riscv SHELL=/bin/bash fs/
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
In file included from arch/riscv/include/asm/pgtable.h:112,
from arch/riscv/include/asm/uaccess.h:12,
from include/linux/uaccess.h:11,
from include/linux/sched/task.h:11,
from include/linux/sched/signal.h:9,
from include/linux/rcuwait.h:6,
from include/linux/percpu-rwsem.h:7,
from include/linux/fs.h:33,
from fs/char_dev.c:9:
arch/riscv/include/asm/pgtable-64.h: In function 'pgtable_l5_enabled':
>> arch/riscv/include/asm/pgtable-64.h:19:13: error: implicit declaration of function 'static_branch_likely' [-Werror=implicit-function-declaration] 19 | if (static_branch_likely(&_pgtable_lx_ready))
| ^~~~~~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors
--
In file included from arch/riscv/include/asm/pgtable.h:112,
from arch/riscv/include/asm/uaccess.h:12,
from include/linux/uaccess.h:11,
from include/linux/sched/task.h:11,
from include/linux/sched/signal.h:9,
from include/linux/rcuwait.h:6,
from include/linux/percpu-rwsem.h:7,
from include/linux/fs.h:33,
from include/uapi/linux/aio_abi.h:31,
from include/linux/syscalls.h:77,
from fs/d_path.c:2:
arch/riscv/include/asm/pgtable-64.h: In function 'pgtable_l5_enabled':
>> arch/riscv/include/asm/pgtable-64.h:19:13: error: implicit declaration of function 'static_branch_likely' [-Werror=implicit-function-declaration] 19 | if (static_branch_likely(&_pgtable_lx_ready))
| ^~~~~~~~~~~~~~~~~~~~
fs/d_path.c: At top level:
fs/d_path.c:318:7: warning: no previous prototype for 'simple_dname' [-Wmissing-prototypes]
318 | char *simple_dname(struct dentry *dentry, char *buffer, int buflen)
| ^~~~~~~~~~~~
cc1: some warnings being treated as errors
--
In file included from arch/riscv/include/asm/pgtable.h:112,
from arch/riscv/include/asm/uaccess.h:12,
from include/linux/uaccess.h:11,
from include/linux/sched/task.h:11,
from include/linux/sched/signal.h:9,
from include/linux/rcuwait.h:6,
from include/linux/percpu-rwsem.h:7,
from include/linux/fs.h:33,
from include/uapi/linux/aio_abi.h:31,
from include/linux/syscalls.h:77,
from fs/io_uring.c:45:
arch/riscv/include/asm/pgtable-64.h: In function 'pgtable_l5_enabled':
>> arch/riscv/include/asm/pgtable-64.h:19:13: error: implicit declaration of function 'static_branch_likely' [-Werror=implicit-function-declaration]
19 | if (static_branch_likely(&_pgtable_lx_ready))
| ^~~~~~~~~~~~~~~~~~~~
fs/io_uring.c: In function '__io_submit_flush_completions':
fs/io_uring.c:2660:40: warning: variable 'prev' set but not used [-Wunused-but-set-variable]
2660 | struct io_wq_work_node *node, *prev;
| ^~~~
cc1: some warnings being treated as errors
--
In file included from arch/riscv/include/asm/pgtable.h:112,
from arch/riscv/include/asm/uaccess.h:12,
from include/linux/uaccess.h:11,
from include/linux/sched/task.h:11,
from include/linux/sched/signal.h:9,
from include/linux/rcuwait.h:6,
from include/linux/percpu-rwsem.h:7,
from include/linux/fs.h:33,
from fs/proc/meminfo.c:2:
arch/riscv/include/asm/pgtable-64.h: In function 'pgtable_l5_enabled':
>> arch/riscv/include/asm/pgtable-64.h:19:13: error: implicit declaration of function 'static_branch_likely' [-Werror=implicit-function-declaration] 19 | if (static_branch_likely(&_pgtable_lx_ready))
| ^~~~~~~~~~~~~~~~~~~~
fs/proc/meminfo.c: At top level:
fs/proc/meminfo.c:22:28: warning: no previous prototype for 'arch_report_meminfo' [-Wmissing-prototypes]
22 | void __attribute__((weak)) arch_report_meminfo(struct seq_file *m)
| ^~~~~~~~~~~~~~~~~~~
cc1: some warnings being treated as errors
vim +/static_branch_likely +19 arch/riscv/include/asm/pgtable-64.h
16
17 static __always_inline bool pgtable_l5_enabled(void)
18 {
> 19 if (static_branch_likely(&_pgtable_lx_ready))
20 return static_branch_likely(&_pgtable_l5_enabled);
21 else
22 return _pgtable_l5_enabled_early;
23 }
24 Signature · 2 lines
0-DAY CI Kernel Test Service https://01.org/lkp
Re: [PATCH v3 2/2] riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
In reply to Jisheng ZhangQuoted 5 lines · Jisheng Zhang
On a specific HW platform, pgtable_l4|[l5]_enabled won't change after boot, and the check sits at hot code path, this characteristic makes it suitable for optimization with static key.
Quoted 7 lines
_pgtable_l4|[l5]_enabled is used very early during boot, even is used with MMU off, so the static key mechanism isn't ready. For this case, we use another static key
_pgtable_lx_readyto indicate whether we have finalised pgtable_l4|[l5]_enabled or not, then fall back to _pgtable_l4|[l5]_enabled_early bool.Signed-off-by Jisheng Zhang <[email protected]>
Overall, this patch looks good to me. Please fix the errors reported by autobuilders.
Regards,
Anup
Quoted 521 lines
--- arch/riscv/include/asm/pgalloc.h | 16 ++++---- arch/riscv/include/asm/pgtable-32.h | 3 ++ arch/riscv/include/asm/pgtable-64.h | 59 +++++++++++++++++--------- arch/riscv/include/asm/pgtable.h | 5 +-- arch/riscv/kernel/cpu.c | 4 +- arch/riscv/mm/init.c | 64 ++++++++++++++++++----------- arch/riscv/mm/kasan_init.c | 16 ++++---- 7 files changed, 102 insertions(+), 65 deletions(-)arch/riscv/include/asm/pgalloc.h +8
-8@@ -41,7 +41,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)static inline void p4d_populate (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud);set_p4d (p4d,__p4d ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));@@ -51,7 +51,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)static inline void p4d_populate_safe (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud); set_p4d_safe(p4d,@@ -61,7 +61,7 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d);set_pgd (pgd,__pgd ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));@@ -71,7 +71,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)static inline void pgd_populate_safe (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d); set_pgd_safe(pgd,@@ -82,7 +82,7 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,#define pud_alloc_one pud_alloc_onestatic inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return __pud_alloc_one (mm, addr);return NULL ;@@ -91,7 +91,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)#define pud_free pud_freestatic inline void pud_free (struct mm_struct *mm,pud_t *pud) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())__pud_free (mm, pud); }@@ -100,7 +100,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)#define p4d_alloc_one p4d_alloc_onestatic inline p4d_t *p4d_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {gfp_t gfp =GFP_PGTABLE_USER ;if (mm == &init_mm)@@ -120,7 +120,7 @@ static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)#define p4d_free p4d_freestatic inline void p4d_free (struct mm_struct *mm,p4d_t *p4d) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())__p4d_free (mm, p4d); }arch/riscv/include/asm/pgtable-32.h +3
-0@@ -16,4 +16,7 @@#define MAX_POSSIBLE_PHYSMEM_BITS 34 +#define pgtable_l5_enabled () 0 +#define pgtable_l4_enabled () 0 +#endif /* _ASM_RISCV_PGTABLE_32_H */ arch/riscv/include/asm/pgtable-64.h +39
-20@@ -8,16 +8,35 @@#include <linux/const.h> -extern bool pgtable_l4_enabled; -extern bool pgtable_l5_enabled; +extern bool _pgtable_l5_enabled_early; +extern bool _pgtable_l4_enabled_early; +extern struct static_key_false _pgtable_l5_enabled; +extern struct static_key_false _pgtable_l4_enabled; +extern struct static_key_false _pgtable_lx_ready; + +static __always_inline bool pgtable_l5_enabled(void) +{ +if (static_branch_likely (&_pgtable_lx_ready)) +return static_branch_likely (&_pgtable_l5_enabled); +else +return _pgtable_l5_enabled_early; +} + +static __always_inline bool pgtable_l4_enabled(void) +{ +if (static_branch_likely (&_pgtable_lx_ready)) +return static_branch_likely (&_pgtable_l4_enabled); +else +return _pgtable_l4_enabled_early; +}#define PGDIR_SHIFT_L3 30#define PGDIR_SHIFT_L4 39#define PGDIR_SHIFT_L5 48#define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) -#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ - (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) +#define PGDIR_SHIFT (pgtable_l5_enabled() ? PGDIR_SHIFT_L5 : \ + (pgtable_l4_enabled() ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))/* Size of region mapped by a page global directory */ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)#define PGDIR_MASK (~(PGDIR_SIZE - 1))@@ -119,7 +138,7 @@ static inline struct page *pud_page(pud_t pud)#define mm_p4d_folded mm_p4d_foldedstatic inline bool mm_p4d_folded (struct mm_struct *mm) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return false;return true;@@ -128,7 +147,7 @@ static inline bool mm_p4d_folded(struct mm_struct *mm)#define mm_pud_folded mm_pud_foldedstatic inline bool mm_pud_folded (struct mm_struct *mm) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return false;return true;@@ -159,7 +178,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd)static inline void set_p4d (p4d_t *p4dp,p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ()) *p4dp = p4d;else set_pud ((pud_t *)p4dp, (pud_t ){p4d_val (p4d) });@@ -167,7 +186,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)static inline int p4d_none (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (p4d_val (p4d) ==0 );return 0 ;@@ -175,7 +194,7 @@ static inline int p4d_none(p4d_t p4d)static inline int p4d_present (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (p4d_val (p4d) & _PAGE_PRESENT);return 1 ;@@ -183,7 +202,7 @@ static inline int p4d_present(p4d_t p4d)static inline int p4d_bad (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return !p4d_present (p4d);return 0 ;@@ -191,7 +210,7 @@ static inline int p4d_bad(p4d_t p4d)static inline void p4d_clear (p4d_t *p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())set_p4d (p4d,__p4d (0 )); }@@ -207,7 +226,7 @@ static inline unsigned long _p4d_pfn(p4d_t p4d)static inline pud_t *p4d_pgtable (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (pud_t *)pfn_to_virt (p4d_val (p4d) >> _PAGE_PFN_SHIFT);return (pud_t *)pud_pgtable ((pud_t ) {p4d_val (p4d) });@@ -224,7 +243,7 @@ static inline struct page *p4d_page(p4d_t p4d)#define pud_offset pud_offsetstatic inline pud_t *pud_offset (p4d_t *p4d,unsigned long address) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return p4d_pgtable (*p4d) +pud_index (address);return (pud_t *)p4d;@@ -232,7 +251,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)static inline void set_pgd (pgd_t *pgdp,pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ()) *pgdp = pgd;else set_p4d ((p4d_t *)pgdp, (p4d_t ){pgd_val (pgd) });@@ -240,7 +259,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)static inline int pgd_none (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (pgd_val (pgd) ==0 );return 0 ;@@ -248,7 +267,7 @@ static inline int pgd_none(pgd_t pgd)static inline int pgd_present (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (pgd_val (pgd) & _PAGE_PRESENT);return 1 ;@@ -256,7 +275,7 @@ static inline int pgd_present(pgd_t pgd)static inline int pgd_bad (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return !pgd_present (pgd);return 0 ;@@ -264,13 +283,13 @@ static inline int pgd_bad(pgd_t pgd)static inline void pgd_clear (pgd_t *pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())set_pgd (pgd,__pgd (0 )); }static inline p4d_t *pgd_pgtable (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (p4d_t *)pfn_to_virt (pgd_val (pgd) >> _PAGE_PFN_SHIFT);return (p4d_t *)p4d_pgtable ((p4d_t ) {pgd_val (pgd) });@@ -288,7 +307,7 @@ static inline struct page *pgd_page(pgd_t pgd)#define p4d_offset p4d_offsetstatic inline p4d_t *p4d_offset (pgd_t *pgd,unsigned long address) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return pgd_pgtable (*pgd) +p4d_index (address);return (p4d_t *)pgd;arch/riscv/include/asm/pgtable.h +2
-3@@ -63,8 +63,8 @@* position vmemmap directly below the VMALLOC region. */ #ifdef CONFIG_64BIT -#define VA_BITS (pgtable_l5_enabled ? \ - 57 : (pgtable_l4_enabled ? 48 : 39)) +#define VA_BITS (pgtable_l5_enabled() ? \ + 57 : (pgtable_l4_enabled() ? 48 : 39))#else #define VA_BITS 32#endif @@ -738,7 +738,6 @@ extern uintptr_t _dtb_early_pa;#define dtb_early_pa _dtb_early_pa#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; -extern bool pgtable_l4_enabled;void paging_init (void );void misc_mem_init (void );arch/riscv/kernel/cpu.c +2
-2@@ -141,9 +141,9 @@ static void print_mmu(struct seq_file *f)#if defined(CONFIG_32BIT )strncpy (sv_type,"sv32" ,5 );#elif defined(CONFIG_64BIT ) -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())strncpy (sv_type,"sv57" ,5 ); -else if (pgtable_l4_enabled) +else if (pgtable_l4_enabled ())strncpy (sv_type,"sv48" ,5 );else strncpy (sv_type,"sv39" ,5 );arch/riscv/mm/init.c +40
-24@@ -44,10 +44,16 @@ u64 satp_mode __ro_after_init = SATP_MODE_32;#endif EXPORT_SYMBOL (satp_mode); -bool pgtable_l4_enabled =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); -bool pgtable_l5_enabled =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); -EXPORT_SYMBOL (pgtable_l4_enabled); -EXPORT_SYMBOL (pgtable_l5_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_l4_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_l5_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_lx_ready); +EXPORT_SYMBOL (_pgtable_l4_enabled); +EXPORT_SYMBOL (_pgtable_l5_enabled); +EXPORT_SYMBOL (_pgtable_lx_ready); +bool _pgtable_l4_enabled_early =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); +bool _pgtable_l5_enabled_early =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); +EXPORT_SYMBOL (_pgtable_l4_enabled_early); +EXPORT_SYMBOL (_pgtable_l5_enabled_early);phys_addr_t phys_ram_base__ro_after_init ;EXPORT_SYMBOL (phys_ram_base);@@ -555,26 +561,26 @@ static void __init create_p4d_mapping(p4d_t *p4dp, }#define pgd_next_t p4d_t -#define alloc_pgd_next (__va) (pgtable_l5_enabled ? \ - pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ +#define alloc_pgd_next (__va) (pgtable_l5_enabled() ? \ + pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled() ? \ pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) -#define get_pgd_next_virt (__pa) (pgtable_l5_enabled ? \ - pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ +#define get_pgd_next_virt (__pa) (pgtable_l5_enabled() ? \ + pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled() ? \ pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa)))#define create_pgd_next_mapping (__nextp, __va, __pa, __sz, __prot) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) -#define fixmap_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ +#define fixmap_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)fixmap_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) -#define trampoline_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ +#define trampoline_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)trampoline_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) -#define early_dtb_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ? \ +#define early_dtb_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd))#else #define pgd_next_t pte_t@@ -680,14 +686,14 @@ static __init pgprot_t pgprot_from_va(uintptr_t va)#ifdef CONFIG_64BIT static void __init disable_pgtable_l5 (void ) { - pgtable_l5_enabled = false; + _pgtable_l5_enabled_early = false; kernel_map.page_offset =PAGE_OFFSET_L4 ; satp_mode =SATP_MODE_48 ; }static void __init disable_pgtable_l4 (void ) { - pgtable_l4_enabled = false; + _pgtable_l4_enabled_early = false; kernel_map.page_offset =PAGE_OFFSET_L3 ; satp_mode =SATP_MODE_39 ; }@@ -816,11 +822,11 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa)PGDIR_SIZE , IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL); -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (early_dtb_p4d,DTB_EARLY_BASE_VA , (uintptr_t )early_dtb_pud,P4D_SIZE ,PAGE_TABLE ); -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (early_dtb_pud,DTB_EARLY_BASE_VA , (uintptr_t )early_dtb_pmd,PUD_SIZE ,PAGE_TABLE );@@ -961,11 +967,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)#ifndef __PAGETABLE_PMD_FOLDED/* Setup fixmap P4D and PUD */ -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (fixmap_p4d,FIXADDR_START , (uintptr_t )fixmap_pud,P4D_SIZE ,PAGE_TABLE );/* Setup fixmap PUD and PMD */ -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (fixmap_pud,FIXADDR_START , (uintptr_t )fixmap_pmd,PUD_SIZE ,PAGE_TABLE ); create_pmd_mapping(fixmap_pmd, FIXADDR_START,@@ -973,10 +979,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)/* Setup trampoline PGD and PMD */ create_pgd_mapping (trampoline_pg_dir, kernel_map.virt_addr, trampoline_pgd_next,PGDIR_SIZE ,PAGE_TABLE ); -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (trampoline_p4d, kernel_map.virt_addr, (uintptr_t )trampoline_pud,P4D_SIZE ,PAGE_TABLE ); -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (trampoline_pud, kernel_map.virt_addr, (uintptr_t )trampoline_pmd,PUD_SIZE ,PAGE_TABLE );#ifdef CONFIG_XIP_KERNEL @@ -1165,8 +1171,18 @@ static void __init reserve_crashkernel(void) crashk_res.end = crash_base + crash_size -1 ; } +static void __init riscv_finalise_pgtable_lx (void ) +{ +if (_pgtable_l5_enabled_early) +static_branch_enable (&_pgtable_l5_enabled); +if (_pgtable_l4_enabled_early) +static_branch_enable (&_pgtable_l4_enabled); +static_branch_enable (&_pgtable_lx_ready); +} +void __init paging_init (void ) { +riscv_finalise_pgtable_lx ();setup_bootmem ();setup_vm_final (); }arch/riscv/mm/kasan_init.c +8
-9@@ -209,15 +209,15 @@ static void __init kasan_populate_p4d(pgd_t *pgd,set_pgd (pgd,pfn_pgd (PFN_DOWN (__pa (base_p4d)),PAGE_TABLE )); } -#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \ +#define kasan_early_shadow_pgd_next (pgtable_l5_enabled() ? \ (uintptr_t)kasan_early_shadow_p4d : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ (uintptr_t)kasan_early_shadow_pud : \ (uintptr_t)kasan_early_shadow_pmd))#define kasan_populate_pgd_next (pgdp, vaddr, next, early) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_populate_p4d(pgdp, vaddr, next, early) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_populate_pud(pgdp, vaddr, next, early) : \ kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))@@ -274,7 +274,7 @@ asmlinkage void __init kasan_early_init(void) (__pa ((uintptr_t )kasan_early_shadow_pte)), PAGE_TABLE)); -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {for (i =0 ; i <PTRS_PER_PUD ; ++i) set_pud(kasan_early_shadow_pud + i, pfn_pud(PFN_DOWN@@ -282,7 +282,7 @@ asmlinkage void __init kasan_early_init(void) PAGE_TABLE)); } -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {for (i =0 ; i <PTRS_PER_P4D ; ++i) set_p4d(kasan_early_shadow_p4d + i, pfn_p4d(PFN_DOWN@@ -393,9 +393,9 @@ static void __init kasan_shallow_populate_p4d(pgd_t *pgdp, }#define kasan_shallow_populate_pgd_next (pgdp, vaddr, next) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_shallow_populate_p4d(pgdp, vaddr, next) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_shallow_populate_pud(pgdp, vaddr, next) : \ kasan_shallow_populate_pmd(pgdp, vaddr, next))) --2.34.1
Re: [PATCH v3 2/2] riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
Quoted 12 lines · Anup Patel
Quoted 11 lines · Jisheng Zhang
On a specific HW platform, pgtable_l4|[l5]_enabled won't change after boot, and the check sits at hot code path, this characteristic makes it suitable for optimization with static key.
_pgtable_l4|[l5]_enabled is used very early during boot, even is used with MMU off, so the static key mechanism isn't ready. For this case, we use another static key
_pgtable_lx_readyto indicate whether we have finalised pgtable_l4|[l5]_enabled or not, then fall back to _pgtable_l4|[l5]_enabled_early bool.
Quoted 6 lines
Quoted 1 line
Signed-off-by Jisheng Zhang <[email protected]>Overall, this patch looks good to me. Please fix the errors reported by autobuilders.
Reviewed-by Anup Patel <[email protected]>
Thank Anup. I sent the v4 two days ago
https://lore.kernel.org/linux-riscv/[email protected]/T/#t
Quoted 525 lines
Regards,
AnupQuoted 521 lines
--- arch/riscv/include/asm/pgalloc.h | 16 ++++---- arch/riscv/include/asm/pgtable-32.h | 3 ++ arch/riscv/include/asm/pgtable-64.h | 59 +++++++++++++++++--------- arch/riscv/include/asm/pgtable.h | 5 +-- arch/riscv/kernel/cpu.c | 4 +- arch/riscv/mm/init.c | 64 ++++++++++++++++++----------- arch/riscv/mm/kasan_init.c | 16 ++++---- 7 files changed, 102 insertions(+), 65 deletions(-)arch/riscv/include/asm/pgalloc.h +8
-8@@ -41,7 +41,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)static inline void p4d_populate (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud);set_p4d (p4d,__p4d ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));@@ -51,7 +51,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)static inline void p4d_populate_safe (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud); set_p4d_safe(p4d,@@ -61,7 +61,7 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d);set_pgd (pgd,__pgd ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));@@ -71,7 +71,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)static inline void pgd_populate_safe (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d); set_pgd_safe(pgd,@@ -82,7 +82,7 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,#define pud_alloc_one pud_alloc_onestatic inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return __pud_alloc_one (mm, addr);return NULL ;@@ -91,7 +91,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)#define pud_free pud_freestatic inline void pud_free (struct mm_struct *mm,pud_t *pud) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())__pud_free (mm, pud); }@@ -100,7 +100,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)#define p4d_alloc_one p4d_alloc_onestatic inline p4d_t *p4d_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {gfp_t gfp =GFP_PGTABLE_USER ;if (mm == &init_mm)@@ -120,7 +120,7 @@ static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)#define p4d_free p4d_freestatic inline void p4d_free (struct mm_struct *mm,p4d_t *p4d) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())__p4d_free (mm, p4d); }arch/riscv/include/asm/pgtable-32.h +3
-0@@ -16,4 +16,7 @@#define MAX_POSSIBLE_PHYSMEM_BITS 34 +#define pgtable_l5_enabled () 0 +#define pgtable_l4_enabled () 0 +#endif /* _ASM_RISCV_PGTABLE_32_H */ arch/riscv/include/asm/pgtable-64.h +39
-20@@ -8,16 +8,35 @@#include <linux/const.h> -extern bool pgtable_l4_enabled; -extern bool pgtable_l5_enabled; +extern bool _pgtable_l5_enabled_early; +extern bool _pgtable_l4_enabled_early; +extern struct static_key_false _pgtable_l5_enabled; +extern struct static_key_false _pgtable_l4_enabled; +extern struct static_key_false _pgtable_lx_ready; + +static __always_inline bool pgtable_l5_enabled(void) +{ +if (static_branch_likely (&_pgtable_lx_ready)) +return static_branch_likely (&_pgtable_l5_enabled); +else +return _pgtable_l5_enabled_early; +} + +static __always_inline bool pgtable_l4_enabled(void) +{ +if (static_branch_likely (&_pgtable_lx_ready)) +return static_branch_likely (&_pgtable_l4_enabled); +else +return _pgtable_l4_enabled_early; +}#define PGDIR_SHIFT_L3 30#define PGDIR_SHIFT_L4 39#define PGDIR_SHIFT_L5 48#define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) -#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ - (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) +#define PGDIR_SHIFT (pgtable_l5_enabled() ? PGDIR_SHIFT_L5 : \ + (pgtable_l4_enabled() ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))/* Size of region mapped by a page global directory */ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)#define PGDIR_MASK (~(PGDIR_SIZE - 1))@@ -119,7 +138,7 @@ static inline struct page *pud_page(pud_t pud)#define mm_p4d_folded mm_p4d_foldedstatic inline bool mm_p4d_folded (struct mm_struct *mm) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return false;return true;@@ -128,7 +147,7 @@ static inline bool mm_p4d_folded(struct mm_struct *mm)#define mm_pud_folded mm_pud_foldedstatic inline bool mm_pud_folded (struct mm_struct *mm) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return false;return true;@@ -159,7 +178,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd)static inline void set_p4d (p4d_t *p4dp,p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ()) *p4dp = p4d;else set_pud ((pud_t *)p4dp, (pud_t ){p4d_val (p4d) });@@ -167,7 +186,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)static inline int p4d_none (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (p4d_val (p4d) ==0 );return 0 ;@@ -175,7 +194,7 @@ static inline int p4d_none(p4d_t p4d)static inline int p4d_present (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (p4d_val (p4d) & _PAGE_PRESENT);return 1 ;@@ -183,7 +202,7 @@ static inline int p4d_present(p4d_t p4d)static inline int p4d_bad (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return !p4d_present (p4d);return 0 ;@@ -191,7 +210,7 @@ static inline int p4d_bad(p4d_t p4d)static inline void p4d_clear (p4d_t *p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())set_p4d (p4d,__p4d (0 )); }@@ -207,7 +226,7 @@ static inline unsigned long _p4d_pfn(p4d_t p4d)static inline pud_t *p4d_pgtable (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (pud_t *)pfn_to_virt (p4d_val (p4d) >> _PAGE_PFN_SHIFT);return (pud_t *)pud_pgtable ((pud_t ) {p4d_val (p4d) });@@ -224,7 +243,7 @@ static inline struct page *p4d_page(p4d_t p4d)#define pud_offset pud_offsetstatic inline pud_t *pud_offset (p4d_t *p4d,unsigned long address) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return p4d_pgtable (*p4d) +pud_index (address);return (pud_t *)p4d;@@ -232,7 +251,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)static inline void set_pgd (pgd_t *pgdp,pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ()) *pgdp = pgd;else set_p4d ((p4d_t *)pgdp, (p4d_t ){pgd_val (pgd) });@@ -240,7 +259,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)static inline int pgd_none (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (pgd_val (pgd) ==0 );return 0 ;@@ -248,7 +267,7 @@ static inline int pgd_none(pgd_t pgd)static inline int pgd_present (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (pgd_val (pgd) & _PAGE_PRESENT);return 1 ;@@ -256,7 +275,7 @@ static inline int pgd_present(pgd_t pgd)static inline int pgd_bad (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return !pgd_present (pgd);return 0 ;@@ -264,13 +283,13 @@ static inline int pgd_bad(pgd_t pgd)static inline void pgd_clear (pgd_t *pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())set_pgd (pgd,__pgd (0 )); }static inline p4d_t *pgd_pgtable (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (p4d_t *)pfn_to_virt (pgd_val (pgd) >> _PAGE_PFN_SHIFT);return (p4d_t *)p4d_pgtable ((p4d_t ) {pgd_val (pgd) });@@ -288,7 +307,7 @@ static inline struct page *pgd_page(pgd_t pgd)#define p4d_offset p4d_offsetstatic inline p4d_t *p4d_offset (pgd_t *pgd,unsigned long address) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return pgd_pgtable (*pgd) +p4d_index (address);return (p4d_t *)pgd;arch/riscv/include/asm/pgtable.h +2
-3@@ -63,8 +63,8 @@* position vmemmap directly below the VMALLOC region. */ #ifdef CONFIG_64BIT -#define VA_BITS (pgtable_l5_enabled ? \ - 57 : (pgtable_l4_enabled ? 48 : 39)) +#define VA_BITS (pgtable_l5_enabled() ? \ + 57 : (pgtable_l4_enabled() ? 48 : 39))#else #define VA_BITS 32#endif @@ -738,7 +738,6 @@ extern uintptr_t _dtb_early_pa;#define dtb_early_pa _dtb_early_pa#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; -extern bool pgtable_l4_enabled;void paging_init (void );void misc_mem_init (void );arch/riscv/kernel/cpu.c +2
-2@@ -141,9 +141,9 @@ static void print_mmu(struct seq_file *f)#if defined(CONFIG_32BIT )strncpy (sv_type,"sv32" ,5 );#elif defined(CONFIG_64BIT ) -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())strncpy (sv_type,"sv57" ,5 ); -else if (pgtable_l4_enabled) +else if (pgtable_l4_enabled ())strncpy (sv_type,"sv48" ,5 );else strncpy (sv_type,"sv39" ,5 );arch/riscv/mm/init.c +40
-24@@ -44,10 +44,16 @@ u64 satp_mode __ro_after_init = SATP_MODE_32;#endif EXPORT_SYMBOL (satp_mode); -bool pgtable_l4_enabled =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); -bool pgtable_l5_enabled =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); -EXPORT_SYMBOL (pgtable_l4_enabled); -EXPORT_SYMBOL (pgtable_l5_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_l4_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_l5_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_lx_ready); +EXPORT_SYMBOL (_pgtable_l4_enabled); +EXPORT_SYMBOL (_pgtable_l5_enabled); +EXPORT_SYMBOL (_pgtable_lx_ready); +bool _pgtable_l4_enabled_early =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); +bool _pgtable_l5_enabled_early =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); +EXPORT_SYMBOL (_pgtable_l4_enabled_early); +EXPORT_SYMBOL (_pgtable_l5_enabled_early);phys_addr_t phys_ram_base__ro_after_init ;EXPORT_SYMBOL (phys_ram_base);@@ -555,26 +561,26 @@ static void __init create_p4d_mapping(p4d_t *p4dp, }#define pgd_next_t p4d_t -#define alloc_pgd_next (__va) (pgtable_l5_enabled ? \ - pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ +#define alloc_pgd_next (__va) (pgtable_l5_enabled() ? \ + pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled() ? \ pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) -#define get_pgd_next_virt (__pa) (pgtable_l5_enabled ? \ - pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ +#define get_pgd_next_virt (__pa) (pgtable_l5_enabled() ? \ + pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled() ? \ pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa)))#define create_pgd_next_mapping (__nextp, __va, __pa, __sz, __prot) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) -#define fixmap_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ +#define fixmap_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)fixmap_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) -#define trampoline_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ +#define trampoline_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)trampoline_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) -#define early_dtb_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ? \ +#define early_dtb_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd))#else #define pgd_next_t pte_t@@ -680,14 +686,14 @@ static __init pgprot_t pgprot_from_va(uintptr_t va)#ifdef CONFIG_64BIT static void __init disable_pgtable_l5 (void ) { - pgtable_l5_enabled = false; + _pgtable_l5_enabled_early = false; kernel_map.page_offset =PAGE_OFFSET_L4 ; satp_mode =SATP_MODE_48 ; }static void __init disable_pgtable_l4 (void ) { - pgtable_l4_enabled = false; + _pgtable_l4_enabled_early = false; kernel_map.page_offset =PAGE_OFFSET_L3 ; satp_mode =SATP_MODE_39 ; }@@ -816,11 +822,11 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa)PGDIR_SIZE , IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL); -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (early_dtb_p4d,DTB_EARLY_BASE_VA , (uintptr_t )early_dtb_pud,P4D_SIZE ,PAGE_TABLE ); -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (early_dtb_pud,DTB_EARLY_BASE_VA , (uintptr_t )early_dtb_pmd,PUD_SIZE ,PAGE_TABLE );@@ -961,11 +967,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)#ifndef __PAGETABLE_PMD_FOLDED/* Setup fixmap P4D and PUD */ -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (fixmap_p4d,FIXADDR_START , (uintptr_t )fixmap_pud,P4D_SIZE ,PAGE_TABLE );/* Setup fixmap PUD and PMD */ -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (fixmap_pud,FIXADDR_START , (uintptr_t )fixmap_pmd,PUD_SIZE ,PAGE_TABLE ); create_pmd_mapping(fixmap_pmd, FIXADDR_START,@@ -973,10 +979,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)/* Setup trampoline PGD and PMD */ create_pgd_mapping (trampoline_pg_dir, kernel_map.virt_addr, trampoline_pgd_next,PGDIR_SIZE ,PAGE_TABLE ); -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (trampoline_p4d, kernel_map.virt_addr, (uintptr_t )trampoline_pud,P4D_SIZE ,PAGE_TABLE ); -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (trampoline_pud, kernel_map.virt_addr, (uintptr_t )trampoline_pmd,PUD_SIZE ,PAGE_TABLE );#ifdef CONFIG_XIP_KERNEL @@ -1165,8 +1171,18 @@ static void __init reserve_crashkernel(void) crashk_res.end = crash_base + crash_size -1 ; } +static void __init riscv_finalise_pgtable_lx (void ) +{ +if (_pgtable_l5_enabled_early) +static_branch_enable (&_pgtable_l5_enabled); +if (_pgtable_l4_enabled_early) +static_branch_enable (&_pgtable_l4_enabled); +static_branch_enable (&_pgtable_lx_ready); +} +void __init paging_init (void ) { +riscv_finalise_pgtable_lx ();setup_bootmem ();setup_vm_final (); }arch/riscv/mm/kasan_init.c +8
-9@@ -209,15 +209,15 @@ static void __init kasan_populate_p4d(pgd_t *pgd,set_pgd (pgd,pfn_pgd (PFN_DOWN (__pa (base_p4d)),PAGE_TABLE )); } -#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \ +#define kasan_early_shadow_pgd_next (pgtable_l5_enabled() ? \ (uintptr_t)kasan_early_shadow_p4d : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ (uintptr_t)kasan_early_shadow_pud : \ (uintptr_t)kasan_early_shadow_pmd))#define kasan_populate_pgd_next (pgdp, vaddr, next, early) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_populate_p4d(pgdp, vaddr, next, early) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_populate_pud(pgdp, vaddr, next, early) : \ kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))@@ -274,7 +274,7 @@ asmlinkage void __init kasan_early_init(void) (__pa ((uintptr_t )kasan_early_shadow_pte)), PAGE_TABLE)); -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {for (i =0 ; i <PTRS_PER_PUD ; ++i) set_pud(kasan_early_shadow_pud + i, pfn_pud(PFN_DOWN@@ -282,7 +282,7 @@ asmlinkage void __init kasan_early_init(void) PAGE_TABLE)); } -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {for (i =0 ; i <PTRS_PER_P4D ; ++i) set_p4d(kasan_early_shadow_p4d + i, pfn_p4d(PFN_DOWN@@ -393,9 +393,9 @@ static void __init kasan_shallow_populate_p4d(pgd_t *pgdp, }#define kasan_shallow_populate_pgd_next (pgdp, vaddr, next) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_shallow_populate_p4d(pgdp, vaddr, next) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_shallow_populate_pud(pgdp, vaddr, next) : \ kasan_shallow_populate_pmd(pgdp, vaddr, next))) --2.34.1
Re: [PATCH v3 2/2] riscv: turn pgtable_l4|[l5]_enabled to static key for RV64
Quoted 16 lines · Jisheng Zhang
Quoted 14 lines · Anup Patel
Quoted 12 lines · Jisheng Zhang
On a specific HW platform, pgtable_l4|[l5]_enabled won't change after boot, and the check sits at hot code path, this characteristic makes it suitable for optimization with static key.
_pgtable_l4|[l5]_enabled is used very early during boot, even is used with MMU off, so the static key mechanism isn't ready. For this case, we use another static key
_pgtable_lx_readyto indicate whether we have finalised pgtable_l4|[l5]_enabled or not, then fall back to _pgtable_l4|[l5]_enabled_early bool.Signed-off-by Jisheng Zhang <[email protected]>
Quoted 7 lines
Quoted 4 lines
Overall, this patch looks good to me. Please fix the errors reported by autobuilders.
Reviewed-by Anup Patel <[email protected]>Thank Anup. I sent the v4 two days ago
https://lore.kernel.org/linux-riscv/[email protected]/T/#t
Sorry, I missed your v4.
I have provided Reviewed-by to your v4 as well.
Regards,
Anup
Quoted 523 lines
Quoted 523 lines
Regards,
AnupQuoted 518 lines
--- arch/riscv/include/asm/pgalloc.h | 16 ++++---- arch/riscv/include/asm/pgtable-32.h | 3 ++ arch/riscv/include/asm/pgtable-64.h | 59 +++++++++++++++++--------- arch/riscv/include/asm/pgtable.h | 5 +-- arch/riscv/kernel/cpu.c | 4 +- arch/riscv/mm/init.c | 64 ++++++++++++++++++----------- arch/riscv/mm/kasan_init.c | 16 ++++---- 7 files changed, 102 insertions(+), 65 deletions(-)arch/riscv/include/asm/pgalloc.h +8
-8@@ -41,7 +41,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)static inline void p4d_populate (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud);set_p4d (p4d,__p4d ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));@@ -51,7 +51,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)static inline void p4d_populate_safe (struct mm_struct *mm,p4d_t *p4d,pud_t *pud) { -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {unsigned long pfn =virt_to_pfn (pud); set_p4d_safe(p4d,@@ -61,7 +61,7 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d,static inline void pgd_populate (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d);set_pgd (pgd,__pgd ((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE));@@ -71,7 +71,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)static inline void pgd_populate_safe (struct mm_struct *mm,pgd_t *pgd,p4d_t *p4d) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {unsigned long pfn =virt_to_pfn (p4d); set_pgd_safe(pgd,@@ -82,7 +82,7 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd,#define pud_alloc_one pud_alloc_onestatic inline pud_t *pud_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return __pud_alloc_one (mm, addr);return NULL ;@@ -91,7 +91,7 @@ static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)#define pud_free pud_freestatic inline void pud_free (struct mm_struct *mm,pud_t *pud) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())__pud_free (mm, pud); }@@ -100,7 +100,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)#define p4d_alloc_one p4d_alloc_onestatic inline p4d_t *p4d_alloc_one (struct mm_struct *mm,unsigned long addr) { -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {gfp_t gfp =GFP_PGTABLE_USER ;if (mm == &init_mm)@@ -120,7 +120,7 @@ static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d)#define p4d_free p4d_freestatic inline void p4d_free (struct mm_struct *mm,p4d_t *p4d) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())__p4d_free (mm, p4d); }arch/riscv/include/asm/pgtable-32.h +3
-0@@ -16,4 +16,7 @@#define MAX_POSSIBLE_PHYSMEM_BITS 34 +#define pgtable_l5_enabled () 0 +#define pgtable_l4_enabled () 0 +#endif /* _ASM_RISCV_PGTABLE_32_H */ arch/riscv/include/asm/pgtable-64.h +39
-20@@ -8,16 +8,35 @@#include <linux/const.h> -extern bool pgtable_l4_enabled; -extern bool pgtable_l5_enabled; +extern bool _pgtable_l5_enabled_early; +extern bool _pgtable_l4_enabled_early; +extern struct static_key_false _pgtable_l5_enabled; +extern struct static_key_false _pgtable_l4_enabled; +extern struct static_key_false _pgtable_lx_ready; + +static __always_inline bool pgtable_l5_enabled(void) +{ +if (static_branch_likely (&_pgtable_lx_ready)) +return static_branch_likely (&_pgtable_l5_enabled); +else +return _pgtable_l5_enabled_early; +} + +static __always_inline bool pgtable_l4_enabled(void) +{ +if (static_branch_likely (&_pgtable_lx_ready)) +return static_branch_likely (&_pgtable_l4_enabled); +else +return _pgtable_l4_enabled_early; +}#define PGDIR_SHIFT_L3 30#define PGDIR_SHIFT_L4 39#define PGDIR_SHIFT_L5 48#define PGDIR_SIZE_L3 (_AC(1, UL) << PGDIR_SHIFT_L3) -#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ - (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) +#define PGDIR_SHIFT (pgtable_l5_enabled() ? PGDIR_SHIFT_L5 : \ + (pgtable_l4_enabled() ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3))/* Size of region mapped by a page global directory */ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)#define PGDIR_MASK (~(PGDIR_SIZE - 1))@@ -119,7 +138,7 @@ static inline struct page *pud_page(pud_t pud)#define mm_p4d_folded mm_p4d_foldedstatic inline bool mm_p4d_folded (struct mm_struct *mm) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return false;return true;@@ -128,7 +147,7 @@ static inline bool mm_p4d_folded(struct mm_struct *mm)#define mm_pud_folded mm_pud_foldedstatic inline bool mm_pud_folded (struct mm_struct *mm) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return false;return true;@@ -159,7 +178,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd)static inline void set_p4d (p4d_t *p4dp,p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ()) *p4dp = p4d;else set_pud ((pud_t *)p4dp, (pud_t ){p4d_val (p4d) });@@ -167,7 +186,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)static inline int p4d_none (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (p4d_val (p4d) ==0 );return 0 ;@@ -175,7 +194,7 @@ static inline int p4d_none(p4d_t p4d)static inline int p4d_present (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (p4d_val (p4d) & _PAGE_PRESENT);return 1 ;@@ -183,7 +202,7 @@ static inline int p4d_present(p4d_t p4d)static inline int p4d_bad (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return !p4d_present (p4d);return 0 ;@@ -191,7 +210,7 @@ static inline int p4d_bad(p4d_t p4d)static inline void p4d_clear (p4d_t *p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())set_p4d (p4d,__p4d (0 )); }@@ -207,7 +226,7 @@ static inline unsigned long _p4d_pfn(p4d_t p4d)static inline pud_t *p4d_pgtable (p4d_t p4d) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return (pud_t *)pfn_to_virt (p4d_val (p4d) >> _PAGE_PFN_SHIFT);return (pud_t *)pud_pgtable ((pud_t ) {p4d_val (p4d) });@@ -224,7 +243,7 @@ static inline struct page *p4d_page(p4d_t p4d)#define pud_offset pud_offsetstatic inline pud_t *pud_offset (p4d_t *p4d,unsigned long address) { -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())return p4d_pgtable (*p4d) +pud_index (address);return (pud_t *)p4d;@@ -232,7 +251,7 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)static inline void set_pgd (pgd_t *pgdp,pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ()) *pgdp = pgd;else set_p4d ((p4d_t *)pgdp, (p4d_t ){pgd_val (pgd) });@@ -240,7 +259,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)static inline int pgd_none (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (pgd_val (pgd) ==0 );return 0 ;@@ -248,7 +267,7 @@ static inline int pgd_none(pgd_t pgd)static inline int pgd_present (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (pgd_val (pgd) & _PAGE_PRESENT);return 1 ;@@ -256,7 +275,7 @@ static inline int pgd_present(pgd_t pgd)static inline int pgd_bad (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return !pgd_present (pgd);return 0 ;@@ -264,13 +283,13 @@ static inline int pgd_bad(pgd_t pgd)static inline void pgd_clear (pgd_t *pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())set_pgd (pgd,__pgd (0 )); }static inline p4d_t *pgd_pgtable (pgd_t pgd) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return (p4d_t *)pfn_to_virt (pgd_val (pgd) >> _PAGE_PFN_SHIFT);return (p4d_t *)p4d_pgtable ((p4d_t ) {pgd_val (pgd) });@@ -288,7 +307,7 @@ static inline struct page *pgd_page(pgd_t pgd)#define p4d_offset p4d_offsetstatic inline p4d_t *p4d_offset (pgd_t *pgd,unsigned long address) { -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())return pgd_pgtable (*pgd) +p4d_index (address);return (p4d_t *)pgd;arch/riscv/include/asm/pgtable.h +2
-3@@ -63,8 +63,8 @@* position vmemmap directly below the VMALLOC region. */ #ifdef CONFIG_64BIT -#define VA_BITS (pgtable_l5_enabled ? \ - 57 : (pgtable_l4_enabled ? 48 : 39)) +#define VA_BITS (pgtable_l5_enabled() ? \ + 57 : (pgtable_l4_enabled() ? 48 : 39))#else #define VA_BITS 32#endif @@ -738,7 +738,6 @@ extern uintptr_t _dtb_early_pa;#define dtb_early_pa _dtb_early_pa#endif /* CONFIG_XIP_KERNEL */ extern u64 satp_mode; -extern bool pgtable_l4_enabled;void paging_init (void );void misc_mem_init (void );arch/riscv/kernel/cpu.c +2
-2@@ -141,9 +141,9 @@ static void print_mmu(struct seq_file *f)#if defined(CONFIG_32BIT )strncpy (sv_type,"sv32" ,5 );#elif defined(CONFIG_64BIT ) -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())strncpy (sv_type,"sv57" ,5 ); -else if (pgtable_l4_enabled) +else if (pgtable_l4_enabled ())strncpy (sv_type,"sv48" ,5 );else strncpy (sv_type,"sv39" ,5 );arch/riscv/mm/init.c +40
-24@@ -44,10 +44,16 @@ u64 satp_mode __ro_after_init = SATP_MODE_32;#endif EXPORT_SYMBOL (satp_mode); -bool pgtable_l4_enabled =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); -bool pgtable_l5_enabled =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); -EXPORT_SYMBOL (pgtable_l4_enabled); -EXPORT_SYMBOL (pgtable_l5_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_l4_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_l5_enabled); +DEFINE_STATIC_KEY_FALSE (_pgtable_lx_ready); +EXPORT_SYMBOL (_pgtable_l4_enabled); +EXPORT_SYMBOL (_pgtable_l5_enabled); +EXPORT_SYMBOL (_pgtable_lx_ready); +bool _pgtable_l4_enabled_early =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); +bool _pgtable_l5_enabled_early =IS_ENABLED (CONFIG_64BIT ) && !IS_ENABLED (CONFIG_XIP_KERNEL ); +EXPORT_SYMBOL (_pgtable_l4_enabled_early); +EXPORT_SYMBOL (_pgtable_l5_enabled_early);phys_addr_t phys_ram_base__ro_after_init ;EXPORT_SYMBOL (phys_ram_base);@@ -555,26 +561,26 @@ static void __init create_p4d_mapping(p4d_t *p4dp, }#define pgd_next_t p4d_t -#define alloc_pgd_next (__va) (pgtable_l5_enabled ? \ - pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ +#define alloc_pgd_next (__va) (pgtable_l5_enabled() ? \ + pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled() ? \ pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) -#define get_pgd_next_virt (__pa) (pgtable_l5_enabled ? \ - pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ +#define get_pgd_next_virt (__pa) (pgtable_l5_enabled() ? \ + pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled() ? \ pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa)))#define create_pgd_next_mapping (__nextp, __va, __pa, __sz, __prot) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) -#define fixmap_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ +#define fixmap_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)fixmap_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) -#define trampoline_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ +#define trampoline_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)trampoline_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) -#define early_dtb_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ? \ +#define early_dtb_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)early_dtb_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd))#else #define pgd_next_t pte_t@@ -680,14 +686,14 @@ static __init pgprot_t pgprot_from_va(uintptr_t va)#ifdef CONFIG_64BIT static void __init disable_pgtable_l5 (void ) { - pgtable_l5_enabled = false; + _pgtable_l5_enabled_early = false; kernel_map.page_offset =PAGE_OFFSET_L4 ; satp_mode =SATP_MODE_48 ; }static void __init disable_pgtable_l4 (void ) { - pgtable_l4_enabled = false; + _pgtable_l4_enabled_early = false; kernel_map.page_offset =PAGE_OFFSET_L3 ; satp_mode =SATP_MODE_39 ; }@@ -816,11 +822,11 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa)PGDIR_SIZE , IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL); -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (early_dtb_p4d,DTB_EARLY_BASE_VA , (uintptr_t )early_dtb_pud,P4D_SIZE ,PAGE_TABLE ); -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (early_dtb_pud,DTB_EARLY_BASE_VA , (uintptr_t )early_dtb_pmd,PUD_SIZE ,PAGE_TABLE );@@ -961,11 +967,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)#ifndef __PAGETABLE_PMD_FOLDED/* Setup fixmap P4D and PUD */ -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (fixmap_p4d,FIXADDR_START , (uintptr_t )fixmap_pud,P4D_SIZE ,PAGE_TABLE );/* Setup fixmap PUD and PMD */ -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (fixmap_pud,FIXADDR_START , (uintptr_t )fixmap_pmd,PUD_SIZE ,PAGE_TABLE ); create_pmd_mapping(fixmap_pmd, FIXADDR_START,@@ -973,10 +979,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)/* Setup trampoline PGD and PMD */ create_pgd_mapping (trampoline_pg_dir, kernel_map.virt_addr, trampoline_pgd_next,PGDIR_SIZE ,PAGE_TABLE ); -if (pgtable_l5_enabled) +if (pgtable_l5_enabled ())create_p4d_mapping (trampoline_p4d, kernel_map.virt_addr, (uintptr_t )trampoline_pud,P4D_SIZE ,PAGE_TABLE ); -if (pgtable_l4_enabled) +if (pgtable_l4_enabled ())create_pud_mapping (trampoline_pud, kernel_map.virt_addr, (uintptr_t )trampoline_pmd,PUD_SIZE ,PAGE_TABLE );#ifdef CONFIG_XIP_KERNEL @@ -1165,8 +1171,18 @@ static void __init reserve_crashkernel(void) crashk_res.end = crash_base + crash_size -1 ; } +static void __init riscv_finalise_pgtable_lx (void ) +{ +if (_pgtable_l5_enabled_early) +static_branch_enable (&_pgtable_l5_enabled); +if (_pgtable_l4_enabled_early) +static_branch_enable (&_pgtable_l4_enabled); +static_branch_enable (&_pgtable_lx_ready); +} +void __init paging_init (void ) { +riscv_finalise_pgtable_lx ();setup_bootmem ();setup_vm_final (); }arch/riscv/mm/kasan_init.c +8
-8@@ -209,15 +209,15 @@ static void __init kasan_populate_p4d(pgd_t *pgd,set_pgd (pgd,pfn_pgd (PFN_DOWN (__pa (base_p4d)),PAGE_TABLE )); } -#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \ +#define kasan_early_shadow_pgd_next (pgtable_l5_enabled() ? \ (uintptr_t)kasan_early_shadow_p4d : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ (uintptr_t)kasan_early_shadow_pud : \ (uintptr_t)kasan_early_shadow_pmd))#define kasan_populate_pgd_next (pgdp, vaddr, next, early) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_populate_p4d(pgdp, vaddr, next, early) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_populate_pud(pgdp, vaddr, next, early) : \ kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))@@ -274,7 +274,7 @@ asmlinkage void __init kasan_early_init(void) (__pa ((uintptr_t )kasan_early_shadow_pte)), PAGE_TABLE)); -if (pgtable_l4_enabled) { +if (pgtable_l4_enabled ()) {for (i =0 ; i <PTRS_PER_PUD ; ++i) set_pud(kasan_early_shadow_pud + i, pfn_pud(PFN_DOWN@@ -282,7 +282,7 @@ asmlinkage void __init kasan_early_init(void) PAGE_TABLE)); } -if (pgtable_l5_enabled) { +if (pgtable_l5_enabled ()) {for (i =0 ; i <PTRS_PER_P4D ; ++i) set_p4d(kasan_early_shadow_p4d + i, pfn_p4d(PFN_DOWN@@ -393,9 +393,9 @@ static void __init kasan_shallow_populate_p4d(pgd_t *pgdp, }#define kasan_shallow_populate_pgd_next (pgdp, vaddr, next) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ kasan_shallow_populate_p4d(pgdp, vaddr, next) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ kasan_shallow_populate_pud(pgdp, vaddr, next) : \ kasan_shallow_populate_pmd(pgdp, vaddr, next)))
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
--
2.34.1
_______________________________________________
linux-riscv mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-riscv