Skip to contentSkip to conversation

arm64: tlb: add support for TTL field

RFC PATCH Superseded Zhenyu Ye 2 participants 7 messages Mar 21, 2020 – Mar 24, 2020

Subsystems (7)

v3 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Zhenyu Ye

author
#0 lore raw

v3 cover arm64: tlb: add support for TTL field

--
ChangeList:
v3:
use vma->vm_flags to replace mm->context.flags.

v2:
build the patch on Marc's NV series[1].

v1:
add support for TTL field in arm64.

--
ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate the level of translation table walk holding the leaf entry for the address that is being invalidated. Hardware can use this information to determine if there was a risk of splintering.

Marc has provided basic support for ARM64-TTL features on his NV series[1] patches. NV is a large feature, however, only patches 62[2] and 67[3] are need by this patch set.
** You only need read those two patches before review this patch. **

Some of this patch depends on a feature powered by @Will Deacon two years ago, which tracking the level of page tables in mm_gather. See more in commit a6d60245.

[1] git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git kvm-arm64/nv-5.6-rc1
[2] https://lore.kernel.org/linux-arm-kernel/[email protected]/
[3] https://lore.kernel.org/linux-arm-kernel/[email protected]/

Zhenyu Ye (4):
arm64: Add level-hinted TLB invalidation helper to tlbi_user
mm: Add page table level flags to vm_flags
arm64: tlb: Use translation level hint in vm_flags
mm: Set VM_LEVEL flags in some tlb_flush functions

 arch/arm64/include/asm/mmu.h      |  2 ++
 arch/arm64/include/asm/tlb.h      | 12 +++++++++
 arch/arm64/include/asm/tlbflush.h | 44 ++++++++++++++++++++++++++-----
 arch/arm64/mm/hugetlbpage.c       |  4 +--
 arch/arm64/mm/mmu.c               | 14 ++++++++++
 include/asm-generic/pgtable.h     | 16 +++++++++--
 include/linux/mm.h                | 10 +++++++
 include/trace/events/mmflags.h    | 15 ++++++++++-
 mm/huge_memory.c                  |  8 +++++-
 9 files changed, 113 insertions(+), 12 deletions(-)
Signature · 1 line
2.19.1
author
#1 lore raw

RFC PATCH v3 1/4 arm64: Add level-hinted TLB invalidation helper to tlbi_user

Add a level-hinted parameter to __tlbi_user, which only gets used if ARMv8.4-TTL gets detected.

ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate the level of translation table walk holding the leaf entry for the address that is being invalidated.

This patch set the default level value to 0.

---
 arch/arm64/include/asm/tlbflush.h | 42 ++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 6 deletions(-)
arch/arm64/include/asm/tlbflush.h +36 -6
index a3f70778a325..d141c080e494 100644
@@ -89,6 +89,36 @@
 		__tlbi(op,  arg);					\
 	} while(0)
 
+#define __tlbi_user_level(op, addr, level)				\
+	do {								\
+		u64 arg = addr;						\
+									\
+		if (!arm64_kernel_unmapped_at_el0())			\
+			break;						\
+									\
+		if (cpus_have_const_cap(ARM64_HAS_ARMv8_4_TTL) &&	\
+		    level) {						\
+			u64 ttl = level;				\
+									\
+			switch (PAGE_SIZE) {				\
+			case SZ_4K:					\
+				ttl |= 1 << 2;				\
+				break;					\
+			case SZ_16K:					\
+				ttl |= 2 << 2;				\
+				break;					\
+			case SZ_64K:					\
+				ttl |= 3 << 2;				\
+				break;					\
+			}						\
+									\
+			arg &= ~TLBI_TTL_MASK;				\
+			arg |= FIELD_PREP(TLBI_TTL_MASK, ttl);		\
+		}							\
+									\
+		__tlbi(op,  (arg) | USER_ASID_FLAG);			\
+	} while (0)
+
 /*
  *	TLB Invalidation
  *	================
@@ -190,8 +220,8 @@ static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
 	unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm));
 
 	dsb(ishst);
-	__tlbi(vale1is, addr);
-	__tlbi_user(vale1is, addr);
+	__tlbi_level(vale1is, addr, 0);
+	__tlbi_user_level(vale1is, addr, 0);
 }
 
 static inline void flush_tlb_page(struct vm_area_struct *vma,
@@ -231,11 +261,11 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 	dsb(ishst);
 	for (addr = start; addr < end; addr += stride) {
 		if (last_level) {
-			__tlbi(vale1is, addr);
-			__tlbi_user(vale1is, addr);
+			__tlbi_level(vale1is, addr, 0);
+			__tlbi_user_level(vale1is, addr, 0);
 		} else {
-			__tlbi(vae1is, addr);
-			__tlbi_user(vae1is, addr);
+			__tlbi_level(vae1is, addr, 0);
+			__tlbi_user_level(vae1is, addr, 0);
 		}
 	}
 	dsb(ish);
Signature · 1 line
2.19.1
author
#2 lore raw

RFC PATCH v3 2/4 mm: Add page table level flags to vm_flags

In reply to Zhenyu Ye

Add VM_LEVEL_[PUD|PMD|PTE] to vm_flags to indicate which level of the page tables the vma is in. Those flags can be used to reduce the cost of TLB invalidation.

These should be common flags for all architectures, however, those flags are only available in 64-bits system currently, because the lower-order flags are fully used.

These flags are only used by ARM64 architecture now. See in next patch.

---
 include/linux/mm.h             | 10 ++++++++++
 include/trace/events/mmflags.h | 15 ++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)
include/linux/mm.h +10 -0
index 52269e56c514..63a04e02a221 100644
@@ -313,6 +313,16 @@ extern unsigned int kobjsize(const void *objp);
 #endif
 #endif /* CONFIG_ARCH_HAS_PKEYS */
 
+#ifdef CONFIG_64BIT
+# define VM_LEVEL_PUD	BIT(37)		/* vma is in pud-level of page table */
+# define VM_LEVEL_PMD	BIT(38)		/* vma is in pmd-level of page table */
+# define VM_LEVEL_PTE	BIT(39)		/* vma is in pte-level of page table */
+#else
+# define VM_LEVEL_PUD	0
+# define VM_LEVEL_PMD	0
+# define VM_LEVEL_PTE	0
+#endif /* CONFIG_64BIT */
+
 #if defined(CONFIG_X86)
 # define VM_PAT		VM_ARCH_1	/* PAT reserves whole VMA at once (x86) */
 #elif defined(CONFIG_PPC)
include/trace/events/mmflags.h +14 -1
index a1675d43777e..9f13cfa96f9f 100644
@@ -130,6 +130,16 @@ IF_HAVE_PG_IDLE(PG_idle,		"idle"		)
 #define IF_HAVE_VM_SOFTDIRTY(flag,name)
 #endif
 
+#ifdef CONFIG_64BIT
+#define IF_HAVE_VM_LEVEL_PUD(flag,name)	{flag, name}
+#define IF_HAVE_VM_LEVEL_PMD(flag,name)	{flag, name}
+#define IF_HAVE_VM_LEVEL_PTE(flag,name)	{flag, name}
+#else
+#define IF_HAVE_VM_LEVEL_PUD(flag,name)
+#define IF_HAVE_VM_LEVEL_PMD(flag,name)
+#define IF_HAVE_VM_LEVEL_PTE(flag,name)
+#endif
+
 #define __def_vmaflag_names						\
 	{VM_READ,			"read"		},		\
 	{VM_WRITE,			"write"		},		\
@@ -161,7 +171,10 @@ IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY,	"softdirty"	)		\
 	{VM_MIXEDMAP,			"mixedmap"	},		\
 	{VM_HUGEPAGE,			"hugepage"	},		\
 	{VM_NOHUGEPAGE,			"nohugepage"	},		\
-	{VM_MERGEABLE,			"mergeable"	}		\
+	{VM_MERGEABLE,			"mergeable"	},		\
+IF_HAVE_VM_LEVEL_PUD(VM_LEVEL_PUD,	"pud-level"	),		\
+IF_HAVE_VM_LEVEL_PMD(VM_LEVEL_PMD,	"pmd-level"	),		\
+IF_HAVE_VM_LEVEL_PTE(VM_LEVEL_PTE,	"pte-level"	)		\
 
 #define show_vma_flags(flags)						\
 	(flags) ? __print_flags(flags, "|",				\
Signature · 1 line
2.19.1
author
#3 lore raw

RFC PATCH v3 3/4 arm64: tlb: Use translation level hint in vm_flags

In reply to Zhenyu Ye

This patch used the VM_LEVEL flags in vma->vm_flags to set the TTL field in tlbi instruction.

---
 arch/arm64/include/asm/mmu.h      |  2 ++
 arch/arm64/include/asm/tlbflush.h | 14 ++++++++------
 arch/arm64/mm/mmu.c               | 14 ++++++++++++++
 3 files changed, 24 insertions(+), 6 deletions(-)
arch/arm64/include/asm/mmu.h +2 -0
index e4d862420bb4..b3f410fee512 100644
@@ -88,6 +88,8 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
 extern void *fixmap_remap_fdt(phys_addr_t dt_phys, int *size, pgprot_t prot);
 extern void mark_linear_text_alias_ro(void);
 extern bool kaslr_requires_kpti(void);
+extern unsigned int get_vma_level(struct vm_area_struct *vma);
+
 
 #define INIT_MM_CONTEXT(name)	\
 	.pgd = init_pg_dir,
arch/arm64/include/asm/tlbflush.h +8 -6
index d141c080e494..93bb09fdfafd 100644
@@ -218,10 +218,11 @@ static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
 					 unsigned long uaddr)
 {
 	unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm));
+	unsigned int level = get_vma_level(vma);
 
 	dsb(ishst);
-	__tlbi_level(vale1is, addr, 0);
-	__tlbi_user_level(vale1is, addr, 0);
+	__tlbi_level(vale1is, addr, level);
+	__tlbi_user_level(vale1is, addr, level);
 }
 
 static inline void flush_tlb_page(struct vm_area_struct *vma,
@@ -242,6 +243,7 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 				     unsigned long stride, bool last_level)
 {
 	unsigned long asid = ASID(vma->vm_mm);
+	unsigned int level = get_vma_level(vma);
 	unsigned long addr;
 
 	start = round_down(start, stride);
@@ -261,11 +263,11 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
 	dsb(ishst);
 	for (addr = start; addr < end; addr += stride) {
 		if (last_level) {
-			__tlbi_level(vale1is, addr, 0);
-			__tlbi_user_level(vale1is, addr, 0);
+			__tlbi_level(vale1is, addr, level);
+			__tlbi_user_level(vale1is, addr, level);
 		} else {
-			__tlbi_level(vae1is, addr, 0);
-			__tlbi_user_level(vae1is, addr, 0);
+			__tlbi_level(vae1is, addr, level);
+			__tlbi_user_level(vae1is, addr, level);
 		}
 	}
 	dsb(ish);
arch/arm64/mm/mmu.c +14 -0
index 128f70852bf3..e6a1221cd86b 100644
@@ -60,6 +60,20 @@ static pud_t bm_pud[PTRS_PER_PUD] __page_aligned_bss __maybe_unused;
 
 static DEFINE_SPINLOCK(swapper_pgdir_lock);
 
+inline unsigned int get_vma_level(struct vm_area_struct *vma)
+{
+	unsigned int level = 0;
+	if (vma->vm_flags & VM_LEVEL_PUD)
+		level = 1;
+	else if (vma->vm_flags & VM_LEVEL_PMD)
+		level = 2;
+	else if (vma->vm_flags & VM_LEVEL_PTE)
+		level = 3;
+
+	vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PMD | VM_LEVEL_PTE);
+	return level;
+}
+
 void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
 {
 	pgd_t *fixmap_pgdp;
Signature · 1 line
2.19.1
author
#4 lore raw

RFC PATCH v3 4/4 mm: Set VM_LEVEL flags in some tlb_flush functions

In reply to Zhenyu Ye

The relevant functions are:

tlb_flush in asm/tlb.h
get_clear_flush and clear_flush in mm/hugetlbpage.c flush_pmd|pud_tlb_range in asm-generic/patable.h do_huge_pmd_numa_page and move_huge_pmd in mm/huge_memory.c

---
 arch/arm64/include/asm/tlb.h  | 12 ++++++++++++
 arch/arm64/mm/hugetlbpage.c   |  4 ++--
 include/asm-generic/pgtable.h | 16 ++++++++++++++--
 mm/huge_memory.c              |  8 +++++++-
 4 files changed, 35 insertions(+), 5 deletions(-)
arch/arm64/include/asm/tlb.h +12 -0
index b76df828e6b7..77fe942b30b6 100644
@@ -27,6 +27,18 @@ static inline void tlb_flush(struct mmu_gather *tlb)
 	bool last_level = !tlb->freed_tables;
 	unsigned long stride = tlb_get_unmap_size(tlb);
 
+	/*
+	 * mm_gather tracked which levels of the page tables
+	 * have been cleared, we can use this info to set
+	 * vm->vm_flags.
+	 */
+	if (tlb->cleared_ptes)
+		vma.vm_flags |= VM_LEVEL_PTE;
+	else if (tlb->cleared_pmds)
+		vma.vm_flags |= VM_LEVEL_PMD;
+	else if (tlb->cleared_puds)
+		vma.vm_flags |= VM_LEVEL_PUD;
+
 	/*
 	 * If we're tearing down the address space then we only care about
 	 * invalidating the walk-cache, since the ASID allocator won't
arch/arm64/mm/hugetlbpage.c +2 -2
index bbeb6a5a6ba6..c35a1bd06bd0 100644
@@ -140,7 +140,7 @@ static pte_t get_clear_flush(struct mm_struct *mm,
 	}
 
 	if (valid) {
-		struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
+		struct vm_area_struct vma = TLB_FLUSH_VMA(mm, VM_LEVEL_PTE);
 		flush_tlb_range(&vma, saddr, addr);
 	}
 	return orig_pte;
@@ -161,7 +161,7 @@ static void clear_flush(struct mm_struct *mm,
 			     unsigned long pgsize,
 			     unsigned long ncontig)
 {
-	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
+	struct vm_area_struct vma = TLB_FLUSH_VMA(mm, VM_LEVEL_PTE);
 	unsigned long i, saddr = addr;
 
 	for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
include/asm-generic/pgtable.h +14 -2
index e2e2bef07dd2..391e704faf7a 100644
@@ -1160,8 +1160,20 @@ static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
  * invalidate the entire TLB which is not desitable.
  * e.g. see arch/arc: flush_pmd_tlb_range
  */
-#define flush_pmd_tlb_range(vma, addr, end)	flush_tlb_range(vma, addr, end)
-#define flush_pud_tlb_range(vma, addr, end)	flush_tlb_range(vma, addr, end)
+#define flush_pmd_tlb_range(vma, addr, end)				\
+	do {								\
+		vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PTE);	\
+		vma->vm_flags |= VM_LEVEL_PMD;				\
+		flush_tlb_range(vma, addr, end);			\
+	} while (0)
+
+#define flush_pud_tlb_range(vma, addr, end)				\
+	do {								\
+		vma->vm_flags &= ~(VM_LEVEL_PMD | VM_LEVEL_PTE);	\
+		vma->vm_flags |= VM_LEVEL_PUD;				\
+		flush_tlb_range(vma, addr, end);			\
+	} while (0)
+
 #else
 #define flush_pmd_tlb_range(vma, addr, end)	BUILD_BUG()
 #define flush_pud_tlb_range(vma, addr, end)	BUILD_BUG()
mm/huge_memory.c +7 -1
index b08b199f9a11..f28ced8d298e 100644
@@ -1646,6 +1646,8 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
 	 * mapping or not. Hence use the tlb range variant
 	 */
 	if (mm_tlb_flush_pending(vma->vm_mm)) {
+		vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PTE);
+		vma->vm_flags |= VM_LEVEL_PMD;
 		flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
 		/*
 		 * change_huge_pmd() released the pmd lock before
@@ -1917,8 +1919,12 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
 		}
 		pmd = move_soft_dirty_pmd(pmd);
 		set_pmd_at(mm, new_addr, new_pmd, pmd);
-		if (force_flush)
+		if (force_flush) {
+			vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PTE);
+			vma->vm_flags |= VM_LEVEL_PMD;
 			flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
+		}
+
 		if (new_ptl != old_ptl)
 			spin_unlock(new_ptl);
 		spin_unlock(old_ptl);
Signature · 1 line
2.19.1
#5 lore raw
In reply to Zhenyu Ye

Hi Zhenyu,

Quoted 17 lines · Zhenyu Ye

--
ChangeList:
v3:
use vma->vm_flags to replace mm->context.flags.

v2:
build the patch on Marc's NV series[1].

v1:
add support for TTL field in arm64.

--
ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate the level of translation table walk holding the leaf entry for the address that is being invalidated. Hardware can use this information to determine if there was a risk of splintering.

Quoted 4 lines

Marc has provided basic support for ARM64-TTL features on his NV series[1] patches. NV is a large feature, however, only patches 62[2] and 67[3] are need by this patch set.
** You only need read those two patches before review this patch. **

It'd be good if you can put the whole thing into a series, otherwise people will have difficulty when reviewing and testing it...

I haven't tracked the previous versions. If Marc is OK to share the two patches below [2][3], I'd suggest you to pick them up, add them in your series, rebase on top of mainline and resend it.

Thanks,
Zenghui

Quoted 26 lines

Some of this patch depends on a feature powered by @Will Deacon two years ago, which tracking the level of page tables in mm_gather. See more in commit a6d60245.

[1] git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git kvm-arm64/nv-5.6-rc1
[2] https://lore.kernel.org/linux-arm-kernel/[email protected]/
[3] https://lore.kernel.org/linux-arm-kernel/[email protected]/

Zhenyu Ye (4):
arm64: Add level-hinted TLB invalidation helper to tlbi_user
mm: Add page table level flags to vm_flags
arm64: tlb: Use translation level hint in vm_flags
mm: Set VM_LEVEL flags in some tlb_flush functions

  arch/arm64/include/asm/mmu.h      |  2 ++
  arch/arm64/include/asm/tlb.h      | 12 +++++++++
  arch/arm64/include/asm/tlbflush.h | 44 ++++++++++++++++++++++++++-----
  arch/arm64/mm/hugetlbpage.c       |  4 +--
  arch/arm64/mm/mmu.c               | 14 ++++++++++
  include/asm-generic/pgtable.h     | 16 +++++++++--
  include/linux/mm.h                | 10 +++++++
  include/trace/events/mmflags.h    | 15 ++++++++++-
  mm/huge_memory.c                  |  8 +++++-
  9 files changed, 113 insertions(+), 12 deletions(-)
author
#6 lore raw

Hi Zenghui,

Quoted 28 lines · Zenghui Yu

Hi Zhenyu,

Quoted 17 lines · Zhenyu Ye

--
ChangeList:
v3:
use vma->vm_flags to replace mm->context.flags.

v2:
build the patch on Marc's NV series[1].

v1:
add support for TTL field in arm64.

--
ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate the level of translation table walk holding the leaf entry for the address that is being invalidated. Hardware can use this information to determine if there was a risk of splintering.

Quoted 4 lines

Marc has provided basic support for ARM64-TTL features on his NV series[1] patches. NV is a large feature, however, only patches 62[2] and 67[3] are need by this patch set.
** You only need read those two patches before review this patch. **

It'd be good if you can put the whole thing into a series, otherwise people will have difficulty when reviewing and testing it...

Quoted 8 lines

I haven't tracked the previous versions. If Marc is OK to share the two patches below [2][3], I'd suggest you to pick them up, add them in your series, rebase on top of mainline and resend it.

Thanks,
Zenghui

Thanks for your review. I'd take your suggestion and resend a new set right now.

Thanks,
Zhenyu

Quoted 27 lines
Quoted 26 lines

Some of this patch depends on a feature powered by @Will Deacon two years ago, which tracking the level of page tables in mm_gather. See more in commit a6d60245.

[1] git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git kvm-arm64/nv-5.6-rc1
[2] https://lore.kernel.org/linux-arm-kernel/[email protected]/
[3] https://lore.kernel.org/linux-arm-kernel/[email protected]/

Zhenyu Ye (4):
arm64: Add level-hinted TLB invalidation helper to tlbi_user
mm: Add page table level flags to vm_flags
arm64: tlb: Use translation level hint in vm_flags
mm: Set VM_LEVEL flags in some tlb_flush functions

  arch/arm64/include/asm/mmu.h      |  2 ++
  arch/arm64/include/asm/tlb.h      | 12 +++++++++
  arch/arm64/include/asm/tlbflush.h | 44 ++++++++++++++++++++++++++-----
  arch/arm64/mm/hugetlbpage.c       |  4 +--
  arch/arm64/mm/mmu.c               | 14 ++++++++++
  include/asm-generic/pgtable.h     | 16 +++++++++--
  include/linux/mm.h                | 10 +++++++
  include/trace/events/mmflags.h    | 15 ++++++++++-
  mm/huge_memory.c                  |  8 +++++-
  9 files changed, 113 insertions(+), 12 deletions(-)