kasan: speed up mte_set_mem_tag_range
PATCH 1/1 kasan: speed up mte_set_mem_tag_range
Use DC GVA / DC GZVA to speed up KASan memory tagging in HW tags mode.
The first cacheline is always tagged using STG/STZG even if the address is cacheline-aligned, as benchmarks show it is faster than a conditional branch.
Signed-off-by Evgenii Stepanov <[email protected]>
--- arch/arm64/include/asm/mte-kasan.h | 40 +------------------ arch/arm64/lib/Makefile | 2 + arch/arm64/lib/mte-kasan.S | 63 ++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 39 deletions(-) create mode 100644 arch/arm64/lib/mte-kasan.S
arch/arm64/include/asm/mte-kasan.h +1 -39
@@ -48,45 +48,7 @@ static inline u8 mte_get_random_tag(void)return mte_get_ptr_tag (addr); } -/* -* Assign allocation tags for a region of memory based on the pointer tag. -* Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and -* size must be non-zero and MTE_GRANULE_SIZE aligned. -*/ -static inline void mte_set_mem_tag_range (void *addr,size_t size, -u8 tag,bool init) -{ -u64 curr, end; - -if (!size) -return ; - - curr = (u64 )__tag_set (addr, tag); - end = curr + size; - -/* -* 'asm volatile' is required to prevent the compiler to move -* the statement outside of the loop. -*/ -if (init) { -do { - asmvolatile (__MTE_PREAMBLE"stzg %0, [%0]" - : - :"r" (curr) - :"memory" ); - curr +=MTE_GRANULE_SIZE ; - }while (curr != end); - }else { -do { - asmvolatile (__MTE_PREAMBLE"stg %0, [%0]" - : - :"r" (curr) - :"memory" ); - curr +=MTE_GRANULE_SIZE ; - }while (curr != end); - } -} - +void mte_set_mem_tag_range (void *addr,size_t size,u8 tag,bool init);void mte_enable_kernel_sync (void );void mte_enable_kernel_async (void );void mte_init_tags (u64 max_tag);
arch/arm64/lib/Makefile +2 -0
@@ -18,3 +18,5 @@ obj-$(CONFIG_CRC32) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_ARM64_MTE) += mte.o + +obj-$(CONFIG_KASAN_HW_TAGS) += mte-kasan.o
arch/arm64/lib/mte-kasan.S new file +63 -0
63 lines not on this page · raw message
Signature · 1 line
2.31.1.751.gd2f1c929bd-goog