Add KernelMemorySanitizer infrastructure

index ed717dd08ff3..1d3aab244897 100644
@@ -198,9 +198,22 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle,
 			       unsigned long **entries)
 {
 	union handle_parts parts = { .handle = handle };
-	void *slab = stack_slabs[parts.slabindex];
+	void *slab;
 	size_t offset = parts.offset << STACK_ALLOC_ALIGN;
-	struct stack_record *stack = slab + offset;
+	struct stack_record *stack;
+
+	if (parts.slabindex > depot_index) {
+		WARN(1, "slab index %d out of bounds (%d) for stack id %08x\n",
+			parts.slabindex, depot_index, handle);
+		*entries = NULL;
+		return 0;
+	}
+	slab = stack_slabs[parts.slabindex];
+	stack = slab + offset;
+	if (!stack) {
+		*entries = NULL;
+		return 0;
+	}
 
 	*entries = stack->entries;
 	return stack->size;

index 1d3aab244897..9d33059d874c 100644
@@ -163,7 +163,7 @@ int stackdepot_memcmp(const unsigned long *u1, const unsigned long *u2,
 			unsigned int n)
 {
 	for ( ; n-- ; u1++, u2++) {
-		if (*u1 != *u2)
+		if ((*u1) != (*u2))
 			return 1;
 	}
 	return 0;

index 3efa97d482cb..24d49c732341 100644
@@ -19,4 +19,6 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries,
 unsigned int stack_depot_fetch(depot_stack_handle_t handle,
 			       unsigned long **entries);
 
+unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries);
+
 #endif

index 9d33059d874c..0efe82fa1e0f 100644
@@ -20,6 +20,7 @@
  */
 
 #include <linux/gfp.h>
+#include <linux/interrupt.h>
 #include <linux/jhash.h>
 #include <linux/kernel.h>
 #include <linux/mm.h>

@@ -314,3 +315,26 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries,
 	return retval;
 }
 EXPORT_SYMBOL_GPL(stack_depot_save);
+
+static inline int in_irqentry_text(unsigned long ptr)
+{
+	return (ptr >= (unsigned long)&__irqentry_text_start &&
+		ptr < (unsigned long)&__irqentry_text_end) ||
+		(ptr >= (unsigned long)&__softirqentry_text_start &&
+		 ptr < (unsigned long)&__softirqentry_text_end);
+}
+
+unsigned int filter_irq_stacks(unsigned long *entries,
+					     unsigned int nr_entries)
+{
+	unsigned int i;
+
+	for (i = 0; i < nr_entries; i++) {
+		if (in_irqentry_text(entries[i])) {
+			/* Include the irqentry function into the stack. */
+			return i + 1;
+		}
+	}
+	return nr_entries;
+}
+EXPORT_SYMBOL_GPL(filter_irq_stacks);

index 2fa710bb6358..3b43899ebc51 100644
@@ -15,7 +15,6 @@
  */
 
 #include <linux/export.h>
-#include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/kasan.h>
 #include <linux/kernel.h>

@@ -42,28 +41,6 @@
 #include "kasan.h"
 #include "../slab.h"
 
-static inline int in_irqentry_text(unsigned long ptr)
-{
-	return (ptr >= (unsigned long)&__irqentry_text_start &&
-		ptr < (unsigned long)&__irqentry_text_end) ||
-		(ptr >= (unsigned long)&__softirqentry_text_start &&
-		 ptr < (unsigned long)&__softirqentry_text_end);
-}
-
-static inline unsigned int filter_irq_stacks(unsigned long *entries,
-					     unsigned int nr_entries)
-{
-	unsigned int i;
-
-	for (i = 0; i < nr_entries; i++) {
-		if (in_irqentry_text(entries[i])) {
-			/* Include the irqentry function into the stack. */
-			return i + 1;
-		}
-	}
-	return nr_entries;
-}
-
 static inline depot_stack_handle_t save_stack(gfp_t flags)
 {
 	unsigned long entries[KASAN_STACK_DEPTH];

index 24d49c732341..ac1b5a78d7f6 100644
@@ -12,6 +12,11 @@
 #define _LINUX_STACKDEPOT_H
 
 typedef u32 depot_stack_handle_t;
+/*
+ * Number of bits in the handle that stack depot doesn't use. Users may store
+ * information in them.
+ */
+#define STACK_DEPOT_EXTRA_BITS 5
 
 depot_stack_handle_t stack_depot_save(unsigned long *entries,
 				      unsigned int nr_entries, gfp_t gfp_flags);

@@ -20,5 +25,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle,
 			       unsigned long **entries);
 
 unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries);
+depot_stack_handle_t set_dsh_extra_bits(depot_stack_handle_t handle,
+					unsigned int bits);
+unsigned int get_dsh_extra_bits(depot_stack_handle_t handle);
 
 #endif

index 0efe82fa1e0f..ecd1a0dfb3fb 100644
@@ -40,8 +40,10 @@
 #define STACK_ALLOC_ALIGN 4
 #define STACK_ALLOC_OFFSET_BITS (STACK_ALLOC_ORDER + PAGE_SHIFT - \
 					STACK_ALLOC_ALIGN)
+
 #define STACK_ALLOC_INDEX_BITS (DEPOT_STACK_BITS - \
-		STACK_ALLOC_NULL_PROTECTION_BITS - STACK_ALLOC_OFFSET_BITS)
+		STACK_ALLOC_NULL_PROTECTION_BITS - \
+		STACK_ALLOC_OFFSET_BITS - STACK_DEPOT_EXTRA_BITS)
 #define STACK_ALLOC_SLABS_CAP 8192
 #define STACK_ALLOC_MAX_SLABS \
 	(((1LL << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \

@@ -54,6 +56,7 @@ union handle_parts {
 		u32 slabindex : STACK_ALLOC_INDEX_BITS;
 		u32 offset : STACK_ALLOC_OFFSET_BITS;
 		u32 valid : STACK_ALLOC_NULL_PROTECTION_BITS;
+		u32 extra : STACK_DEPOT_EXTRA_BITS;
 	};
 };
 

@@ -72,6 +75,24 @@ static int next_slab_inited;
 static size_t depot_offset;
 static DEFINE_SPINLOCK(depot_lock);
 
+depot_stack_handle_t set_dsh_extra_bits(depot_stack_handle_t handle,
+					u32 bits)
+{
+	union handle_parts parts = { .handle = handle };
+
+	parts.extra = bits & ((1U << STACK_DEPOT_EXTRA_BITS) - 1);
+	return parts.handle;
+}
+EXPORT_SYMBOL_GPL(set_dsh_extra_bits);
+
+u32 get_dsh_extra_bits(depot_stack_handle_t handle)
+{
+	union handle_parts parts = { .handle = handle };
+
+	return parts.extra;
+}
+EXPORT_SYMBOL_GPL(get_dsh_extra_bits);
+
 static bool init_stack_slab(void **prealloc)
 {
 	if (!*prealloc)

@@ -132,6 +153,7 @@ static struct stack_record *depot_alloc_stack(unsigned long *entries, int size,
 	stack->handle.slabindex = depot_index;
 	stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN;
 	stack->handle.valid = 1;
+	stack->handle.extra = 0;
 	memcpy(stack->entries, entries, size * sizeof(unsigned long));
 	depot_offset += required_size;
 

index e5b817cb86e7..30f8d3ed9bde 100644
@@ -44,6 +44,7 @@ struct vm_area_struct;
 #else
 #define ___GFP_NOLOCKDEP	0
 #endif
+#define ___GFP_NO_KMSAN_SHADOW  0x1000000u
 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
 
 /*

@@ -212,12 +213,13 @@ struct vm_area_struct;
 #define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)
 #define __GFP_COMP	((__force gfp_t)___GFP_COMP)
 #define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)
+#define __GFP_NO_KMSAN_SHADOW  ((__force gfp_t)___GFP_NO_KMSAN_SHADOW)
 
 /* Disable lockdep for GFP context tracking */
 #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
 
 /* Room for N __GFP_FOO bits */
-#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP))
+#define __GFP_BITS_SHIFT (25)
 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
 
 /**

index 333a6695a918..01e1f754f6d3 100644
@@ -24,6 +24,13 @@
 #define __no_sanitize_address
 #endif
 
+#if __has_feature(memory_sanitizer)
+# define __SANITIZE_MEMORY__
+# define __no_sanitize_memory __attribute__((no_sanitize("kernel-memory")))
+#else
+# define __no_sanitize_memory
+#endif
+
 /*
  * Not all versions of clang implement the the type-generic versions
  * of the builtin overflow checkers. Fortunately, clang implements

index d7ee4c6bad48..e5ebc788dde4 100644
@@ -145,6 +145,11 @@
 #define __no_sanitize_address
 #endif
 
+/*
+ * GCC doesn't support KMSAN.
+ */
+#define __no_sanitize_memory
+
 #if GCC_VERSION >= 50100
 #define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
 #endif

index 52e5f5f2240d..586629e20436 100644
@@ -139,7 +139,22 @@ extern unsigned int ptrs_per_p4d;
 # define VMEMMAP_START		__VMEMMAP_BASE_L4
 #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 
+#ifndef CONFIG_KMSAN
 #define VMALLOC_END		(VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1)
+#else
+/*
+ * In KMSAN builds vmalloc area is four times smaller, and the remaining 3/4
+ * are used to keep the metadata for virtual pages.
+ */
+#define VMALLOC_QUARTER_SIZE	((VMALLOC_SIZE_TB << 40) >> 2)
+#define VMALLOC_END		(VMALLOC_START + VMALLOC_QUARTER_SIZE - 1)
+#define VMALLOC_SHADOW_OFFSET	VMALLOC_QUARTER_SIZE
+#define VMALLOC_ORIGIN_OFFSET	(VMALLOC_QUARTER_SIZE * 2)
+#define VMALLOC_META_END	(VMALLOC_END + VMALLOC_ORIGIN_OFFSET)
+#define MODULES_SHADOW_START	(VMALLOC_META_END + 1)
+#define MODULES_ORIGIN_START	(MODULES_SHADOW_START + MODULES_LEN)
+#define MODULES_ORIGIN_END	(MODULES_ORIGIN_START + MODULES_LEN)
+#endif
 
 #define MODULES_VADDR		(__START_KERNEL_map + KERNEL_IMAGE_SIZE)
 /* The module sections ends with the start of the fixmap */

127 lines not on this page · raw message

335 lines not on this page · raw message

index 270aa8fd2800..37411d907ef4 100644
@@ -216,6 +216,15 @@ struct page {
 					   not kmapped, ie. highmem) */
 #endif /* WANT_PAGE_VIRTUAL */
 
+#ifdef CONFIG_KMSAN
+	/*
+	 * Bits in struct page are scarce, so the LSB in *shadow is used to
+	 * indicate whether the page should be ignored by KMSAN or not.
+	 */
+	struct page *shadow;
+	struct page *origin;
+#endif
+
 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
 	int _last_cpupid;
 #endif

index 467d26046416..ba9d43192cad 100644
@@ -15,6 +15,7 @@
 #include <linux/sem.h>
 #include <linux/shm.h>
 #include <linux/kcov.h>
+#include <linux/kmsan.h>
 #include <linux/mutex.h>
 #include <linux/plist.h>
 #include <linux/hrtimer.h>

@@ -1180,6 +1181,10 @@ struct task_struct {
 	unsigned int			kasan_depth;
 #endif
 
+#ifdef CONFIG_KMSAN
+	struct kmsan_task_state		kmsan;
+#endif
+
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	/* Index of current stored address in ret_stack: */
 	int				curr_ret_stack;

547 lines not on this page · raw message

161 lines not on this page · raw message

79 lines not on this page · raw message

143 lines not on this page · raw message

456 lines not on this page · raw message

new file mode 100644
index 000000000000..eaa7f771b6a5
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * KMSAN shadow API.
+ *
+ * This should be agnostic to shadow implementation details.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#ifndef __MM_KMSAN_KMSAN_SHADOW_H
+#define __MM_KMSAN_KMSAN_SHADOW_H
+
+#include <asm/cpu_entry_area.h>  /* for CPU_ENTRY_AREA_MAP_SIZE */
+
+struct shadow_origin_ptr {
+	void *s, *o;
+};
+
+struct shadow_origin_ptr kmsan_get_shadow_origin_ptr(void *addr, u64 size,
+						     bool store);
+void *kmsan_get_metadata(void *addr, size_t size, bool is_origin);
+void __init kmsan_init_alloc_meta_for_range(void *start, void *end);
+
+#endif  /* __MM_KMSAN_KMSAN_SHADOW_H */

new file mode 100644
index 000000000000..0de8aafac510
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN compiler API.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include "kmsan.h"
+#include <linux/gfp.h>
+#include <linux/mm.h>
+
+static bool is_bad_asm_addr(void *addr, u64 size, bool is_store)
+{
+	if ((u64)addr < TASK_SIZE)
+		return true;
+	if (!kmsan_get_metadata(addr, size, META_SHADOW))
+		return true;
+	return false;
+}
+
+struct shadow_origin_ptr __msan_metadata_ptr_for_load_n(void *addr, u64 size)
+{
+	return kmsan_get_shadow_origin_ptr(addr, size, /*store*/false);
+}
+EXPORT_SYMBOL(__msan_metadata_ptr_for_load_n);
+
+struct shadow_origin_ptr __msan_metadata_ptr_for_store_n(void *addr, u64 size)
+{
+	return kmsan_get_shadow_origin_ptr(addr, size, /*store*/true);
+}
+EXPORT_SYMBOL(__msan_metadata_ptr_for_store_n);
+
+#define DECLARE_METADATA_PTR_GETTER(size)	\
+struct shadow_origin_ptr __msan_metadata_ptr_for_load_##size(void *addr) \
+{		\
+	return kmsan_get_shadow_origin_ptr(addr, size, /*store*/false);	\
+}		\
+EXPORT_SYMBOL(__msan_metadata_ptr_for_load_##size);			\
+		\
+struct shadow_origin_ptr __msan_metadata_ptr_for_store_##size(void *addr) \
+{									\
+	return kmsan_get_shadow_origin_ptr(addr, size, /*store*/true);	\
+}									\
+EXPORT_SYMBOL(__msan_metadata_ptr_for_store_##size)
+
+DECLARE_METADATA_PTR_GETTER(1);
+DECLARE_METADATA_PTR_GETTER(2);
+DECLARE_METADATA_PTR_GETTER(4);
+DECLARE_METADATA_PTR_GETTER(8);
+
+void __msan_instrument_asm_store(void *addr, u64 size)
+{
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+	/*
+	 * Most of the accesses are below 32 bytes. The two exceptions so far
+	 * are clwb() (64 bytes) and FPU state (512 bytes).
+	 * It's unlikely that the assembly will touch more than 512 bytes.
+	 */
+	if (size > 512) {
+		WARN_ONCE(1, "assembly store size too big: %d\n", size);
+		size = 8;
+	}
+	if (is_bad_asm_addr(addr, size, /*is_store*/true))
+		return;
+	irq_flags = kmsan_enter_runtime();
+	/* Unpoisoning the memory on best effort. */
+	kmsan_internal_unpoison_shadow(addr, size, /*checked*/false);
+	kmsan_leave_runtime(irq_flags);
+}
+EXPORT_SYMBOL(__msan_instrument_asm_store);
+
+void *__msan_memmove(void *dst, const void *src, size_t n)
+{
+	void *result;
+
+	result = __memmove(dst, src, n);
+	if (!n)
+		/* Some people call memmove() with zero length. */
+		return result;
+	if (!kmsan_ready || kmsan_in_runtime())
+		return result;
+
+	kmsan_memmove_metadata(dst, (void *)src, n);
+
+	return result;
+}
+EXPORT_SYMBOL(__msan_memmove);
+
+void *__msan_memmove_nosanitize(void *dst, void *src, u64 n)
+{
+	return __memmove(dst, src, n);
+}
+EXPORT_SYMBOL(__msan_memmove_nosanitize);
+
+void *__msan_memcpy(void *dst, const void *src, u64 n)
+{
+	void *result;
+
+	result = __memcpy(dst, src, n);
+	if (!n)
+		/* Some people call memcpy() with zero length. */
+		return result;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return result;
+
+	kmsan_memcpy_metadata(dst, (void *)src, n);
+
+	return result;
+}
+EXPORT_SYMBOL(__msan_memcpy);
+
+void *__msan_memcpy_nosanitize(void *dst, void *src, u64 n)
+{
+	return __memcpy(dst, src, n);
+}
+EXPORT_SYMBOL(__msan_memcpy_nosanitize);
+
+void *__msan_memset(void *dst, int c, size_t n)
+{
+	void *result;
+	unsigned long irq_flags;
+
+	result = __memset(dst, c, n);
+	if (!kmsan_ready || kmsan_in_runtime())
+		return result;
+
+	irq_flags = kmsan_enter_runtime();
+	/*
+	 * Clang doesn't pass parameter metadata here, so it is impossible to
+	 * use shadow of @c to set up the shadow for @dst.
+	 */
+	kmsan_internal_unpoison_shadow(dst, n, /*checked*/false);
+	kmsan_leave_runtime(irq_flags);
+
+	return result;
+}
+EXPORT_SYMBOL(__msan_memset);
+
+void *__msan_memset_nosanitize(void *dst, int c, size_t n)
+{
+	return __memset(dst, c, n);
+}
+EXPORT_SYMBOL(__msan_memset_nosanitize);
+
+depot_stack_handle_t __msan_chain_origin(depot_stack_handle_t origin)
+{
+	depot_stack_handle_t ret = 0;
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return ret;
+
+	/* Creating new origins may allocate memory. */
+	irq_flags = kmsan_enter_runtime();
+	ret = kmsan_internal_chain_origin(origin);
+	kmsan_leave_runtime(irq_flags);
+	return ret;
+}
+EXPORT_SYMBOL(__msan_chain_origin);
+
+void __msan_poison_alloca(void *address, u64 size, char *descr)
+{
+	depot_stack_handle_t handle;
+	unsigned long entries[4];
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+
+	kmsan_internal_memset_shadow(address, -1, size, /*checked*/true);
+
+	entries[0] = KMSAN_ALLOCA_MAGIC_ORIGIN;
+	entries[1] = (u64)descr;
+	entries[2] = (u64)__builtin_return_address(0);
+	entries[3] = (u64)kmsan_internal_return_address(1);
+
+	/* stack_depot_save() may allocate memory. */
+	irq_flags = kmsan_enter_runtime();
+	handle = stack_depot_save(entries, ARRAY_SIZE(entries), GFP_ATOMIC);
+	kmsan_leave_runtime(irq_flags);
+	kmsan_internal_set_origin(address, size, handle);
+}
+EXPORT_SYMBOL(__msan_poison_alloca);
+
+void __msan_unpoison_alloca(void *address, u64 size)
+{
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+
+	irq_flags = kmsan_enter_runtime();
+	kmsan_internal_unpoison_shadow(address, size, /*checked*/true);
+	kmsan_leave_runtime(irq_flags);
+}
+EXPORT_SYMBOL(__msan_unpoison_alloca);
+
+void __msan_warning(u32 origin)
+{
+	unsigned long irq_flags;
+
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+	irq_flags = kmsan_enter_runtime();
+	kmsan_report(origin, /*address*/0, /*size*/0,
+		/*off_first*/0, /*off_last*/0, /*user_addr*/0, REASON_ANY);
+	kmsan_leave_runtime(irq_flags);
+}
+EXPORT_SYMBOL(__msan_warning);
+
+struct kmsan_context_state *__msan_get_context_state(void)
+{
+	struct kmsan_context_state *ret;
+
+	ret = kmsan_task_context_state();
+	BUG_ON(!ret);
+	return ret;
+}
+EXPORT_SYMBOL(__msan_get_context_state);

new file mode 100644
index 000000000000..f924f29f90f9
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Assembly bits to safely invoke KMSAN hooks from .S files.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+#ifndef _ASM_X86_KMSAN_H
+#define _ASM_X86_KMSAN_H
+
+#ifdef CONFIG_KMSAN
+
+#ifdef __ASSEMBLY__
+.macro KMSAN_PUSH_REGS
+	pushq	%rax
+	pushq	%rcx
+	pushq	%rdx
+	pushq	%rdi
+	pushq	%rsi
+	pushq	%r8
+	pushq	%r9
+	pushq	%r10
+	pushq	%r11
+.endm
+
+.macro KMSAN_POP_REGS
+	popq	%r11
+	popq	%r10
+	popq	%r9
+	popq	%r8
+	popq	%rsi
+	popq	%rdi
+	popq	%rdx
+	popq	%rcx
+	popq	%rax
+
+.endm
+
+.macro KMSAN_CALL_HOOK fname
+	KMSAN_PUSH_REGS
+	call \fname
+	KMSAN_POP_REGS
+.endm
+
+.macro KMSAN_CONTEXT_ENTER
+	KMSAN_CALL_HOOK kmsan_context_enter
+.endm
+
+.macro KMSAN_CONTEXT_EXIT
+	KMSAN_CALL_HOOK kmsan_context_exit
+.endm
+
+#define KMSAN_INTERRUPT_ENTER KMSAN_CONTEXT_ENTER
+#define KMSAN_INTERRUPT_EXIT KMSAN_CONTEXT_EXIT
+
+#define KMSAN_SOFTIRQ_ENTER KMSAN_CONTEXT_ENTER
+#define KMSAN_SOFTIRQ_EXIT KMSAN_CONTEXT_EXIT
+
+#define KMSAN_NMI_ENTER KMSAN_CONTEXT_ENTER
+#define KMSAN_NMI_EXIT KMSAN_CONTEXT_EXIT
+
+#define KMSAN_IST_ENTER(shift_ist) KMSAN_CONTEXT_ENTER
+#define KMSAN_IST_EXIT(shift_ist) KMSAN_CONTEXT_EXIT
+
+.macro KMSAN_UNPOISON_PT_REGS
+	KMSAN_CALL_HOOK kmsan_unpoison_pt_regs
+.endm
+
+#else
+#error this header must be included into an assembly file
+#endif
+
+#else /* ifdef CONFIG_KMSAN */
+
+#define KMSAN_INTERRUPT_ENTER
+#define KMSAN_INTERRUPT_EXIT
+#define KMSAN_SOFTIRQ_ENTER
+#define KMSAN_SOFTIRQ_EXIT
+#define KMSAN_NMI_ENTER
+#define KMSAN_NMI_EXIT
+#define KMSAN_SYSCALL_ENTER
+#define KMSAN_SYSCALL_EXIT
+#define KMSAN_IST_ENTER(shift_ist)
+#define KMSAN_IST_EXIT(shift_ist)
+#define KMSAN_UNPOISON_PT_REGS
+
+#endif /* ifdef CONFIG_KMSAN */
+#endif /* ifndef _ASM_X86_KMSAN_H */

new file mode 100644
index 000000000000..7af31642cd45
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KMSAN hooks for entry_64.S
+ *
+ * Copyright (C) 2018-2019 Google LLC
+ * Author: Alexander Potapenko <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+#include "kmsan.h"
+
+void kmsan_context_enter(void)
+{
+	int level = this_cpu_inc_return(kmsan_context_level);
+
+	BUG_ON(level >= KMSAN_NESTED_CONTEXT_MAX);
+}
+EXPORT_SYMBOL(kmsan_context_enter);
+
+void kmsan_context_exit(void)
+{
+	int level = this_cpu_dec_return(kmsan_context_level);
+
+	BUG_ON(level < 0);
+}
+EXPORT_SYMBOL(kmsan_context_exit);
+
+void kmsan_unpoison_pt_regs(struct pt_regs *regs)
+{
+	if (!kmsan_ready || kmsan_in_runtime())
+		return;
+	kmsan_internal_unpoison_shadow(regs, sizeof(*regs), /*checked*/true);
+}
+EXPORT_SYMBOL(kmsan_unpoison_pt_regs);

416 lines not on this page · raw message

index ecd1a0dfb3fb..2e7d2232ed3c 100644
@@ -295,7 +295,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries,
 		 */
 		alloc_flags &= ~GFP_ZONEMASK;
 		alloc_flags &= (GFP_ATOMIC | GFP_KERNEL);
-		alloc_flags |= __GFP_NOWARN;
+		alloc_flags |= (__GFP_NOWARN | __GFP_NO_KMSAN_SHADOW);
 		page = alloc_pages(alloc_flags, STACK_ALLOC_ORDER);
 		if (page)
 			prealloc = page_address(page);

index 5e88e7e33abe..de7942423836 100644
@@ -255,6 +255,7 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
  */
 #include <asm/barrier.h>
 #include <linux/kasan-checks.h>
+#include <linux/kmsan-checks.h>
 
 #define __READ_ONCE(x, check)						\
 ({									\

@@ -270,9 +271,9 @@ static __always_inline void __write_once_size(volatile void *p, void *res, int s
 
 /*
  * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need
- * to hide memory access from KASAN.
+ * to hide memory access from KASAN or KMSAN.
  */
-#define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0)
+#define READ_ONCE_NOCHECK(x) KMSAN_INIT_VALUE(__READ_ONCE(x, 0))
 
 static __no_kasan_or_inline
 unsigned long read_word_at_a_time(const void *addr)

index 499578f7e6d7..82c3bceb9999 100644
@@ -4,6 +4,7 @@
 
 #include <linux/sched.h>
 #include <linux/ftrace.h>
+#include <linux/kmsan-checks.h>
 #include <asm/ptrace.h>
 #include <asm/stacktrace.h>
 

@@ -100,9 +101,10 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
 #endif
 
 /*
- * This disables KASAN checking when reading a value from another task's stack,
- * since the other task could be running on another CPU and could have poisoned
- * the stack in the meantime.
+ * This disables KASAN/KMSAN checking when reading a value from another task's
+ * stack, since the other task could be running on another CPU and could have
+ * poisoned the stack in the meantime. Frame pointers are uninitialized by
+ * default, so for KMSAN we mark the return value initialized unconditionally.
  */
 #define READ_ONCE_TASK_STACK(task, x)			\
 ({							\

@@ -111,7 +113,7 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
 		val = READ_ONCE(x);			\
 	else						\
 		val = READ_ONCE_NOCHECK(x);		\
-	val;						\
+	KMSAN_INIT_VALUE(val);				\
 })
 
 static inline bool task_on_another_cpu(struct task_struct *task)

index 304d31d8cbbc..37834a7b88cd 100644
@@ -320,7 +320,17 @@ static void dump_pagetable(unsigned long address)
 
 void vmalloc_sync_all(void)
 {
+#ifdef CONFIG_KMSAN
+	/*
+	 * For KMSAN, make sure metadata pages for vmalloc area and modules are
+	 * also synced.
+	 */
+	sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_META_END);
+	sync_global_pgds(MODULES_SHADOW_START & PGDIR_MASK,
+		MODULES_ORIGIN_END);
+#else
 	sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
+#endif
 }
 
 /*

@@ -337,7 +347,17 @@ static noinline int vmalloc_fault(unsigned long address)
 	pte_t *pte;
 
 	/* Make sure we are in vmalloc area: */
+#ifdef CONFIG_KMSAN
+	/*
+	 * For KMSAN, make sure metadata pages for vmalloc area and modules are
+	 * also synced.
+	 */
+	if (!(address >= VMALLOC_START && address < VMALLOC_META_END) &&
+		!(address >= MODULES_SHADOW_START &&
+		  address < MODULES_ORIGIN_END))
+#else
 	if (!(address >= VMALLOC_START && address < VMALLOC_END))
+#endif
 		return -1;
 
 	/*

229 lines not on this page · raw message

index 0427a86743a4..98c5f4062cbf 100644
@@ -11,6 +11,7 @@
 
 #include <linux/export.h>
 #include <linux/kernel_stat.h>
+#include <linux/kmsan.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/mm.h>

@@ -370,7 +371,9 @@ static inline void invoke_softirq(void)
 		 * it is the irq stack, because it should be near empty
 		 * at this stage.
 		 */
+		kmsan_context_enter();
 		__do_softirq();
+		kmsan_context_exit();
 #else
 		/*
 		 * Otherwise, irq_exit() is called on the task stack that can

@@ -600,7 +603,9 @@ static void run_ksoftirqd(unsigned int cpu)
 		 * We can safely run softirq on inline stack, as we are not deep
 		 * in the task stack here.
 		 */
+		kmsan_context_enter();
 		__do_softirq();
+		kmsan_context_exit();
 		local_irq_enable();
 		cond_resched();
 		return;

index 288b065955b7..ea9fbf09f43b 100644
@@ -7,18 +7,20 @@
 #endif
 
 #ifdef CONFIG_KASAN
-#define KASAN_STACK_ORDER 1
+#define EXTRA_STACK_ORDER 1
+#elif defined(CONFIG_KMSAN)
+#define EXTRA_STACK_ORDER 2
 #else
-#define KASAN_STACK_ORDER 0
+#define EXTRA_STACK_ORDER 0
 #endif
 
-#define THREAD_SIZE_ORDER	(2 + KASAN_STACK_ORDER)
+#define THREAD_SIZE_ORDER	(2 + EXTRA_STACK_ORDER)
 #define THREAD_SIZE  (PAGE_SIZE << THREAD_SIZE_ORDER)
 
-#define EXCEPTION_STACK_ORDER (0 + KASAN_STACK_ORDER)
+#define EXCEPTION_STACK_ORDER (0 + EXTRA_STACK_ORDER)
 #define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER)
 
-#define IRQ_STACK_ORDER (2 + KASAN_STACK_ORDER)
+#define IRQ_STACK_ORDER (2 + EXTRA_STACK_ORDER)
 #define IRQ_STACK_SIZE (PAGE_SIZE << IRQ_STACK_ORDER)
 
 /*

index d11389710b12..c5097171c37e 100644
@@ -21,6 +21,8 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/kasan.h>
+#include <linux/kmsan.h>
+#include <linux/kmsan-checks.h> /* KMSAN_INIT_VALUE */
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/mempolicy.h>

@@ -283,17 +285,27 @@ static void prefetch_freepointer(const struct kmem_cache *s, void *object)
 	prefetch(object + s->offset);
 }
 
+/*
+ * When running under KMSAN, get_freepointer_safe() may return an uninitialized
+ * pointer value in the case the current thread loses the race for the next
+ * memory chunk in the freelist. In that case this_cpu_cmpxchg_double() in
+ * slab_alloc_node() will fail, so the uninitialized value won't be used, but
+ * KMSAN will still check all arguments of cmpxchg because of imperfect
+ * handling of inline assembly.
+ * To work around this problem, use KMSAN_INIT_VALUE() to force initialize the
+ * return value of get_freepointer_safe().
+ */
 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
 {
 	unsigned long freepointer_addr;
 	void *p;
 
 	if (!debug_pagealloc_enabled())
-		return get_freepointer(s, object);
+		return KMSAN_INIT_VALUE(get_freepointer(s, object));
 
 	freepointer_addr = (unsigned long)object + s->offset;
 	probe_kernel_read(&p, (void **)freepointer_addr, sizeof(p));
-	return freelist_ptr(s, p, freepointer_addr);
+	return KMSAN_INIT_VALUE(freelist_ptr(s, p, freepointer_addr));
 }
 
 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)

@@ -1391,6 +1403,7 @@ static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
 	ptr = kasan_kmalloc_large(ptr, size, flags);
 	/* As ptr might get tagged, call kmemleak hook after KASAN. */
 	kmemleak_alloc(ptr, size, 1, flags);
+	kmsan_kmalloc_large(ptr, size, flags);
 	return ptr;
 }
 

@@ -1398,6 +1411,7 @@ static __always_inline void kfree_hook(void *x)
 {
 	kmemleak_free(x);
 	kasan_kfree_large(x, _RET_IP_);
+	kmsan_kfree_large(x);
 }
 
 static __always_inline bool slab_free_hook(struct kmem_cache *s, void *x)

@@ -1441,6 +1455,7 @@ static inline bool slab_free_freelist_hook(struct kmem_cache *s,
 	do {
 		object = next;
 		next = get_freepointer(s, object);
+		kmsan_slab_free(s, object);
 
 		if (slab_want_init_on_free(s)) {
 			/*

@@ -2756,6 +2771,7 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s,
 	if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object)
 		memset(object, 0, s->object_size);
 
+	kmsan_slab_alloc(s, object, gfpflags);
 	slab_post_alloc_hook(s, gfpflags, 1, &object);
 
 	return object;

@@ -3137,7 +3153,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 			  void **p)
 {
 	struct kmem_cache_cpu *c;
-	int i;
+	int i, j;
 
 	/* memcg and kmem_cache debug support */
 	s = slab_pre_alloc_hook(s, flags);

@@ -3178,11 +3194,11 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 
 	/* Clear memory outside IRQ disabled fastpath loop */
 	if (unlikely(slab_want_init_on_alloc(flags, s))) {
-		int j;
-
 		for (j = 0; j < i; j++)
 			memset(p[j], 0, s->object_size);
 	}
+	for (j = 0; j < i; j++)
+		kmsan_slab_alloc(s, p[j], flags);
 
 	/* memcg and kmem_cache debug support */
 	slab_post_alloc_hook(s, flags, size, p);

@@ -3783,6 +3799,7 @@ static int __init setup_slub_min_objects(char *str)
 
 __setup("slub_min_objects=", setup_slub_min_objects);
 
+__no_sanitize_memory
 void *__kmalloc(size_t size, gfp_t flags)
 {
 	struct kmem_cache *s;

@@ -5692,6 +5709,7 @@ static char *create_unique_id(struct kmem_cache *s)
 	p += sprintf(p, "%07u", s->size);
 
 	BUG_ON(p > name + ID_STR_LENGTH - 1);
+	kmsan_unpoison_shadow(name, p - name);
 	return name;
 }
 

@@ -5841,6 +5859,7 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
 	al->name = name;
 	al->next = alias_list;
 	alias_list = al;
+	kmsan_unpoison_shadow(al, sizeof(struct saved_alias));
 	return 0;
 }
 

index 939b1cff4a7b..045856c38f49 100644
@@ -44,14 +44,27 @@ void clear_page_orig(void *page);
 void clear_page_rep(void *page);
 void clear_page_erms(void *page);
 
+/* This is an assembly header, avoid including too much of kmsan.h */
+#ifdef CONFIG_KMSAN
+void kmsan_unpoison_shadow(const void *addr, size_t size);
+#endif
+__no_sanitize_memory
 static inline void clear_page(void *page)
 {
+#ifdef CONFIG_KMSAN
+	/* alternative_call_2() changes |page|. */
+	void *page_copy = page;
+#endif
 	alternative_call_2(clear_page_orig,
 			   clear_page_rep, X86_FEATURE_REP_GOOD,
 			   clear_page_erms, X86_FEATURE_ERMS,
 			   "=D" (page),
 			   "0" (page)
 			   : "cc", "memory", "rax", "rcx");
+#ifdef CONFIG_KMSAN
+	/* Clear KMSAN shadow for the pages that have it. */
+	kmsan_unpoison_shadow(page_copy, PAGE_SIZE);
+#endif
 }
 
 void copy_page(void *to, void *from);

index b3a2936377b5..e93585dbfc49 100644
@@ -7,6 +7,7 @@
  * (C) Copyright 1995 1996 Linus Torvalds
  */
 
+#include <linux/kmsan.h>
 #include <linux/memblock.h>
 #include <linux/init.h>
 #include <linux/io.h>

@@ -451,6 +452,8 @@ void iounmap(volatile void __iomem *addr)
 		return;
 	}
 
+	kmsan_iounmap_page_range((unsigned long)addr,
+		(unsigned long)addr + get_vm_area_size(p));
 	free_memtype(p->phys_addr, p->phys_addr + get_vm_area_size(p));
 
 	/* Finally remove it */

index ea5cdbd8c2c3..9f6efa26e9b5 100644
@@ -5,6 +5,7 @@
 #include <linux/fs.h>
 #include <linux/kernel.h>
 #include <linux/bug.h>
+#include <linux/kmsan.h>
 #include <linux/mm.h>
 #include <linux/uaccess.h>
 #include <linux/hardirq.h>

@@ -255,6 +256,7 @@ static inline void copy_user_highpage(struct page *to, struct page *from,
 	vfrom = kmap_atomic(from);
 	vto = kmap_atomic(to);
 	copy_user_page(vto, vfrom, vaddr, to);
+	kmsan_unpoison_shadow(page_address(to), PAGE_SIZE);
 	kunmap_atomic(vto);
 	kunmap_atomic(vfrom);
 }

@@ -270,6 +272,7 @@ static inline void copy_highpage(struct page *to, struct page *from)
 	vfrom = kmap_atomic(from);
 	vto = kmap_atomic(to);
 	copy_page(vto, vfrom);
+	kmsan_copy_page_meta(to, from);
 	kunmap_atomic(vto);
 	kunmap_atomic(vfrom);
 }

index 3f0e18543de8..14b0325b6fa9 100644
@@ -6,6 +6,7 @@
  *
  * (C) Copyright 1995 1996 Linus Torvalds
  */
+#include <linux/kmsan.h>
 #include <linux/vmalloc.h>
 #include <linux/mm.h>
 #include <linux/sched.h>

@@ -214,6 +215,8 @@ int ioremap_page_range(unsigned long addr,
 	unsigned long start;
 	unsigned long next;
 	int err;
+	unsigned long old_addr = addr;
+	phys_addr_t old_phys_addr = phys_addr;
 
 	might_sleep();
 	BUG_ON(addr >= end);

@@ -228,6 +231,8 @@ int ioremap_page_range(unsigned long addr,
 	} while (pgd++, phys_addr += (next - addr), addr = next, addr != end);
 
 	flush_cache_vmap(start, end);
+	if (!err)
+		kmsan_ioremap_page_range(old_addr, end, old_phys_addr, prot);
 
 	return err;
 }

index 7646bf993b25..f2f4a6bbadf9 100644
@@ -4,6 +4,7 @@
 #include <linux/err.h>
 #include <linux/spinlock.h>
 
+#include <linux/kmsan.h>
 #include <linux/mm.h>
 #include <linux/memremap.h>
 #include <linux/pagemap.h>

@@ -2363,6 +2364,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
 	    gup_fast_permitted(start, end)) {
 		local_irq_save(flags);
 		gup_pgd_range(start, end, write ? FOLL_WRITE : 0, pages, &nr);
+		kmsan_gup_pgd_range(pages, nr);
 		local_irq_restore(flags);
 	}
 

@@ -2432,6 +2434,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages,
 	    gup_fast_permitted(start, end)) {
 		local_irq_disable();
 		gup_pgd_range(addr, end, gup_flags, pages, &nr);
+		kmsan_gup_pgd_range(pages, nr);
 		local_irq_enable();
 		ret = nr;
 	}

index 606da187d1de..143ce6549f02 100644
@@ -51,6 +51,7 @@
 #include <linux/highmem.h>
 #include <linux/pagemap.h>
 #include <linux/memremap.h>
+#include <linux/kmsan.h>
 #include <linux/ksm.h>
 #include <linux/rmap.h>
 #include <linux/export.h>

@@ -2423,6 +2424,7 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
 				put_page(old_page);
 			return 0;
 		}
+		kmsan_copy_page_meta(new_page, old_page);
 	}
 
 	if (mem_cgroup_try_charge_delay(new_page, mm, GFP_KERNEL, &memcg, false))

index 4785a8a2040e..4940901b79b3 100644
@@ -26,6 +26,8 @@
 #include <linux/compiler.h>
 #include <linux/kernel.h>
 #include <linux/kasan.h>
+#include <linux/kmsan.h>
+#include <linux/kmsan-checks.h>
 #include <linux/module.h>
 #include <linux/suspend.h>
 #include <linux/pagevec.h>

@@ -1133,6 +1135,7 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	VM_BUG_ON_PAGE(PageTail(page), page);
 
 	trace_mm_page_free(page, order);
+	kmsan_free_page(page, order);
 
 	/*
 	 * Check tail pages before head page information is cleared to

@@ -3129,6 +3132,7 @@ void split_page(struct page *page, unsigned int order)
 	VM_BUG_ON_PAGE(PageCompound(page), page);
 	VM_BUG_ON_PAGE(!page_count(page), page);
 
+	kmsan_split_page(page, order);
 	for (i = 1; i < (1 << order); i++)
 		set_page_refcounted(page + i);
 	split_page_owner(page, order);

@@ -3261,6 +3265,14 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone,
 /*
  * Allocate a page from the given zone. Use pcplists for order-0 allocations.
  */
+
+/*
+ * Do not instrument rmqueue() with KMSAN. This function may call
+ * __msan_poison_alloca() through a call to set_pfnblock_flags_mask().
+ * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it
+ * may call rmqueue() again, which will result in a deadlock.
+ */
+__no_sanitize_memory
 static inline
 struct page *rmqueue(struct zone *preferred_zone,
 			struct zone *zone, unsigned int order,

@@ -4784,6 +4796,11 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
 
 	trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
 
+	if (page)
+		if (kmsan_alloc_page(page, order, gfp_mask)) {
+			__free_pages(page, order);
+			page = NULL;
+		}
 	return page;
 }
 EXPORT_SYMBOL(__alloc_pages_nodemask);

index 4d3b3d60d893..409ef632a24e 100644
@@ -29,6 +29,7 @@
 #include <linux/rcupdate.h>
 #include <linux/pfn.h>
 #include <linux/kmemleak.h>
+#include <linux/kmsan.h>
 #include <linux/atomic.h>
 #include <linux/compiler.h>
 #include <linux/llist.h>

@@ -119,7 +120,8 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end)
 	} while (p4d++, addr = next, addr != end);
 }
 
-static void vunmap_page_range(unsigned long addr, unsigned long end)
+/* Exported for KMSAN, visible in mm/kmsan/kmsan.h only. */
+void __vunmap_page_range(unsigned long addr, unsigned long end)
 {
 	pgd_t *pgd;
 	unsigned long next;

@@ -133,6 +135,13 @@ static void vunmap_page_range(unsigned long addr, unsigned long end)
 		vunmap_p4d_range(pgd, addr, next);
 	} while (pgd++, addr = next, addr != end);
 }
+EXPORT_SYMBOL(__vunmap_page_range);
+
+static void vunmap_page_range(unsigned long addr, unsigned long end)
+{
+	kmsan_vunmap_page_range(addr, end);
+	__vunmap_page_range(addr, end);
+}
 
 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
 		unsigned long end, pgprot_t prot, struct page **pages, int *nr)

@@ -216,8 +225,11 @@ static int vmap_p4d_range(pgd_t *pgd, unsigned long addr,
  * will have pfns corresponding to the "pages" array.
  *
  * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
+ *
+ * This function is exported for use in KMSAN, but is only declared in KMSAN
+ * headers.
  */
-static int vmap_page_range_noflush(unsigned long start, unsigned long end,
+int __vmap_page_range_noflush(unsigned long start, unsigned long end,
 				   pgprot_t prot, struct page **pages)
 {
 	pgd_t *pgd;

@@ -237,6 +249,14 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end,
 
 	return nr;
 }
+EXPORT_SYMBOL(__vmap_page_range_noflush);
+
+static int vmap_page_range_noflush(unsigned long start, unsigned long end,
+				   pgprot_t prot, struct page **pages)
+{
+	kmsan_vmap_page_range_noflush(start, end, prot, pages);
+	return __vmap_page_range_noflush(start, end, prot, pages);
+}
 
 static int vmap_page_range(unsigned long start, unsigned long end,
 			   pgprot_t prot, struct page **pages)

index 0eab79f82ce4..5bdb54d71c2e 100644
@@ -8,6 +8,7 @@
 #include <linux/bitops.h>
 #include <linux/slab.h>
 #include <linux/log2.h>
+#include <linux/kmsan-checks.h>
 #include <linux/usb.h>
 #include <linux/wait.h>
 #include <linux/usb/hcd.h>

@@ -401,6 +402,7 @@ int usb_submit_urb(struct urb *urb, gfp_t mem_flags)
 			URB_SETUP_MAP_SINGLE | URB_SETUP_MAP_LOCAL |
 			URB_DMA_SG_COMBINED);
 	urb->transfer_flags |= (is_out ? URB_DIR_OUT : URB_DIR_IN);
+	kmsan_handle_urb(urb, is_out);
 
 	if (xfertype != USB_ENDPOINT_XFER_CONTROL &&
 			dev->state < USB_STATE_CONFIGURED)

index bcbd59888e67..9bdf732185c3 100644
@@ -60,6 +60,7 @@
 #include <linux/writeback.h>
 #include <linux/shm.h>
 #include <linux/kcov.h>
+#include <linux/kmsan.h>
 #include <linux/random.h>
 #include <linux/rcuwait.h>
 #include <linux/compat.h>

@@ -719,6 +720,7 @@ void __noreturn do_exit(long code)
 
 	profile_task_exit(tsk);
 	kcov_task_exit(tsk);
+	kmsan_task_exit(tsk);
 
 	WARN_ON(blk_needs_flush_plug(tsk));
 

index 2508a4f238a3..22c0d6213eca 100644
@@ -37,6 +37,7 @@
 #include <linux/fdtable.h>
 #include <linux/iocontext.h>
 #include <linux/key.h>
+#include <linux/kmsan.h>
 #include <linux/binfmts.h>
 #include <linux/mman.h>
 #include <linux/mmu_notifier.h>

@@ -934,6 +935,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
 	account_kernel_stack(tsk, 1);
 
 	kcov_task_init(tsk);
+	kmsan_task_create(tsk);
 
 #ifdef CONFIG_FAULT_INJECTION
 	tsk->fail_nth = 0;

index b262f47046ca..33ca743ca8b5 100644
@@ -17,6 +17,7 @@
 #include <linux/unistd.h>
 #include <linux/file.h>
 #include <linux/export.h>
+#include <linux/kmsan.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/freezer.h>

@@ -350,6 +351,7 @@ struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
 		set_cpus_allowed_ptr(task, cpu_all_mask);
 	}
 	kfree(create);
+	kmsan_task_create(task);
 	return task;
 }
 

index 5ab696efca95..3d9701bdf695 100644
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
+#include <linux/kmsan-checks.h>
 #include <linux/sched.h>
 #include <linux/string.h>
 #include <linux/errno.h>

@@ -654,6 +655,7 @@ static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
 	} else {
 		WRITE_ONCE(qdisc->empty, true);
 	}
+	kmsan_check_skb(skb);
 
 	return skb;
 }

index 1ef6f75d92f1..cda33cfabafa 100644
@@ -1915,6 +1915,12 @@ int vprintk_store(int facility, int level,
 	 * prefix which might be passed-in as a parameter.
 	 */
 	text_len = vscnprintf(text, sizeof(textbuf), fmt, args);
+	/*
+	 * If any of vscnprintf() arguments is uninitialized, KMSAN will report
+	 * one or more errors and also probably mark text_len as uninitialized.
+	 * Initialize |text_len| to prevent the errors from spreading further.
+	 */
+	text_len = KMSAN_INIT_VALUE(text_len);
 
 	/* mark and strip a trailing newline */
 	if (text_len && text[text_len-1] == '\n') {

index 9747876980b5..7707b5b65ee0 100644
@@ -279,6 +279,8 @@ __visible inline void syscall_return_slowpath(struct pt_regs *regs)
 }
 
 #ifdef CONFIG_X86_64
+/* Tell KMSAN to not instrument this function and to initialize |regs|. */
+__no_sanitize_memory
 __visible void do_syscall_64(unsigned long nr, struct pt_regs *regs)
 {
 	struct thread_info *ti;

index 187ce59aea28..a6fc1641e286 100644
@@ -14,6 +14,8 @@
 
 DECLARE_PER_CPU(struct pt_regs *, irq_regs);
 
+/* Tell KMSAN to return an initialized struct pt_regs. */
+__no_sanitize_memory
 static inline struct pt_regs *get_irq_regs(void)
 {
 	return __this_cpu_read(irq_regs);

index e2389ce9bf58..098b1a8d6bc4 100644
@@ -196,6 +196,8 @@ struct pt_regs;
 	ALLOW_ERROR_INJECTION(__x64_sys##name, ERRNO);			\
 	static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__));	\
 	static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__));\
+	/* Tell KMSAN to initialize |regs|. */				\
+	__no_sanitize_memory						\
 	asmlinkage long __x64_sys##name(const struct pt_regs *regs)	\
 	{								\
 		return __se_sys##name(SC_X86_64_REGS_TO_ARGS(x,__VA_ARGS__));\

index 28446fa6bf18..e8e55a6661f9 100644
@@ -1118,6 +1118,9 @@ static void local_apic_timer_interrupt(void)
  * [ if a single-CPU system runs an SMP kernel then we call the local
  *   interrupt as well. Thus we cannot inline the local irq ... ]
  */
+
+/* Tell KMSAN to initialize |regs|. */
+__no_sanitize_memory
 __visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);

index 87b97897a881..3d1691f81cad 100644
@@ -150,6 +150,11 @@ static bool in_irq_stack(unsigned long *stack, struct stack_info *info)
 	return true;
 }
 
+/*
+ * This function may touch stale uninitialized values on stack. Do not
+ * instrument it with KMSAN to avoid false positives.
+ */
+__no_sanitize_memory
 int get_stack_info(unsigned long *stack, struct task_struct *task,
 		   struct stack_info *info, unsigned long *visit_mask)
 {

index 506d66830d4d..310bf00a0351 100644
@@ -425,6 +425,11 @@ void compat_start_thread(struct pt_regs *regs, u32 new_ip, u32 new_sp)
  * Kprobes not supported here. Set the probe on schedule instead.
  * Function graph tracer not supported too.
  */
+/*
+ * Avoid touching KMSAN state or reporting anything here, as __switch_to() does
+ * weird things with tasks.
+ */
+__no_sanitize_memory
 __visible __notrace_funcgraph struct task_struct *
 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 {

index 05da6b5b167b..e46e34a4893a 100644
@@ -622,7 +622,11 @@ NOKPROBE_SYMBOL(do_int3);
  * Help handler running on a per-cpu (IST or entry trampoline) stack
  * to switch to the normal thread stack if the interrupted code was in
  * user mode. The actual stack switch is done in entry_64.S
+ *
  */
+
+/* This function switches the registers - don't instrument it with KMSAN. */
+__no_sanitize_memory
 asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs)
 {
 	struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1;

@@ -638,6 +642,11 @@ struct bad_iret_stack {
 };
 
 asmlinkage __visible notrace
+/*
+ * Dark magic happening here, let's not instrument this function.
+ * Also avoid copying any metadata by using raw __memmove().
+ */
+__no_sanitize_memory
 struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s)
 {
 	/*

@@ -652,10 +661,10 @@ struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s)
 		(struct bad_iret_stack *)this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1;
 
 	/* Copy the IRET target to the new stack. */
-	memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);
+	__memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8);
 
 	/* Copy the remainder of the stack from the current stack. */
-	memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));
+	__memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip));
 
 	BUG_ON(!user_mode(&new_stack->regs));
 	return new_stack;

index 90e4b00ace89..c49f729baeed 100644
@@ -476,6 +476,11 @@ void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
 		put_task_struct(task);
 }
 
+/*
+ * Context switch here may lead to KMSAN task state corruption. Disable KMSAN
+ * instrumentation.
+ */
+__no_sanitize_memory
 void wake_up_q(struct wake_q_head *head)
 {
 	struct wake_q_node *node = head->first;

@@ -3181,6 +3186,12 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
  * past. prev == current is still correct but we need to recalculate this_rq
  * because prev may have moved to another CPU.
  */
+
+/*
+ * Context switch here may lead to KMSAN task state corruption. Disable KMSAN
+ * instrumentation.
+ */
+__no_sanitize_memory
 static struct rq *finish_task_switch(struct task_struct *prev)
 	__releases(rq->lock)
 {

@@ -3998,6 +4009,12 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
  *
  * WARNING: must be called with preemption disabled!
  */
+
+/*
+ * Context switch here may lead to KMSAN task state corruption. Disable KMSAN
+ * instrumentation.
+ */
+__no_sanitize_memory
 static void __sched notrace __schedule(bool preempt)
 {
 	struct task_struct *prev, *next;

@@ -6736,6 +6753,11 @@ static inline int preempt_count_equals(int preempt_offset)
 	return (nested == preempt_offset);
 }
 
+/*
+ * This function might be called from code that is not instrumented with KMSAN.
+ * Nevertheless, treat its arguments as initialized.
+ */
+__no_sanitize_memory
 void __might_sleep(const char *file, int line, int preempt_offset)
 {
 	/*

index 7d70e5c78f97..28105c267268 100644
@@ -1,6 +1,7 @@
 #include <linux/gfp.h>
 #include <linux/highmem.h>
 #include <linux/kernel.h>
+#include <linux/kmsan-checks.h>
 #include <linux/mmdebug.h>
 #include <linux/mm_types.h>
 #include <linux/pagemap.h>

@@ -206,6 +207,15 @@ void tlb_flush_mmu(struct mmu_gather *tlb)
 void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
 			unsigned long start, unsigned long end)
 {
+	/*
+	 * struct mmu_gather contains 7 1-bit fields packed into a 32-bit
+	 * unsigned int value. The remaining 25 bits remain uninitialized
+	 * and are never used, but KMSAN updates the origin for them in
+	 * zap_pXX_range() in mm/memory.c, thus creating very long origin
+	 * chains. This is technically correct, but consumes too much memory.
+	 * Unpoisoning the whole structure will prevent creating such chains.
+	 */
+	kmsan_unpoison_shadow(tlb, sizeof(*tlb));
 	tlb->mm = mm;
 
 	/* Is it from 0 to ~0? */

index c8181392f70d..dd4bd8c5d97a 100644
@@ -12,6 +12,7 @@
 #undef CONFIG_PARAVIRT_XXL
 #undef CONFIG_PARAVIRT_SPINLOCKS
 #undef CONFIG_KASAN
+#undef CONFIG_KMSAN
 
 /* cpu_feature_enabled() cannot be used this early */
 #define USE_EARLY_PGTABLE_L5

index 75314c3dbe47..0442e679b307 100644
@@ -11,11 +11,23 @@
    function. */
 
 #define __HAVE_ARCH_MEMCPY 1
+#if defined(CONFIG_KMSAN)
+#undef memcpy
+/* __msan_memcpy() is defined in compiler.h */
+#define memcpy(dst, src, len) __msan_memcpy(dst, src, len)
+#else
 extern void *memcpy(void *to, const void *from, size_t len);
+#endif
 extern void *__memcpy(void *to, const void *from, size_t len);
 
 #define __HAVE_ARCH_MEMSET
+#if defined(CONFIG_KMSAN)
+extern void *__msan_memset(void *s, int c, size_t n);
+#undef memset
+#define memset(dst, c, len) __msan_memset(dst, c, len)
+#else
 void *memset(void *s, int c, size_t n);
+#endif
 void *__memset(void *s, int c, size_t n);
 
 #define __HAVE_ARCH_MEMSET16

@@ -55,7 +67,13 @@ static inline void *memset64(uint64_t *s, uint64_t v, size_t n)
 }
 
 #define __HAVE_ARCH_MEMMOVE
+#if defined(CONFIG_KMSAN)
+#undef memmove
+void *__msan_memmove(void *dest, const void *src, size_t len);
+#define memmove(dst, src, len) __msan_memmove(dst, src, len)
+#else
 void *memmove(void *dest, const void *src, size_t count);
+#endif
 void *__memmove(void *dest, const void *src, size_t count);
 
 int memcmp(const void *cs, const void *ct, size_t count);

@@ -64,7 +82,8 @@ char *strcpy(char *dest, const char *src);
 char *strcat(char *dest, const char *src);
 int strcmp(const char *cs, const char *ct);
 
-#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
+#if (defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)) || \
+	(defined(CONFIG_KMSAN) && !defined(__SANITIZE_MEMORY__))
 
 /*
  * For files that not instrumented (e.g. mm/slub.c) we

@@ -73,7 +92,9 @@ int strcmp(const char *cs, const char *ct);
 
 #undef memcpy
 #define memcpy(dst, src, len) __memcpy(dst, src, len)
+#undef memmove
 #define memmove(dst, src, len) __memmove(dst, src, len)
+#undef memset
 #define memset(s, c, n) __memset(s, c, n)
 
 #ifndef __NO_FORTIFY

index e02579907f2e..79393691533a 100644
@@ -5,7 +5,12 @@
  * implementation files.
  *
  * Copyright 2011 Intel Corporation; author Matt Fleming
+ *
+ *
+ * This file is not linked with KMSAN runtime.
+ * Do not replace memcpy with __memcpy.
  */
+#undef CONFIG_KMSAN
 
 #include <linux/efi.h>
 #include <asm/efi.h>

index eb9af83e4d59..1b5ff4f7eb3c 100644
@@ -6,7 +6,12 @@
  * Copyright (C) 2017 Google, Inc.
  *     Matthew Garrett <[email protected]>
  *     Thiebaud Weksteen <[email protected]>
+ *
+ *
+ * This file is not linked with KMSAN runtime.
+ * Do not replace memcpy with __memcpy.
  */
+#undef CONFIG_KMSAN
 #include <linux/efi.h>
 #include <linux/tpm_eventlog.h>
 #include <asm/efi.h>

index de7942423836..0ed22e909caf 100644
@@ -179,6 +179,13 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
 
 #include <uapi/linux/types.h>
 
+#ifdef CONFIG_KMSAN
+void *__msan_memcpy(void *dst, const void *src, u64 size);
+#define __DO_MEMCPY(res, p, size) __msan_memcpy(res, p, size)
+#else
+#define __DO_MEMCPY(res, p, size) __builtin_memcpy(res, p, size)
+#endif
+
 #define __READ_ONCE_SIZE						\
 ({									\
 	switch (size) {							\

@@ -188,7 +195,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
 	case 8: *(__u64 *)res = *(volatile __u64 *)p; break;		\
 	default:							\
 		barrier();						\
-		__builtin_memcpy((void *)res, (const void *)p, size);	\
+		__DO_MEMCPY((void *)res, (const void *)p, size);	\
 		barrier();						\
 	}								\
 })

index 02894e417565..80d2a8d828fc 100644
@@ -365,6 +365,7 @@ __FORTIFY_INLINE void *memset(void *p, int c, __kernel_size_t size)
 	return __builtin_memset(p, c, size);
 }
 
+#ifndef CONFIG_KMSAN
 __FORTIFY_INLINE void *memcpy(void *p, const void *q, __kernel_size_t size)
 {
 	size_t p_size = __builtin_object_size(p, 0);

@@ -379,6 +380,7 @@ __FORTIFY_INLINE void *memcpy(void *p, const void *q, __kernel_size_t size)
 		fortify_panic(__func__);
 	return __builtin_memcpy(p, q, size);
 }
+#endif
 
 __FORTIFY_INLINE void *memmove(void *p, const void *q, __kernel_size_t size)
 {

index 61d93f062a36..bfb55fdba5df 100644
@@ -6,6 +6,7 @@
  */
 #include <linux/compiler.h>
 #include <linux/kasan-checks.h>
+#include <linux/kmsan-checks.h>
 #include <linux/string.h>
 #include <asm/asm.h>
 #include <asm/page.h>

@@ -174,6 +175,7 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
 			ASM_CALL_CONSTRAINT				\
 		     : "0" (ptr), "i" (sizeof(*(ptr))));		\
 	(x) = (__force __typeof__(*(ptr))) __val_gu;			\
+	kmsan_unpoison_shadow(&(x), sizeof(*(ptr)));			\
 	__builtin_expect(__ret_gu, 0);					\
 })
 

@@ -248,6 +250,7 @@ extern void __put_user_8(void);
 	__chk_user_ptr(ptr);					\
 	might_fault();						\
 	__pu_val = x;						\
+	kmsan_check_memory(&(__pu_val), sizeof(*(ptr)));	\
 	switch (sizeof(*(ptr))) {				\
 	case 1:							\
 		__put_user_x(1, __pu_val, ptr, __ret_pu);	\

@@ -270,7 +273,9 @@ extern void __put_user_8(void);
 
 #define __put_user_size(x, ptr, size, label)				\
 do {									\
+	__typeof__(*(ptr)) __pus_val = x;				\
 	__chk_user_ptr(ptr);						\
+	kmsan_check_memory(&(__pus_val), size);				\
 	switch (size) {							\
 	case 1:								\
 		__put_user_goto(x, ptr, "b", "b", "iq", label);	\

@@ -295,7 +300,9 @@ do {									\
  */
 #define __put_user_size_ex(x, ptr, size)				\
 do {									\
+	__typeof__(*(ptr)) __puse_val = x;				\
 	__chk_user_ptr(ptr);						\
+	kmsan_check_memory(&(__puse_val), size);			\
 	switch (size) {							\
 	case 1:								\
 		__put_user_asm_ex(x, ptr, "b", "b", "iq");		\

@@ -363,6 +370,7 @@ do {									\
 	default:							\
 		(x) = __get_user_bad();					\
 	}								\
+	kmsan_unpoison_shadow(&(x), size);				\
 } while (0)
 
 #define __get_user_asm(x, addr, err, itype, rtype, ltype, errret)	\

@@ -413,6 +421,7 @@ do {									\
 	default:							\
 		(x) = __get_user_bad();					\
 	}								\
+	kmsan_unpoison_shadow(&(x), size);				\
 } while (0)
 
 #define __get_user_asm_ex(x, addr, itype, rtype, ltype)			\

@@ -433,6 +442,7 @@ do {									\
 	__typeof__(ptr) __pu_ptr = (ptr);			\
 	__typeof__(size) __pu_size = (size);			\
 	__uaccess_begin();					\
+	kmsan_check_memory(&(__pu_val), size);			\
 	__put_user_size(__pu_val, __pu_ptr, __pu_size, __pu_label);	\
 	__pu_err = 0;						\
 __pu_label:							\

index a950a22c4890..707531dccf5e 100644
@@ -4,6 +4,7 @@
 
 /* Keep includes the same across arches.  */
 #include <linux/mm.h>
+#include <linux/kmsan-checks.h>
 
 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 0
 

@@ -72,10 +73,14 @@ static inline void flush_cache_vunmap(unsigned long start, unsigned long end)
 
 #define copy_to_user_page(vma, page, vaddr, dst, src, len) \
 	do { \
+		kmsan_check_memory(src, len); \
 		memcpy(dst, src, len); \
 		flush_icache_user_range(vma, page, vaddr, len); \
 	} while (0)
 #define copy_from_user_page(vma, page, vaddr, dst, src, len) \
-	memcpy(dst, src, len)
+	do { \
+		memcpy(dst, src, len); \
+		kmsan_unpoison_shadow(dst, len); \
+	} while (0)
 
 #endif /* __ASM_CACHEFLUSH_H */

index e935318804f8..88b626c3ef2d 100644
@@ -142,7 +142,11 @@ static inline int __access_ok(unsigned long addr, unsigned long size)
 
 static inline int __put_user_fn(size_t size, void __user *ptr, void *x)
 {
-	return unlikely(raw_copy_to_user(ptr, x, size)) ? -EFAULT : 0;
+	int n;
+
+	n = raw_copy_to_user(ptr, x, size);
+	kmsan_copy_to_user(ptr, x, size, n);
+	return unlikely(n) ? -EFAULT : 0;
 }
 
 #define __put_user_fn(sz, u, k)	__put_user_fn(sz, u, k)

@@ -203,7 +207,11 @@ extern int __put_user_bad(void) __attribute__((noreturn));
 #ifndef __get_user_fn
 static inline int __get_user_fn(size_t size, const void __user *ptr, void *x)
 {
-	return unlikely(raw_copy_from_user(x, ptr, size)) ? -EFAULT : 0;
+	int res;
+
+	res = raw_copy_from_user(x, ptr, size);
+	kmsan_unpoison_shadow(x, size - res);
+	return unlikely(res) ? -EFAULT : 0;
 }
 
 #define __get_user_fn(sz, u, k)	__get_user_fn(sz, u, k)

index 67f016010aad..efb3cd554140 100644
@@ -5,6 +5,7 @@
 #include <linux/sched.h>
 #include <linux/thread_info.h>
 #include <linux/kasan-checks.h>
+#include <linux/kmsan-checks.h>
 
 #define uaccess_kernel() segment_eq(get_fs(), KERNEL_DS)
 

@@ -58,18 +59,26 @@
 static __always_inline __must_check unsigned long
 __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
 {
+	unsigned long res;
+
 	kasan_check_write(to, n);
 	check_object_size(to, n, false);
-	return raw_copy_from_user(to, from, n);
+	res = raw_copy_from_user(to, from, n);
+	kmsan_unpoison_shadow(to, n - res);
+	return res;
 }
 
 static __always_inline __must_check unsigned long
 __copy_from_user(void *to, const void __user *from, unsigned long n)
 {
+	unsigned long res;
+
 	might_fault();
 	kasan_check_write(to, n);
 	check_object_size(to, n, false);
-	return raw_copy_from_user(to, from, n);
+	res = raw_copy_from_user(to, from, n);
+	kmsan_unpoison_shadow(to, n - res);
+	return res;
 }
 
 /**

@@ -88,18 +97,26 @@ __copy_from_user(void *to, const void __user *from, unsigned long n)
 static __always_inline __must_check unsigned long
 __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
 {
+	unsigned long res;
+
 	kasan_check_read(from, n);
 	check_object_size(from, n, true);
-	return raw_copy_to_user(to, from, n);
+	res = raw_copy_to_user(to, from, n);
+	kmsan_copy_to_user((const void *)to, from, n, res);
+	return res;
 }
 
 static __always_inline __must_check unsigned long
 __copy_to_user(void __user *to, const void *from, unsigned long n)
 {
+	unsigned long res;
+
 	might_fault();
 	kasan_check_read(from, n);
 	check_object_size(from, n, true);
-	return raw_copy_to_user(to, from, n);
+	res = raw_copy_to_user(to, from, n);
+	kmsan_copy_to_user((const void *)to, from, n, res);
+	return res;
 }
 
 #ifdef INLINE_COPY_FROM_USER

@@ -107,10 +124,12 @@ static inline __must_check unsigned long
 _copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	unsigned long res = n;
+
 	might_fault();
 	if (likely(access_ok(from, n))) {
 		kasan_check_write(to, n);
 		res = raw_copy_from_user(to, from, n);
+		kmsan_unpoison_shadow(to, n - res);
 	}
 	if (unlikely(res))
 		memset(to + (n - res), 0, res);

@@ -125,12 +144,15 @@ _copy_from_user(void *, const void __user *, unsigned long);
 static inline __must_check unsigned long
 _copy_to_user(void __user *to, const void *from, unsigned long n)
 {
+	unsigned long res;
+
 	might_fault();
 	if (access_ok(to, n)) {
 		kasan_check_read(from, n);
-		n = raw_copy_to_user(to, from, n);
+		res = raw_copy_to_user(to, from, n);
+		kmsan_copy_to_user(to, from, n, res);
 	}
-	return n;
+	return res;
 }
 #else
 extern __must_check unsigned long

index fb29c02c6a3c..3262db1ace59 100644
@@ -137,20 +137,26 @@
 
 static int copyout(void __user *to, const void *from, size_t n)
 {
+	int res;
+
 	if (access_ok(to, n)) {
 		kasan_check_read(from, n);
-		n = raw_copy_to_user(to, from, n);
+		res = raw_copy_to_user(to, from, n);
+		kmsan_copy_to_user(to, from, n, res);
 	}
-	return n;
+	return res;
 }
 
 static int copyin(void *to, const void __user *from, size_t n)
 {
+	size_t res;
+
 	if (access_ok(from, n)) {
 		kasan_check_write(to, n);
-		n = raw_copy_from_user(to, from, n);
+		res = raw_copy_from_user(to, from, n);
+		kmsan_unpoison_shadow(to, n - res);
 	}
-	return n;
+	return res;
 }
 
 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,