Add KernelMemorySanitizer infrastructure

Some users (currently only KMSAN) may want to use spare bits in depot_stack_handle_t. Let them do so and provide get_dsh_extra_bits() and set_dsh_extra_bits() to access those bits.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]
---

Change-Id: I23580dbde85908eeda0bdd8f83a8c3882ab3e012

---
 include/linux/stackdepot.h |  8 ++++++++
 lib/stackdepot.c           | 24 +++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

include/linux/stackdepot.h +0 -0

index 24d49c732341a..ac1b5a78d7f65 100644

include/linux/stackdepot.h +4 -0

+#define STACK_DEPOT_EXTRA_BITS 5
 
 depot_stack_handle_t stack_depot_save(unsigned long *entries,
                                      unsigned int nr_entries, gfp_t gfp_flags);

@@ -20,5 +25,8 @@ unsigned int stack_depot_fetch(depot_stack_handle_t handle,
                               unsigned long **entries);
 
 unsigned int filter_irq_stacks(unsigned long *entries, unsigned int nr_entries);
+depot_stack_handle_t set_dsh_extra_bits(depot_stack_handle_t handle,
+                                       unsigned int bits);
+unsigned int get_dsh_extra_bits(depot_stack_handle_t handle);
 
 #endif

lib/stackdepot.c +23 -2

index 2caffc64e4c82..195ce3dc7c37e 100644
@@ -40,8 +40,10 @@
 #define STACK_ALLOC_ALIGN 4
 #define STACK_ALLOC_OFFSET_BITS (STACK_ALLOC_ORDER + PAGE_SHIFT - \
                                        STACK_ALLOC_ALIGN)
+
 #define STACK_ALLOC_INDEX_BITS (DEPOT_STACK_BITS - \
-               STACK_ALLOC_NULL_PROTECTION_BITS - STACK_ALLOC_OFFSET_BITS)
+               STACK_ALLOC_NULL_PROTECTION_BITS - \
+               STACK_ALLOC_OFFSET_BITS - STACK_DEPOT_EXTRA_BITS)
 #define STACK_ALLOC_SLABS_CAP 8192
 #define STACK_ALLOC_MAX_SLABS \
        (((1LL << (STACK_ALLOC_INDEX_BITS)) < STACK_ALLOC_SLABS_CAP) ? \

@@ -54,6 +56,7 @@ union handle_parts {
                u32 slabindex : STACK_ALLOC_INDEX_BITS;
                u32 offset : STACK_ALLOC_OFFSET_BITS;
                u32 valid : STACK_ALLOC_NULL_PROTECTION_BITS;
+               u32 extra : STACK_DEPOT_EXTRA_BITS;
        };
 };
 

@@ -72,6 +75,24 @@ static int next_slab_inited;
 static size_t depot_offset;
 static DEFINE_SPINLOCK(depot_lock);
 
+depot_stack_handle_t set_dsh_extra_bits(depot_stack_handle_t handle,
+                                       u32 bits)
+{
+       union handle_parts parts = { .handle = handle };
+
+       parts.extra = bits & ((1U << STACK_DEPOT_EXTRA_BITS) - 1);
+       return parts.handle;
+}
+EXPORT_SYMBOL_GPL(set_dsh_extra_bits);
+
+u32 get_dsh_extra_bits(depot_stack_handle_t handle)
+{
+       union handle_parts parts = { .handle = handle };
+
+       return parts.extra;
+}
+EXPORT_SYMBOL_GPL(get_dsh_extra_bits);
+
 static bool init_stack_slab(void **prealloc)
 {
        if (!*prealloc)

@@ -136,6 +157,7 @@ static struct stack_record *depot_alloc_stack(unsigned long *entries, int size,
        stack->handle.slabindex = depot_index;
        stack->handle.offset = depot_offset >> STACK_ALLOC_ALIGN;
        stack->handle.valid = 1;
+       stack->handle.extra = 0;
        memcpy(stack->entries, entries, size * sizeof(unsigned long));
        depot_offset += required_size;
 
--

2.25.1.696.g5e7596f4ac-goog

Add Documentation/dev-tools/kmsan.rst and reference it in the dev-tools index.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---
v4:
- address comments by Marco Elver:
- remove contractions
- fix references
- minor fixes

Change-Id: Iac6345065e6804ef811f1124fdf779c67ff1530e

---
 Documentation/dev-tools/index.rst |   1 +
 Documentation/dev-tools/kmsan.rst | 424 ++++++++++++++++++++++++++++++
 2 files changed, 425 insertions(+)
 create mode 100644 Documentation/dev-tools/kmsan.rst

Documentation/dev-tools/index.rst +1 -0

index f7809c7b1ba9e..a3b9579fc810c 100644
@@ -19,6 +19,7 @@ whole; patches welcome!
    kcov
    gcov
    kasan
+   kmsan
    ubsan
    kmemleak
    kcsan

Documentation/dev-tools/kmsan.rst new file +194 -0

new file mode 100644
index 0000000000000..591c4809d46f3
@@ -0,0 +1,424 @@
+=============================
+KernelMemorySanitizer (KMSAN)
+=============================
+
+KMSAN is a dynamic memory error detector aimed at finding uses of uninitialized
+memory.
+It is based on compiler instrumentation, and is quite similar to the userspace
+`MemorySanitizer tool`_.
+
+Example report
+==============
+Here is an example of a real KMSAN report in ``packet_bind_spkt()``::
+
+  ==================================================================
+  BUG: KMSAN: uninit-value in strlen
+  CPU: 0 PID: 1074 Comm: packet Not tainted 4.8.0-rc6+ #1891
+  Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Bochs 01/01/2011
+   0000000000000000 ffff88006b6dfc08 ffffffff82559ae8 ffff88006b6dfb48
+   ffffffff818a7c91 ffffffff85b9c870 0000000000000092 ffffffff85b9c550
+   0000000000000000 0000000000000092 00000000ec400911 0000000000000002
+  Call Trace:
+   [<     inline     >] __dump_stack lib/dump_stack.c:15
+   [<ffffffff82559ae8>] dump_stack+0x238/0x290 lib/dump_stack.c:51
+   [<ffffffff818a6626>] kmsan_report+0x276/0x2e0 mm/kmsan/kmsan.c:1003
+   [<ffffffff818a783b>] __msan_warning+0x5b/0xb0 mm/kmsan/kmsan_instr.c:424
+   [<     inline     >] strlen lib/string.c:484
+   [<ffffffff8259b58d>] strlcpy+0x9d/0x200 lib/string.c:144
+   [<ffffffff84b2eca4>] packet_bind_spkt+0x144/0x230 net/packet/af_packet.c:3132
+   [<ffffffff84242e4d>] SYSC_bind+0x40d/0x5f0 net/socket.c:1370
+   [<ffffffff84242a22>] SyS_bind+0x82/0xa0 net/socket.c:1356
+   [<ffffffff8515991b>] entry_SYSCALL_64_fastpath+0x13/0x8f arch/x86/entry/entry_64.o:?
+  chained origin:
+   [<ffffffff810bb787>] save_stack_trace+0x27/0x50 arch/x86/kernel/stacktrace.c:67
+   [<     inline     >] kmsan_save_stack_with_flags mm/kmsan/kmsan.c:322
+   [<     inline     >] kmsan_save_stack mm/kmsan/kmsan.c:334
+   [<ffffffff818a59f8>] kmsan_internal_chain_origin+0x118/0x1e0 mm/kmsan/kmsan.c:527
+   [<ffffffff818a7773>] __msan_set_alloca_origin4+0xc3/0x130 mm/kmsan/kmsan_instr.c:380
+   [<ffffffff84242b69>] SYSC_bind+0x129/0x5f0 net/socket.c:1356
+   [<ffffffff84242a22>] SyS_bind+0x82/0xa0 net/socket.c:1356
+   [<ffffffff8515991b>] entry_SYSCALL_64_fastpath+0x13/0x8f arch/x86/entry/entry_64.o:?
+  origin description: ----address@SYSC_bind (origin=00000000eb400911)
+  ==================================================================
+
+The report tells that the local variable ``address`` was created uninitialized
+in ``SYSC_bind()`` (the ``bind`` system call implementation). The lower stack
+trace corresponds to the place where this variable was created.
+
+The upper stack shows where the uninit value was used - in ``strlen()``.
+It turned out that the contents of ``address`` were partially copied from the
+userspace, but the buffer was not zero-terminated and contained some trailing
+uninitialized bytes.
+
+``packet_bind_spkt()`` did not check the length of the buffer, but called
+``strlcpy()`` on it, which called ``strlen()``, which started reading the
+buffer byte by byte till it hit the uninitialized memory.
+
+
+
+KMSAN and Clang
+===============
+
+In order for KMSAN to work the kernel must be
+built with Clang, which so far is the only compiler that has KMSAN support.
+The kernel instrumentation pass is based on the userspace
+`MemorySanitizer tool`_. Because of the instrumentation complexity it is
+unlikely that any other compiler will support KMSAN soon.
+
+Right now the instrumentation pass supports x86_64 only.
+
+How to build
+============
+
+In order to build a kernel with KMSAN you will need a fresh Clang (10.0.0+,
+trunk version r365008 or greater). Please refer to `LLVM documentation`_
+for the instructions on how to build Clang::
+
+  export KMSAN_CLANG_PATH=/path/to/clang
+  # Now configure and build the kernel with CONFIG_KMSAN enabled.
+  make CC=$KMSAN_CLANG_PATH
+
+How KMSAN works
+===============
+
+KMSAN shadow memory
+-------------------
+
+KMSAN associates a metadata byte (also called shadow byte) with every byte of
+kernel memory.
+A bit in the shadow byte is set iff the corresponding bit of the kernel memory
+byte is uninitialized.
+Marking the memory uninitialized (i.e. setting its shadow bytes to 0xff) is
+called poisoning, marking it initialized (setting the shadow bytes to 0x00) is
+called unpoisoning.
+
+When a new variable is allocated on the stack, it is poisoned by default by
+instrumentation code inserted by the compiler (unless it is a stack variable
+that is immediately initialized). Any new heap allocation done without
+``__GFP_ZERO`` is also poisoned.
+
+Compiler instrumentation also tracks the shadow values with the help from the
+runtime library in ``mm/kmsan/``.
+
+The shadow value of a basic or compound type is an array of bytes of the same
+length.
+When a constant value is written into memory, that memory is unpoisoned.
+When a value is read from memory, its shadow memory is also obtained and
+propagated into all the operations which use that value. For every instruction
+that takes one or more values the compiler generates code that calculates the
+shadow of the result depending on those values and their shadows.
+
+Example::
+
+  int a = 0xff;
+  int b;
+  int c = a | b;
+
+In this case the shadow of ``a`` is ``0``, shadow of ``b`` is ``0xffffffff``,
+shadow of ``c`` is ``0xffffff00``. This means that the upper three bytes of
+``c`` are uninitialized, while the lower byte is initialized.
+
+
+Origin tracking
+---------------
+
+Every four bytes of kernel memory also have a so-called origin assigned to
+them.
+This origin describes the point in program execution at which the uninitialized
+value was created. Every origin is associated with a creation stack, which lets
+the user figure out what is going on.
+
+When an uninitialized variable is allocated on stack or heap, a new origin
+value is created, and that variable's origin is filled with that value.
+When a value is read from memory, its origin is also read and kept together
+with the shadow. For every instruction that takes one or more values the origin
+of the result is one of the origins corresponding to any of the uninitialized
+inputs.
+If a poisoned value is written into memory, its origin is written to the
+corresponding storage as well.
+
+Example 1::
+
+  int a = 0;
+  int b;
+  int c = a + b;
+
+In this case the origin of ``b`` is generated upon function entry, and is
+stored to the origin of ``c`` right before the addition result is written into
+memory.
+
+Several variables may share the same origin address, if they are stored in the
+same four-byte chunk.
+In this case every write to either variable updates the origin for all of them.
+
+Example 2::
+
+  int combine(short a, short b) {
+    union ret_t {
+      int i;
+      short s[2];
+    } ret;
+    ret.s[0] = a;
+    ret.s[1] = b;
+    return ret.i;
+  }
+
+If ``a`` is initialized and ``b`` is not, the shadow of the result would be
+0xffff0000, and the origin of the result would be the origin of ``b``.
+``ret.s[0]`` would have the same origin, but it will be never used, because
+that variable is initialized.
+
+If both function arguments are uninitialized, only the origin of the second
+argument is preserved.
+
+Origin chaining
+~~~~~~~~~~~~~~~
+To ease debugging, KMSAN creates a new origin for every memory store.
+The new origin references both its creation stack and the previous origin the
+memory location had.
+This may cause increased memory consumption, so we limit the length of origin
+chains in the runtime.
+
+Clang instrumentation API
+-------------------------
+
+Clang instrumentation pass inserts calls to functions defined in
+``mm/kmsan/kmsan_instr.c`` into the kernel code.
+
+Shadow manipulation
+~~~~~~~~~~~~~~~~~~~
+For every memory access the compiler emits a call to a function that returns a
+pair of pointers to the shadow and origin addresses of the given memory::
+
+  typedef struct {
+    void *s, *o;

Documentation/dev-tools/kmsan.rst new file +170 -0

+
+The compiler makes sure that for every loaded value its shadow and origin
+values are read from memory.
+When a value is stored to memory, its shadow and origin are also stored using
+the metadata pointers.
+
+Origin tracking
+~~~~~~~~~~~~~~~
+A special function is used to create a new origin value for a local variable
+and set the origin of that variable to that value::
+
+  void __msan_poison_alloca(u64 address, u64 size, char *descr)
+
+Access to per-task data
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+At the beginning of every instrumented function KMSAN inserts a call to
+``__msan_get_context_state()``::
+
+  kmsan_context_state *__msan_get_context_state(void)
+
+``kmsan_context_state`` is declared in ``include/linux/kmsan.h``::
+
+  struct kmsan_context_s {
+    char param_tls[KMSAN_PARAM_SIZE];
+    char retval_tls[RETVAL_SIZE];
+    char va_arg_tls[KMSAN_PARAM_SIZE];
+    char va_arg_origin_tls[KMSAN_PARAM_SIZE];
+    u64 va_arg_overflow_size_tls;
+    depot_stack_handle_t param_origin_tls[PARAM_ARRAY_SIZE];
+    depot_stack_handle_t retval_origin_tls;
+    depot_stack_handle_t origin_tls;
+  };
+
+This structure is used by KMSAN to pass parameter shadows and origins between
+instrumented functions.
+
+String functions
+~~~~~~~~~~~~~~~~
+
+The compiler replaces calls to ``memcpy()``/``memmove()``/``memset()`` with the
+following functions. These functions are also called when data structures are
+initialized or copied, making sure shadow and origin values are copied alongside
+with the data::
+
+  void *__msan_memcpy(void *dst, void *src, u64 n)
+  void *__msan_memmove(void *dst, void *src, u64 n)
+  void *__msan_memset(void *dst, int c, size_t n)
+
+Error reporting
+~~~~~~~~~~~~~~~
+
+For each pointer dereference and each condition the compiler emits a shadow
+check that calls ``__msan_warning()`` in the case a poisoned value is being
+used::
+
+  void __msan_warning(u32 origin)
+
+``__msan_warning()`` causes KMSAN runtime to print an error report.
+
+Inline assembly instrumentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+KMSAN instruments every inline assembly output with a call to::
+
+  void __msan_instrument_asm_store(u64 addr, u64 size)
+
+, which unpoisons the memory region.
+
+This approach may mask certain errors, but it also helps to avoid a lot of
+false positives in bitwise operations, atomics etc.
+
+Sometimes the pointers passed into inline assembly do not point to valid memory.
+In such cases they are ignored at runtime.
+
+Disabling the instrumentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A function can be marked with ``__no_sanitize_memory``.
+Doing so does not remove KMSAN instrumentation from it, however it makes the
+compiler ignore the uninitialized values coming from the function's inputs,
+and initialize the function's outputs.
+The compiler will not inline functions marked with this attribute into functions
+not marked with it, and vice versa.
+
+It is also possible to disable KMSAN for a single file (e.g. main.o)::
+
+  KMSAN_SANITIZE_main.o := n
+
+or for the whole directory::
+
+  KMSAN_SANITIZE := n
+
+in the Makefile. This comes at a cost however: stack allocations from such files
+and parameters of instrumented functions called from them will have incorrect
+shadow/origin values. As a rule of thumb, avoid using KMSAN_SANITIZE.
+
+Runtime library
+---------------
+The code is located in ``mm/kmsan/``.
+
+Per-task KMSAN state
+~~~~~~~~~~~~~~~~~~~~
+
+Every task_struct has an associated KMSAN task state that holds the KMSAN
+context (see above) and a per-task flag disallowing KMSAN reports::
+
+  struct kmsan_task_state {
+    ...
+    bool allow_reporting;
+    struct kmsan_context_state cstate;
+    ...
+  }
+
+  struct task_struct {
+    ...
+    struct kmsan_task_state kmsan;
+    ...
+  }
+
+
+KMSAN contexts
+~~~~~~~~~~~~~~
+
+When running in a kernel task context, KMSAN uses ``current->kmsan.cstate`` to
+hold the metadata for function parameters and return values.
+
+But in the case the kernel is running in the interrupt, softirq or NMI context,
+where ``current`` is unavailable, KMSAN switches to per-cpu interrupt state::
+
+  DEFINE_PER_CPU(kmsan_context_state[KMSAN_NESTED_CONTEXT_MAX],
+                 kmsan_percpu_cstate);
+
+Metadata allocation
+~~~~~~~~~~~~~~~~~~~
+There are several places in the kernel for which the metadata is stored.
+
+1. Each ``struct page`` instance contains two pointers to its shadow and
+origin pages::
+
+  struct page {
+    ...
+    struct page *shadow, *origin;
+    ...
+  };
+
+Every time a ``struct page`` is allocated, the runtime library allocates two
+additional pages to hold its shadow and origins. This is done by adding hooks
+to ``alloc_pages()``/``free_pages()`` in ``mm/page_alloc.c``.
+To avoid allocating the metadata for non-interesting pages (right now only the
+shadow/origin page themselves and Metadata allocationstackdepot storage) the
+``__GFP_NO_KMSAN_SHADOW`` flag is used.
+
+There is a problem related to this allocation algorithm: when two contiguous
+memory blocks are allocated with two different ``alloc_pages()`` calls, their
+shadow pages may not be contiguous. So, if a memory access crosses the boundary
+of a memory block, accesses to shadow/origin memory may potentially corrupt
+other pages or read incorrect values from them.
+
+As a workaround, we check the access size in
+``__msan_metadata_ptr_for_XXX_YYY()`` and return a pointer to a fake shadow
+region in the case of an error::
+
+  char dummy_load_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+  char dummy_store_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+
+``dummy_load_page`` is zero-initialized, so reads from it always yield zeroes.
+All stores to ``dummy_store_page`` are ignored.
+
+Unfortunately at boot time we need to allocate shadow and origin pages for the
+kernel data (``.data``, ``.bss`` etc.) and percpu memory regions, the size of

Documentation/dev-tools/kmsan.rst new file +40 -1

+
+2. For vmalloc memory and modules, there is a direct mapping between the memory
+range, its shadow and origin. KMSAN lessens the vmalloc area by 3/4, making only
+the first quarter available to ``vmalloc()``. The second quarter of the vmalloc
+area contains shadow memory for the first quarter, the third one holds the
+origins. A small part of the fourth quarter contains shadow and origins for the
+kernel modules. Please refer to ``arch/x86/include/asm/pgtable_64_types.h`` for
+more details.
+
+When an array of pages is mapped into a contiguous virtual memory space, their
+shadow and origin pages are similarly mapped into contiguous regions.
+
+3. For CPU entry area there are separate per-CPU arrays that hold its
+metadata::
+
+  DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_shadow);
+  DEFINE_PER_CPU(char[CPU_ENTRY_AREA_SIZE], cpu_entry_area_origin);
+
+When calculating shadow and origin addresses for a given memory address, the
+runtime checks whether the address belongs to the physical page range, the
+virtual page range or CPU entry area.
+
+Handling ``pt_regs``
+~~~~~~~~~~~~~~~~~~~~
+
+Many functions receive a ``struct pt_regs`` holding the register state at a
+certain point. Registers do not have (easily calculatable) shadow or origin
+associated with them.
+We can assume that the registers are always initialized.
+
+References
+==========
+
+E. Stepanov, K. Serebryany. `MemorySanitizer: fast detector of uninitialized
+memory use in C++
+<https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43308.pdf>`_.
+In Proceedings of CGO 2015.
+
+.. _MemorySanitizer tool: https://clang.llvm.org/docs/MemorySanitizer.html
+.. _LLVM documentation: https://llvm.org/docs/GettingStarted.html
--

2.25.1.696.g5e7596f4ac-goog

Documentation/dev-tools/kmsan.rst line 204

Quoted 3 lines

Documentation/dev-tools/kmsan.rst new file +3 -0

+The function name depends on the memory access size.
+Each such function also checks if the shadow of the memory in the range
+[``addr``, ``addr + n``) is contiguous and reports an error otherwise.

Makes sense to refer to the "Metadata allocation" section here, which explains what happens in case of an error.

Documentation/dev-tools/kmsan.rst line 384

Quoted 5 lines

Documentation/dev-tools/kmsan.rst new file +5 -0

+
+LSB of the ``shadow`` pointer in a ``struct page`` may be set to 1. In this case
+shadow and origin pages are allocated, but KMSAN ignores accesses to them by
+falling back to dummy pages. Allocating the metadata pages is still needed to
+support ``vmap()/vunmap()`` operations on this struct page.

This part is not clear. We allocate shadow for vmap()'ed regions but don't do any initialization checks for that memory?

Nit: some sections have empty lines after the section header, while others don't.

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---
We can't decide what to do here:
- do we need to conditionally define ___GFP_NO_KMSAN_SHADOW depending on CONFIG_KMSAN like LOCKDEP does?
- if KMSAN is defined, and LOCKDEP is not, do we want to "compactify" the GFP bits?

Change-Id: If5d0352fd5711ad103328e2c185eb885e826423a

---
 include/linux/gfp.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

include/linux/gfp.h +3 -1

index be2754841369e..e1ab42b5e9ce2 100644
@@ -44,6 +44,7 @@ struct vm_area_struct;
 #else
 #define ___GFP_NOLOCKDEP	0
 #endif
+#define ___GFP_NO_KMSAN_SHADOW  0x1000000u
 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
 
 /*

@@ -212,12 +213,13 @@ struct vm_area_struct;
 #define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)
 #define __GFP_COMP	((__force gfp_t)___GFP_COMP)
 #define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)
+#define __GFP_NO_KMSAN_SHADOW  ((__force gfp_t)___GFP_NO_KMSAN_SHADOW)
 
 /* Disable lockdep for GFP context tracking */
 #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
 
 /* Room for N __GFP_FOO bits */
-#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP))
+#define __GFP_BITS_SHIFT (25)
 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
 
 /**

Signature · 1 line

2.25.1.696.g5e7596f4ac-goog

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Quoted 5 lines · Michal Hocko

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

Quoted 4 lines

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Great idea, will do!
Guess PF_ flags isn't a scarce resource?

Quoted 12 lines · Alexander Potapenko

Quoted 9 lines · Michal Hocko

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Quoted 2 lines

Great idea, will do!
Guess PF_ flags isn't a scarce resource?

Actually, no, we are out of bits in current->flags already. I could introduce a separate flag into struct task, but that won't work in interrupt contexts - how do you solve that problem for FS/IO allocations?

I would suggest using bits in the section labelled:

        /* Unserialized, strictly 'current' */

Quoted 3 lines

I would suggest using bits in the section labelled:

        /* Unserialized, strictly 'current' */

The main problem is that |current| is unavailable in the interrupt context, so we'll also need to:
- disable interrupts when preparing for a KMSAN internal memory
allocation - sounds costly, huh?
- store the context flag in a per-cpu variable in the case |current|
is unavailable.

Quoted 10 lines · Alexander Potapenko

Quoted 3 lines

I would suggest using bits in the section labelled:

        /* Unserialized, strictly 'current' */

The main problem is that |current| is unavailable in the interrupt context, so we'll also need to:
- disable interrupts when preparing for a KMSAN internal memory
allocation - sounds costly, huh?
- store the context flag in a per-cpu variable in the case |current|
is unavailable.

It's not /unavailable/ ... it's whatever task happens to be running at the time the interrupt is triggered. You can borrow its task_struct.

You'll have to save off the current value of the flag before setting it, just like memalloc_nofs_save() does.
But this does rather call into question whether Michal's advice to use task_struct is good advice to begin with. For memalloc_nofs/noio, it works well this way because allocations in interrupt context are inherently at a more restrictive context than task level. It's not clear to me what this kmsan GFP flag is being used for, and whether allocations that happen in interrupt context should inherit the kmsan setting.

I will have to read these patches more carefully to determine that; I was really just responding to the "where can I find some free bits" part of the question.

Quoted 13 lines · Matthew Wilcox

Quoted 10 lines · Alexander Potapenko

Quoted 3 lines

I would suggest using bits in the section labelled:

        /* Unserialized, strictly 'current' */

The main problem is that |current| is unavailable in the interrupt context, so we'll also need to:
- disable interrupts when preparing for a KMSAN internal memory
allocation - sounds costly, huh?
- store the context flag in a per-cpu variable in the case |current|
is unavailable.

Quoted 2 lines

It's not /unavailable/ ... it's whatever task happens to be running at the time the interrupt is triggered. You can borrow its task_struct.

That didn't come to my mind, interesting!

Quoted 8 lines

You'll have to save off the current value of the flag before setting it, just like memalloc_nofs_save() does.
But this does rather call into question whether Michal's advice to use task_struct is good advice to begin with. For memalloc_nofs/noio, it works well this way because allocations in interrupt context are inherently at a more restrictive context than task level. It's not clear to me what this kmsan GFP flag is being used for, and whether allocations that happen in interrupt context should inherit the kmsan setting.

The idea behind this flag is as follows.
KMSAN must allocate metadata pages for every allocation performed by alloc_pages().
Metadata allocations are also done via alloc_pages(), and for those no further metadata needs to be allocated.
Thus the GFP flag is used to prevent the recursion in alloc_pages().

Quoted 13 lines · Alexander Potapenko

Quoted 12 lines · Alexander Potapenko

Quoted 9 lines · Michal Hocko

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Quoted 7 lines

Quoted 2 lines

Great idea, will do!
Guess PF_ flags isn't a scarce resource?

Actually, no, we are out of bits in current->flags already. I could introduce a separate flag into struct task, but that won't work in interrupt contexts - how do you solve that problem for FS/IO allocations?

I would suggest using bits in the section labelled:

        /* Unserialized, strictly 'current' */

since this doesn't need to be accessed from any other task. Michal, can we move PF_MEMALLOC_NOIO and NOFS to that area too?

Quoted 12 lines · Alexander Potapenko

Quoted 9 lines · Michal Hocko

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Quoted 2 lines

Great idea, will do!
Guess PF_ flags isn't a scarce resource?

Actually, no, we are out of bits in current->flags already. I could introduce a separate flag into struct task, but that won't work in interrupt contexts - how do you solve that problem for FS/IO allocations?

Quoted 13 lines · Alexander Potapenko

Quoted 12 lines · Alexander Potapenko

Quoted 9 lines · Michal Hocko

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Quoted 7 lines

Quoted 2 lines

Great idea, will do!
Guess PF_ flags isn't a scarce resource?

Actually, no, we are out of bits in current->flags already. I could introduce a separate flag into struct task, but that won't work in interrupt contexts - how do you solve that problem for FS/IO allocations?

NOFS/NOIO is not a problem for IRQ context because we never do reclaim from that context.

I was also not aware that there are users from the IRQ context. I thought this would be an internal KMSAN stuff. What would be the IRQ context you call this from?

Anyway, if you cannot go with the task_struct then a per-cpu value should work, right?

* - as mentioned in the cover letter, a lot of problems could've been solved if we pre-allocate the metadata pages at boot time so that for every two contiguous physical pages their metadata pages are also contiguous.
I haven't come up with a good idea about how to implement that. Suggestions are very welcome.

Quoted 5 lines · Michal Hocko

Quoted 2 lines · On Wed 25-03-20 17:12:14, [email protected] wrote:

This flag is to be used by KMSAN runtime to mark that newly created memory pages don't need KMSAN metadata backing them.

Quoted 4 lines

I really dislike an idea of the gfp flag. If you need some form of
exclusion for kmsan allocations then follow the pattern of memalloc_no{fs,io}_{save,restore} History tells us that single usecase gfp flags are too tempting to abuse
and using incorrectly.

Great idea, will do!
Guess PF_ flags isn't a scarce resource?

__no_sanitize_memory is a function attribute that makes KMSAN ignore the uninitialized values coming from the function's inputs, and initialize the function's outputs.

Functions marked with this attribute can't be inlined into functions not marked with it, and vice versa.

__SANITIZE_MEMORY__ is a macro that's defined iff the file is instrumented with KMSAN. This is not the same as CONFIG_KMSAN, which is defined for every file.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>

Cc Vegard Nossum <[email protected]>

Cc Dmitry Vyukov <[email protected]>

Cc Marco Elver <[email protected]>

Cc Andrey Konovalov <[email protected]>

Cc [email protected]

Acked-by Marco Elver <[email protected]>

---

v4:
- dropped an unnecessary comment as requested by Marco Elver

Change-Id: I1f1672652c8392f15f7ca8ac26cd4e71f9cc1e4b

---
 include/linux/compiler-clang.h | 7 +++++++
 include/linux/compiler-gcc.h   | 5 +++++
 2 files changed, 12 insertions(+)

include/linux/compiler-clang.h +7 -0

index 2cb42d8bdedc6..d4f929b4a6705 100644
@@ -33,6 +33,13 @@
 #define __no_sanitize_thread
 #endif
 
+#if __has_feature(memory_sanitizer)
+# define __SANITIZE_MEMORY__
+# define __no_sanitize_memory __attribute__((no_sanitize("kernel-memory")))
+#else
+# define __no_sanitize_memory
+#endif
+
 /*
  * Not all versions of clang implement the the type-generic versions
  * of the builtin overflow checkers. Fortunately, clang implements

include/linux/compiler-gcc.h +5 -1

index cf294faec2f87..1121557252f88 100644
@@ -151,6 +151,11 @@
 #define __no_sanitize_thread
 #endif
 
+/*
+ * GCC doesn't support KMSAN.
+ */
+#define __no_sanitize_memory
+
 #if GCC_VERSION >= 50100
 #define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
 #endif
--

2.25.1.696.g5e7596f4ac-goog

KMSAN is going to use 3/4 of existing vmalloc space to hold the metadata, therefore we lower VMALLOC_END to make sure vmalloc() doesn't allocate past the first 1/4.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---

Change-Id: Iaa5e8e0fc2aa66c956f937f5a1de6e5ef40d57cc

---
 arch/x86/include/asm/pgtable_64_types.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

arch/x86/include/asm/pgtable_64_types.h +1 -0

index 52e5f5f2240d9..586629e204366 100644
@@ -139,7 +139,22 @@ extern unsigned int ptrs_per_p4d;
 # define VMEMMAP_START         __VMEMMAP_BASE_L4
 #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 
+#ifndef CONFIG_KMSAN

arch/x86/include/asm/pgtable_64_types.h +0 -1

 #define MODULES_VADDR          (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
 /* The module sections ends with the start of the fixmap */
--

2.25.1.696.g5e7596f4ac-goog

Quoted 29 lines · On Wed, Mar 25, 2020 at 5:13 PM <[email protected]> wrote:

KMSAN is going to use 3/4 of existing vmalloc space to hold the metadata, therefore we lower VMALLOC_END to make sure vmalloc() doesn't allocate past the first 1/4.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---

Change-Id: Iaa5e8e0fc2aa66c956f937f5a1de6e5ef40d57cc

---
 arch/x86/include/asm/pgtable_64_types.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

arch/x86/include/asm/pgtable_64_types.h +0 -0

index 52e5f5f2240d9..586629e204366 100644
@@ -139,7 +139,22 @@ extern unsigned int ptrs_per_p4d;
 # define VMEMMAP_START         __VMEMMAP_BASE_L4
 #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 

arch/x86/include/asm/pgtable_64_types.h extern unsigned int ptrs_per_p4d; · line 152

Quoted 11 lines

arch/x86/include/asm/pgtable_64_types.h +10 -0

+#ifndef CONFIG_KMSAN
 #define VMALLOC_END            (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1)
+#else
+/*
+ * In KMSAN builds vmalloc area is four times smaller, and the remaining 3/4
+ * are used to keep the metadata for virtual pages.
+ */
+#define VMALLOC_QUARTER_SIZE   ((VMALLOC_SIZE_TB << 40) >> 2)
+#define VMALLOC_END            (VMALLOC_START + VMALLOC_QUARTER_SIZE - 1)
+#define VMALLOC_SHADOW_OFFSET  VMALLOC_QUARTER_SIZE
+#define VMALLOC_ORIGIN_OFFSET  (VMALLOC_QUARTER_SIZE * 2)

"<< 1" instead of "* 2" for consistency (since we're using ">> 2" just above")?

Quoted 4 lines

arch/x86/include/asm/pgtable_64_types.h +4 -0

+#define VMALLOC_META_END       (VMALLOC_END + VMALLOC_ORIGIN_OFFSET)
+#define MODULES_SHADOW_START   (VMALLOC_META_END + 1)
+#define MODULES_ORIGIN_START   (MODULES_SHADOW_START + MODULES_LEN)
+#define MODULES_ORIGIN_END     (MODULES_ORIGIN_START + MODULES_LEN)

arch/x86/include/asm/pgtable_64_types.h extern unsigned int ptrs_per_p4d; · line 157

Quoted 1 line

arch/x86/include/asm/pgtable_64_types.h +1 -0

+#endif

These macros are a bit hard to understand. VMALLOC_SHADOW_OFFSET and VMALLOC_ORIGIN_OFFSET are offsets from VMALLOC_END and denote where shadow and origin areas start? What is stored in (VMALLOC_END, VMALLOC_END + VMALLOC_SHADOW_OFFSET] then? Maybe sorting these constants in some logical order would help, or adding a comment on how exactly those 3/4th of vmalloc space are split.

Quoted 31 lines · Andrey Konovalov

Quoted 29 lines · On Wed, Mar 25, 2020 at 5:13 PM <[email protected]> wrote:

KMSAN is going to use 3/4 of existing vmalloc space to hold the metadata, therefore we lower VMALLOC_END to make sure vmalloc() doesn't allocate past the first 1/4.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---

Change-Id: Iaa5e8e0fc2aa66c956f937f5a1de6e5ef40d57cc

---
 arch/x86/include/asm/pgtable_64_types.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

arch/x86/include/asm/pgtable_64_types.h +0 -0

index 52e5f5f2240d9..586629e204366 100644
@@ -139,7 +139,22 @@ extern unsigned int ptrs_per_p4d;
 # define VMEMMAP_START         __VMEMMAP_BASE_L4
 #endif /* CONFIG_DYNAMIC_MEMORY_LAYOUT */
 

Quoted 13 lines

arch/x86/include/asm/pgtable_64_types.h extern unsigned int ptrs_per_p4d; · line 152

Quoted 11 lines

arch/x86/include/asm/pgtable_64_types.h +10 -0

+#ifndef CONFIG_KMSAN
 #define VMALLOC_END            (VMALLOC_START + (VMALLOC_SIZE_TB << 40) - 1)
+#else
+/*
+ * In KMSAN builds vmalloc area is four times smaller, and the remaining 3/4
+ * are used to keep the metadata for virtual pages.
+ */
+#define VMALLOC_QUARTER_SIZE   ((VMALLOC_SIZE_TB << 40) >> 2)
+#define VMALLOC_END            (VMALLOC_START + VMALLOC_QUARTER_SIZE - 1)
+#define VMALLOC_SHADOW_OFFSET  VMALLOC_QUARTER_SIZE
+#define VMALLOC_ORIGIN_OFFSET  (VMALLOC_QUARTER_SIZE * 2)

"<< 1" instead of "* 2" for consistency (since we're using ">> 2" just above")?

Done, thanks!

Quoted 4 lines

Quoted 4 lines

arch/x86/include/asm/pgtable_64_types.h +4 -0

+#define VMALLOC_META_END       (VMALLOC_END + VMALLOC_ORIGIN_OFFSET)
+#define MODULES_SHADOW_START   (VMALLOC_META_END + 1)
+#define MODULES_ORIGIN_START   (MODULES_SHADOW_START + MODULES_LEN)
+#define MODULES_ORIGIN_END     (MODULES_ORIGIN_START + MODULES_LEN)

Quoted 8 lines

arch/x86/include/asm/pgtable_64_types.h extern unsigned int ptrs_per_p4d; · line 157

Quoted 1 line

arch/x86/include/asm/pgtable_64_types.h +1 -0

+#endif

These macros are a bit hard to understand. VMALLOC_SHADOW_OFFSET and VMALLOC_ORIGIN_OFFSET are offsets from VMALLOC_END and denote where shadow and origin areas start? What is stored in (VMALLOC_END, VMALLOC_END + VMALLOC_SHADOW_OFFSET] then? Maybe sorting these constants in some logical order would help, or adding a comment on how exactly those 3/4th of vmalloc space are split.

Added an extensive comment describing this in v6.

We assume an uninitialized value couldn't come from stackdepot, so we don't track stackdepot allocations with KMSAN.

Signed-off-by Alexander Potapenko <[email protected]>

Cc Andrey Ryabinin <[email protected]>

Cc Jens Axboe <[email protected]>

Cc Andy Lutomirski <[email protected]>

Cc Vegard Nossum <[email protected]>

Cc Dmitry Vyukov <[email protected]>

Cc Marco Elver <[email protected]>

Cc Andrey Konovalov <[email protected]>

Cc Christoph Hellwig <[email protected]>

Cc [email protected]

---

v4:
- set __GFP_NO_KMSAN_SHADOW explicitly for allocations

Change-Id: Ic3ec9b3dff3fff2732d874508a3582fb26ff0b1f

---
 lib/stackdepot.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

lib/stackdepot.c +1 -2

index 195ce3dc7c37e..ba584910ad66b 100644
@@ -297,7 +297,7 @@ depot_stack_handle_t stack_depot_save(unsigned long *entries,
                 */
                alloc_flags &= ~GFP_ZONEMASK;
                alloc_flags &= (GFP_ATOMIC | GFP_KERNEL);
-               alloc_flags |= __GFP_NOWARN;
+               alloc_flags |= (__GFP_NOWARN | __GFP_NO_KMSAN_SHADOW);
                page = alloc_pages(alloc_flags, STACK_ALLOC_ORDER);
                if (page)
                        prealloc = page_address(page);
--

2.25.1.696.g5e7596f4ac-goog

Quoted 4 lines · On Wed, Mar 25, 2020 at 5:13 PM <[email protected]> wrote:

We assume an uninitialized value couldn't come from stackdepot, so we don't track stackdepot allocations with KMSAN.

Quoted 10 lines

Signed-off-by Alexander Potapenko <[email protected]>

Cc Andrey Ryabinin <[email protected]>

Cc Jens Axboe <[email protected]>

Cc Andy Lutomirski <[email protected]>

Cc Vegard Nossum <[email protected]>

Cc Dmitry Vyukov <[email protected]>

Cc Marco Elver <[email protected]>

Cc Andrey Konovalov <[email protected]>

Cc Christoph Hellwig <[email protected]>

Cc [email protected]

Reviewed-by Andrey Konovalov <[email protected]>

READ_ONCE_NOCHECK() is already used by KASAN to ignore memory accesses from e.g. stack unwinders.
Define READ_ONCE_NOCHECK() for KMSAN so that it returns initialized values. This helps defeat false positives from leftover stack contents.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---
v3:
- removed unnecessary #ifdef as requested by Mark Rutland
v4:
- added an #include as requested by Marco Elver

Change-Id: Ib38369ba038ab3b581d8e45b81036c3304fb79cb

---
 include/linux/compiler.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

include/linux/compiler.h +3 -3

index f504edebd5d71..c6c67729729e3 100644
@@ -279,6 +279,7 @@ void __write_once_size(volatile void *p, void *res, int size)
  */
 #include <asm/barrier.h>
 #include <linux/kasan-checks.h>
+#include <linux/kmsan-checks.h>
 
 #define __READ_ONCE(x, check)                                          \
 ({                                                                     \

@@ -294,9 +295,9 @@ void __write_once_size(volatile void *p, void *res, int size)
 
 /*
  * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need
- * to hide memory access from KASAN.
+ * to hide memory access from KASAN or KMSAN.
  */
-#define READ_ONCE_NOCHECK(x) __READ_ONCE(x, 0)
+#define READ_ONCE_NOCHECK(x) KMSAN_INIT_VALUE(__READ_ONCE(x, 0))
 
 static __no_kasan_or_inline
 unsigned long read_word_at_a_time(const void *addr)
--

2.25.1.696.g5e7596f4ac-goog

To avoid false positives, assume that reading from the task stack always produces initialized values.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>

Cc Vegard Nossum <[email protected]>

Cc Dmitry Vyukov <[email protected]>

Cc Marco Elver <[email protected]>

Cc Andrey Konovalov <[email protected]>

Cc [email protected]

Acked-by Marco Elver <[email protected]>

---
v4:
- added an #include as requested by Marco Elver

Change-Id: Ie73e5a41fdc8195699928e65f5cbe0d3d3c9e2fa

---
 arch/x86/include/asm/unwind.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

arch/x86/include/asm/unwind.h +6 -5

index 499578f7e6d7b..82c3bceb9999c 100644
@@ -4,6 +4,7 @@
 
 #include <linux/sched.h>
 #include <linux/ftrace.h>
+#include <linux/kmsan-checks.h>
 #include <asm/ptrace.h>
 #include <asm/stacktrace.h>
 

@@ -100,9 +101,10 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
 #endif
 
 /*
- * This disables KASAN checking when reading a value from another task's stack,
- * since the other task could be running on another CPU and could have poisoned
- * the stack in the meantime.
+ * This disables KASAN/KMSAN checking when reading a value from another task's
+ * stack, since the other task could be running on another CPU and could have
+ * poisoned the stack in the meantime. Frame pointers are uninitialized by
+ * default, so for KMSAN we mark the return value initialized unconditionally.
  */
 #define READ_ONCE_TASK_STACK(task, x)                  \
 ({                                                     \

@@ -111,7 +113,7 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
                val = READ_ONCE(x);                     \
        else                                            \
                val = READ_ONCE_NOCHECK(x);             \
-       val;                                            \
+       KMSAN_INIT_VALUE(val);                          \
 })
 
 static inline bool task_on_another_cpu(struct task_struct *task)
--

2.25.1.696.g5e7596f4ac-goog

Quoted 4 lines · On Wed, Mar 25, 2020 at 5:13 PM <[email protected]> wrote:

To avoid false positives, assume that reading from the task stack always produces initialized values.

Quoted 8 lines

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>

Cc Vegard Nossum <[email protected]>

Cc Dmitry Vyukov <[email protected]>

Cc Marco Elver <[email protected]>

Cc Andrey Konovalov <[email protected]>

Cc [email protected]

Acked-by Marco Elver <[email protected]>

Reviewed-by Andrey Konovalov <[email protected]>

Quoted 35 lines

---
v4:
- added an #include as requested by Marco Elver

Change-Id: Ie73e5a41fdc8195699928e65f5cbe0d3d3c9e2fa

---
 arch/x86/include/asm/unwind.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

arch/x86/include/asm/unwind.h +5 -3

index 499578f7e6d7b..82c3bceb9999c 100644
@@ -4,6 +4,7 @@
 
 #include <linux/sched.h>
 #include <linux/ftrace.h>
+#include <linux/kmsan-checks.h>
 #include <asm/ptrace.h>
 #include <asm/stacktrace.h>
 

@@ -100,9 +101,10 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
 #endif
 
 /*
- * This disables KASAN checking when reading a value from another task's stack,
- * since the other task could be running on another CPU and could have poisoned
- * the stack in the meantime.
+ * This disables KASAN/KMSAN checking when reading a value from another task's
+ * stack, since the other task could be running on another CPU and could have
+ * poisoned the stack in the meantime. Frame pointers are uninitialized by
+ * default, so for KMSAN we mark the return value initialized unconditionally.
  */

arch/x86/include/asm/unwind.h void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, · line 116

Quoted 8 lines

arch/x86/include/asm/unwind.h +1 -1

 #define READ_ONCE_TASK_STACK(task, x)                  \
 ({                                                     \

@@ -111,7 +113,7 @@ void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size,
                val = READ_ONCE(x);                     \
        else                                            \
                val = READ_ONCE_NOCHECK(x);             \
-       val;                                            \
+       KMSAN_INIT_VALUE(val);                          \

Quoted 6 lines

arch/x86/include/asm/unwind.h +0 -1

 })
 
 static inline bool task_on_another_cpu(struct task_struct *task)
--

2.25.1.696.g5e7596f4ac-goog

KMSAN assumes shadow and origin pages for every allocated page are accessible. For pages in vmalloc region those metadata pages reside in [VMALLOC_END, VMALLOC_META_END), therefore we must sync a bigger memory region.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---

Change-Id: I0d54855489870ef1180b37fe2120b601da464bf7

---
 arch/x86/mm/fault.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

arch/x86/mm/fault.c +20 -1

index a51df516b87bf..d22e373fa2124 100644
@@ -331,11 +331,21 @@ static void dump_pagetable(unsigned long address)
 
 void vmalloc_sync_mappings(void)
 {
+#ifndef CONFIG_KMSAN
        /*
         * 64-bit mappings might allocate new p4d/pud pages
         * that need to be propagated to all tasks' PGDs.
         */
        sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_END);
+#else
+       /*
+        * For KMSAN, make sure metadata pages for vmalloc area and modules are
+        * also synced.
+        */
+       sync_global_pgds(VMALLOC_START & PGDIR_MASK, VMALLOC_META_END);
+       sync_global_pgds(MODULES_SHADOW_START & PGDIR_MASK,
+               MODULES_ORIGIN_END);
+#endif
 }
 
 void vmalloc_sync_unmappings(void)

@@ -360,7 +370,17 @@ static noinline int vmalloc_fault(unsigned long address)
        pte_t *pte;
 
        /* Make sure we are in vmalloc area: */
+#ifdef CONFIG_KMSAN
+       /*
+        * For KMSAN, make sure metadata pages for vmalloc area and modules are
+        * also synced.
+        */
+       if (!(address >= VMALLOC_START && address < VMALLOC_META_END) &&
+               !(address >= MODULES_SHADOW_START &&
+                 address < MODULES_ORIGIN_END))
+#else
        if (!(address >= VMALLOC_START && address < VMALLOC_END))
+#endif
                return -1;
 
        /*
--

2.25.1.696.g5e7596f4ac-goog

The initial commit adds several tests that trigger KMSAN warnings in simple cases.
To use, build the kernel with CONFIG_TEST_KMSAN and do insmod test_kmsan.ko

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]
---
v2:
- added printk_test()
v4:
- test_kmsan: don't report -Wuninitialized warnings in the test
- test_kmsan.c: addressed comments by Andrey Konovalov

Change-Id: I287e86ae83a82b770f2baa46e5bbdce1dfa65195

---
 lib/Makefile     |   2 +
 lib/test_kmsan.c | 229 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 231 insertions(+)
 create mode 100644 lib/test_kmsan.c

lib/Makefile +2 -0

index ab68a86743607..d8058c5c05826 100644
@@ -68,6 +68,8 @@ CFLAGS_test_kasan.o += $(call cc-disable-warning, vla)
 obj-$(CONFIG_TEST_UBSAN) += test_ubsan.o
 CFLAGS_test_ubsan.o += $(call cc-disable-warning, vla)
 UBSAN_SANITIZE_test_ubsan.o := y
+obj-$(CONFIG_TEST_KMSAN) += test_kmsan.o
+CFLAGS_test_kmsan.o += $(call cc-disable-warning, uninitialized)
 obj-$(CONFIG_TEST_KSTRTOX) += test-kstrtox.o
 obj-$(CONFIG_TEST_LIST_SORT) += test_list_sort.o
 obj-$(CONFIG_TEST_MIN_HEAP) += test_min_heap.o

lib/test_kmsan.c new file +24 -0

new file mode 100644
index 0000000000000..f1780ed0cd315
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Module for testing KMSAN.
+ *
+ * Copyright (C) 2017-2019 Google LLC
+ * Author: Alexander Potapenko <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+
+/*
+ * Tests below use noinline and volatile to work around compiler optimizations
+ * that may mask KMSAN bugs.
+ */
+#define pr_fmt(fmt) "kmsan test: %s : " fmt, __func__
+
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/printk.h>
+#include <linux/slab.h>
+#include <linux/kmsan-checks.h>

lib/test_kmsan.c new file +182 -1

+}
+noinline void init_kmalloc_test(void)
+{
+       int *ptr;
+
+       pr_info("-----------------------------\n");
+       pr_info("initialized kmalloc test (no reports)\n");
+       ptr = kmalloc(sizeof(int), GFP_KERNEL);
+       memset(ptr, 0, sizeof(int));
+       pr_info("kmalloc returned %p\n", ptr);
+       CHECK(*ptr);
+}
+
+noinline void init_kzalloc_test(void)
+{
+       int *ptr;
+
+       pr_info("-----------------------------\n");
+       pr_info("initialized kzalloc test (no reports)\n");
+       ptr = kzalloc(sizeof(int), GFP_KERNEL);
+       pr_info("kzalloc returned %p\n", ptr);
+       CHECK(*ptr);
+}
+
+noinline void uninit_multiple_args_test(void)
+{
+       volatile int a;
+       volatile char b = 3, c;
+
+       pr_info("-----------------------------\n");
+       pr_info("uninitialized local passed to fn (UMR report)\n");
+       CHECK(signed_sum3(a, b, c));
+}
+
+noinline void uninit_stack_var_test(void)
+{
+       int cond;
+
+       pr_info("-----------------------------\n");
+       pr_info("uninitialized stack variable (UMR report)\n");
+       CHECK(cond);
+}
+
+noinline void init_stack_var_test(void)
+{
+       volatile int cond = 1;
+
+       pr_info("-----------------------------\n");
+       pr_info("initialized stack variable (no reports)\n");
+       CHECK(cond);
+}
+
+noinline void two_param_fn_2(int arg1, int arg2)
+{
+       CHECK(arg1);
+       CHECK(arg2);
+}
+
+noinline void one_param_fn(int arg)
+{
+       two_param_fn_2(arg, arg);
+       CHECK(arg);
+}
+
+noinline void two_param_fn(int arg1, int arg2)
+{
+       int init = 0;
+
+       one_param_fn(init);
+       CHECK(arg1);
+       CHECK(arg2);
+}
+
+noinline void params_test(void)
+{
+       volatile int uninit, init = 1;
+
+       pr_info("-----------------------------\n");
+       pr_info("uninit passed through a function parameter (UMR report)\n");
+       two_param_fn(uninit, init);
+}
+
+noinline void do_uninit_local_array(char *array, int start, int stop)
+{
+       int i;
+       volatile char uninit;
+
+       for (i = start; i < stop; i++)
+               array[i] = uninit;
+}
+
+noinline void uninit_kmsan_check_memory_test(void)
+{
+       volatile char local_array[8];
+
+       pr_info("-----------------------------\n");
+       pr_info("kmsan_check_memory() called on uninit local (UMR report)\n");
+       do_uninit_local_array((char *)local_array, 5, 7);
+
+       kmsan_check_memory((char *)local_array, 8);
+}
+
+noinline void init_kmsan_vmap_vunmap_test(void)
+{
+       const int npages = 2;
+       struct page *pages[npages];
+       void *vbuf;
+       int i;
+
+       pr_info("-----------------------------\n");
+       pr_info("pages initialized via vmap (no reports)\n");
+
+       for (i = 0; i < npages; i++)
+               pages[i] = alloc_page(GFP_KERNEL);
+       vbuf = vmap(pages, npages, VM_MAP, PAGE_KERNEL);
+       memset(vbuf, 0xfe, npages * PAGE_SIZE);
+       for (i = 0; i < npages; i++)
+               kmsan_check_memory(page_address(pages[i]), PAGE_SIZE);
+
+       if (vbuf)
+               vunmap(vbuf);
+       for (i = 0; i < npages; i++)
+               if (pages[i])
+                       __free_page(pages[i]);
+}
+
+noinline void init_vmalloc_test(void)
+{
+       char *buf;
+       int npages = 8, i;
+
+       pr_info("-----------------------------\n");
+       pr_info("pages initialized via vmap (no reports)\n");
+       buf = vmalloc(PAGE_SIZE * npages);
+       buf[0] = 1;
+       memset(buf, 0xfe, PAGE_SIZE * npages);
+       CHECK(buf[0]);
+       for (i = 0; i < npages; i++)
+               kmsan_check_memory(&buf[PAGE_SIZE * i], PAGE_SIZE);
+       vfree(buf);
+}
+
+noinline void uaf_test(void)
+{
+       volatile int *var;
+
+       pr_info("-----------------------------\n");
+       pr_info("use-after-free in kmalloc-ed buffer (UMR report)\n");
+       var = kmalloc(80, GFP_KERNEL);
+       var[3] = 0xfeedface;
+       kfree((int *)var);
+       CHECK(var[3]);
+}
+
+noinline void printk_test(void)
+{
+       volatile int uninit;
+
+       pr_info("-----------------------------\n");
+       pr_info("uninit local passed to pr_info() (UMR report)\n");
+       pr_info("%px contains %d\n", &uninit, uninit);
+}
+
+static noinline int __init kmsan_tests_init(void)
+{
+       uninit_kmalloc_test();
+       init_kmalloc_test();
+       init_kzalloc_test();
+       uninit_multiple_args_test();
+       uninit_stack_var_test();
+       init_stack_var_test();
+       params_test();
+       uninit_kmsan_check_memory_test();
+       init_kmsan_vmap_vunmap_test();
+       init_vmalloc_test();
+       uaf_test();
+       printk_test();
+       return -EAGAIN;
+}
+
+module_init(kmsan_tests_init);
+MODULE_LICENSE("GPL");
--

2.25.1.696.g5e7596f4ac-goog

KMSAN is unable to understand when initialized values come from assembly. Disable accelerated configs in KMSAN builds to prevent false positive reports.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Herbert Xu <[email protected]>
Cc: "David S. Miller" <[email protected]>
Cc: Eric Biggers <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: [email protected]

---

v4:
- shorten comments as requested by Marco Elver

v5:
- move the 'depends' directives together, added missing configs as requested by Eric Biggers

Change-Id: Iddc71a2a27360e036d719c0940ebf15553cf8de8

---
 crypto/Kconfig | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

crypto/Kconfig +30 -1

index c24a47406f8f5..5035e8b2b033f 100644
@@ -268,6 +268,7 @@ config CRYPTO_CURVE25519
 config CRYPTO_CURVE25519_X86
        tristate "x86_64 accelerated Curve25519 scalar multiplication library"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_LIB_CURVE25519_GENERIC
        select CRYPTO_ARCH_HAVE_LIB_CURVE25519
 

@@ -317,11 +318,13 @@ config CRYPTO_AEGIS128_SIMD
        bool "Support SIMD acceleration for AEGIS-128"
        depends on CRYPTO_AEGIS128 && ((ARM || ARM64) && KERNEL_MODE_NEON)
        depends on !ARM || CC_IS_CLANG || GCC_VERSION >= 40800
+       depends on !KMSAN # avoid false positives from assembly
        default y
 
 config CRYPTO_AEGIS128_AESNI_SSE2
        tristate "AEGIS-128 AEAD algorithm (x86_64 AESNI+SSE2 implementation)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_AEAD
        select CRYPTO_SIMD
        help

@@ -458,6 +461,7 @@ config CRYPTO_NHPOLY1305
 config CRYPTO_NHPOLY1305_SSE2
        tristate "NHPoly1305 hash function (x86_64 SSE2 implementation)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_NHPOLY1305
        help
          SSE2 optimized implementation of the hash function used by the

@@ -466,6 +470,7 @@ config CRYPTO_NHPOLY1305_SSE2
 config CRYPTO_NHPOLY1305_AVX2
        tristate "NHPoly1305 hash function (x86_64 AVX2 implementation)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_NHPOLY1305
        help
          AVX2 optimized implementation of the hash function used by the

@@ -579,6 +584,7 @@ config CRYPTO_CRC32C
 config CRYPTO_CRC32C_INTEL
        tristate "CRC32c INTEL hardware acceleration"
        depends on X86
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_HASH
        help
          In Intel processor with SSE4.2 supported, the processor will

@@ -619,6 +625,7 @@ config CRYPTO_CRC32
 config CRYPTO_CRC32_PCLMUL
        tristate "CRC32 PCLMULQDQ hardware acceleration"
        depends on X86
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_HASH
        select CRC32
        help

@@ -684,6 +691,7 @@ config CRYPTO_BLAKE2S
 config CRYPTO_BLAKE2S_X86
        tristate "BLAKE2s digest algorithm (x86 accelerated version)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_LIB_BLAKE2S_GENERIC
        select CRYPTO_ARCH_HAVE_LIB_BLAKE2S
 

@@ -698,6 +706,7 @@ config CRYPTO_CRCT10DIF
 config CRYPTO_CRCT10DIF_PCLMUL
        tristate "CRCT10DIF PCLMULQDQ hardware acceleration"
        depends on X86 && 64BIT && CRC_T10DIF
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_HASH
        help
          For x86_64 processors with SSE4.2 and PCLMULQDQ supported,

@@ -745,6 +754,7 @@ config CRYPTO_POLY1305
 config CRYPTO_POLY1305_X86_64
        tristate "Poly1305 authenticator algorithm (x86_64/SSE2/AVX2)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_LIB_POLY1305_GENERIC
        select CRYPTO_ARCH_HAVE_LIB_POLY1305
        help

@@ -870,6 +880,7 @@ config CRYPTO_SHA1
 config CRYPTO_SHA1_SSSE3
        tristate "SHA1 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SHA1
        select CRYPTO_HASH
        help

@@ -881,6 +892,7 @@ config CRYPTO_SHA1_SSSE3
 config CRYPTO_SHA256_SSSE3
        tristate "SHA256 digest algorithm (SSSE3/AVX/AVX2/SHA-NI)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SHA256
        select CRYPTO_HASH
        help

@@ -893,6 +905,7 @@ config CRYPTO_SHA256_SSSE3
 config CRYPTO_SHA512_SSSE3
        tristate "SHA512 digest algorithm (SSSE3/AVX/AVX2)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SHA512
        select CRYPTO_HASH
        help

@@ -1064,6 +1077,7 @@ config CRYPTO_WP512
 config CRYPTO_GHASH_CLMUL_NI_INTEL
        tristate "GHASH hash function (CLMUL-NI accelerated)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_CRYPTD
        help
          This is the x86_64 CLMUL-NI accelerated implementation of

@@ -1114,6 +1128,7 @@ config CRYPTO_AES_TI
 config CRYPTO_AES_NI_INTEL
        tristate "AES cipher algorithms (AES-NI)"
        depends on X86
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_AEAD
        select CRYPTO_LIB_AES
        select CRYPTO_ALGAPI

@@ -1237,6 +1252,7 @@ config CRYPTO_BLOWFISH_COMMON
 config CRYPTO_BLOWFISH_X86_64
        tristate "Blowfish cipher algorithm (x86_64)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_BLOWFISH_COMMON
        help

@@ -1268,6 +1284,7 @@ config CRYPTO_CAMELLIA_X86_64
        tristate "Camellia cipher algorithm (x86_64)"
        depends on X86 && 64BIT
        depends on CRYPTO
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_GLUE_HELPER_X86
        help

@@ -1285,6 +1302,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX_X86_64
        tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX)"
        depends on X86 && 64BIT
        depends on CRYPTO
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_CAMELLIA_X86_64
        select CRYPTO_GLUE_HELPER_X86

@@ -1305,6 +1323,7 @@ config CRYPTO_CAMELLIA_AESNI_AVX2_X86_64
        tristate "Camellia cipher algorithm (x86_64/AES-NI/AVX2)"
        depends on X86 && 64BIT
        depends on CRYPTO
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_CAMELLIA_AESNI_AVX_X86_64
        help
          Camellia cipher algorithm module (x86_64/AES-NI/AVX2).

@@ -1351,6 +1370,7 @@ config CRYPTO_CAST5
 config CRYPTO_CAST5_AVX_X86_64
        tristate "CAST5 (CAST-128) cipher algorithm (x86_64/AVX)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_CAST5
        select CRYPTO_CAST_COMMON

@@ -1373,6 +1393,7 @@ config CRYPTO_CAST6
 config CRYPTO_CAST6_AVX_X86_64
        tristate "CAST6 (CAST-256) cipher algorithm (x86_64/AVX)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_CAST6
        select CRYPTO_CAST_COMMON

@@ -1406,6 +1427,7 @@ config CRYPTO_DES_SPARC64
 config CRYPTO_DES3_EDE_X86_64
        tristate "Triple DES EDE cipher algorithm (x86-64)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_LIB_DES
        help

@@ -1473,6 +1495,7 @@ config CRYPTO_CHACHA20
 config CRYPTO_CHACHA20_X86_64
        tristate "ChaCha stream cipher algorithms (x86_64/SSSE3/AVX2/AVX-512VL)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_LIB_CHACHA_GENERIC
        select CRYPTO_ARCH_HAVE_LIB_CHACHA

@@ -1516,6 +1539,7 @@ config CRYPTO_SERPENT
 config CRYPTO_SERPENT_SSE2_X86_64
        tristate "Serpent cipher algorithm (x86_64/SSE2)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_GLUE_HELPER_X86
        select CRYPTO_SERPENT

@@ -1535,6 +1559,7 @@ config CRYPTO_SERPENT_SSE2_X86_64
 config CRYPTO_SERPENT_SSE2_586
        tristate "Serpent cipher algorithm (i586/SSE2)"
        depends on X86 && !64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_GLUE_HELPER_X86
        select CRYPTO_SERPENT

@@ -1554,6 +1579,7 @@ config CRYPTO_SERPENT_SSE2_586
 config CRYPTO_SERPENT_AVX_X86_64
        tristate "Serpent cipher algorithm (x86_64/AVX)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_GLUE_HELPER_X86
        select CRYPTO_SERPENT

@@ -1574,6 +1600,7 @@ config CRYPTO_SERPENT_AVX_X86_64
 config CRYPTO_SERPENT_AVX2_X86_64
        tristate "Serpent cipher algorithm (x86_64/AVX2)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SERPENT_AVX_X86_64
        help
          Serpent cipher algorithm, by Anderson, Biham & Knudsen.

@@ -1669,6 +1696,7 @@ config CRYPTO_TWOFISH_586
 config CRYPTO_TWOFISH_X86_64
        tristate "Twofish cipher algorithm (x86_64)"
        depends on (X86 || UML_X86) && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_ALGAPI
        select CRYPTO_TWOFISH_COMMON
        help

@@ -1685,6 +1713,7 @@ config CRYPTO_TWOFISH_X86_64
 config CRYPTO_TWOFISH_X86_64_3WAY
        tristate "Twofish cipher algorithm (x86_64, 3-way parallel)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_TWOFISH_COMMON
        select CRYPTO_TWOFISH_X86_64

@@ -1706,6 +1735,7 @@ config CRYPTO_TWOFISH_X86_64_3WAY
 config CRYPTO_TWOFISH_AVX_X86_64
        tristate "Twofish cipher algorithm (x86_64/AVX)"
        depends on X86 && 64BIT
+       depends on !KMSAN # avoid false positives from assembly
        select CRYPTO_SKCIPHER
        select CRYPTO_GLUE_HELPER_X86
        select CRYPTO_SIMD
--

2.25.1.696.g5e7596f4ac-goog

KMSAN doesn't currently support UNWINDER_ORC, causing the kernel to freeze at boot time.
See http://github.com/google/kmsan/issues/48.

Signed-off-by Alexander Potapenko <[email protected]>

Cc Qian Cai <[email protected]>

Cc Christoph Hellwig <[email protected]>

Cc Herbert Xu <[email protected]>

Cc Harry Wentland <[email protected]>

Cc Vegard Nossum <[email protected]>

Cc Dmitry Vyukov <[email protected]>

Cc Marco Elver <[email protected]>

Cc Andrey Konovalov <[email protected]>

Cc [email protected]

---
This patch is part of "kmsan: Kconfig changes to disable options incompatible with KMSAN", which was split into smaller pieces.

Change-Id: I9cb6ebbaeb9a38e9e1d015c68ab77d40420a7ad0

---
 arch/x86/Kconfig.debug | 3 +++
 1 file changed, 3 insertions(+)

arch/x86/Kconfig.debug +0 -0

index 2e74690b028a5..ad71eb2a416ec 100644

arch/x86/Kconfig.debug +0 -1

        select STACK_VALIDATION
        ---help---
          This option enables the ORC (Oops Rewind Capability) unwinder for
--

2.25.1.696.g5e7596f4ac-goog

Add assembly helpers to entry_64.S that invoke hooks from kmsan_entry.c and notify KMSAN about interrupts.
Also call these hooks from kernel/softirq.c
This is needed to switch between several KMSAN contexts holding function parameter metadata.

Signed-off-by Alexander Potapenko <[email protected]>

To: Alexander Potapenko <[email protected]>
Cc: Jens Axboe <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Vegard Nossum <[email protected]>
Cc: Dmitry Vyukov <[email protected]>
Cc: Marco Elver <[email protected]>
Cc: Andrey Konovalov <[email protected]>
Cc: Christoph Hellwig <[email protected]>
Cc: [email protected]

---

v4:
- moved softirq changes to this patch

Change-Id: I3037d51672fe69d09e588b27adb2d9fdc6ad3a7d

---
 arch/x86/entry/entry_64.S | 16 ++++++++++++++++
 kernel/softirq.c          |  5 +++++
 2 files changed, 21 insertions(+)

arch/x86/entry/entry_64.S +16 -0

index 0e9504fabe526..03f5a32b0af4d 100644
@@ -35,6 +35,7 @@
 #include <asm/asm.h>
 #include <asm/smap.h>
 #include <asm/pgtable_types.h>
+#include <asm/kmsan.h>
 #include <asm/export.h>
 #include <asm/frame.h>
 #include <asm/nospec-branch.h>

@@ -575,6 +576,7 @@ SYM_CODE_START(interrupt_entry)
 
 1:
        ENTER_IRQ_STACK old_rsp=%rdi save_ret=1
+       KMSAN_INTERRUPT_ENTER
        /* We entered an interrupt context - irqs are off: */
        TRACE_IRQS_OFF
 

@@ -604,12 +606,14 @@ SYM_CODE_START_LOCAL(common_interrupt)
        addq    $-0x80, (%rsp)                  /* Adjust vector to [-256, -1] range */
        call    interrupt_entry
        UNWIND_HINT_REGS indirect=1
+       KMSAN_UNPOISON_PT_REGS
        call    do_IRQ  /* rdi points to pt_regs */
        /* 0(%rsp): old RSP */
 ret_from_intr:
        DISABLE_INTERRUPTS(CLBR_ANY)
        TRACE_IRQS_OFF
 
+       KMSAN_INTERRUPT_EXIT
        LEAVE_IRQ_STACK
 
        testb   $3, CS(%rsp)

@@ -801,6 +805,7 @@ SYM_CODE_START(\sym)
 .Lcommon_\sym:
        call    interrupt_entry
        UNWIND_HINT_REGS indirect=1
+       KMSAN_UNPOISON_PT_REGS
        call    \do_sym /* rdi points to pt_regs */
        jmp     ret_from_intr
 SYM_CODE_END(\sym)

@@ -908,15 +913,18 @@ apicinterrupt IRQ_WORK_VECTOR                     irq_work_interrupt              smp_irq_work_interrupt
 
        .if \shift_ist != -1
        subq    $\ist_offset, CPU_TSS_IST(\shift_ist)
+       KMSAN_IST_ENTER(\shift_ist)
        .endif
 
        .if \read_cr2
        movq    %r12, %rdx                      /* Move CR2 into 3rd argument */
        .endif
 
+       KMSAN_UNPOISON_PT_REGS
        call    \do_sym
 
        .if \shift_ist != -1
+       KMSAN_IST_EXIT(\shift_ist)
        addq    $\ist_offset, CPU_TSS_IST(\shift_ist)
        .endif
 

@@ -1079,7 +1087,9 @@ SYM_FUNC_START(do_softirq_own_stack)
        pushq   %rbp
        mov     %rsp, %rbp
        ENTER_IRQ_STACK regs=0 old_rsp=%r11
+       KMSAN_SOFTIRQ_ENTER
        call    __do_softirq
+       KMSAN_SOFTIRQ_EXIT
        LEAVE_IRQ_STACK regs=0
        leaveq
        ret

@@ -1466,9 +1476,12 @@ SYM_CODE_START(nmi)
         * done with the NMI stack.
         */
 
+       KMSAN_NMI_ENTER
        movq    %rsp, %rdi
        movq    $-1, %rsi
+       KMSAN_UNPOISON_PT_REGS
        call    do_nmi
+       KMSAN_NMI_EXIT
 
        /*
         * Return back to user mode.  We must *not* do the normal exit

@@ -1678,10 +1691,13 @@ end_repeat_nmi:
        call    paranoid_entry
        UNWIND_HINT_REGS
 
+       KMSAN_NMI_ENTER
        /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
        movq    %rsp, %rdi
        movq    $-1, %rsi
+       KMSAN_UNPOISON_PT_REGS
        call    do_nmi
+       KMSAN_NMI_EXIT
 
        /* Always restore stashed CR3 value (see paranoid_entry) */
        RESTORE_CR3 scratch_reg=%r15 save_reg=%r14

kernel/softirq.c +5 -1

index 0427a86743a46..98c5f4062cbfe 100644
@@ -11,6 +11,7 @@
 
 #include <linux/export.h>
 #include <linux/kernel_stat.h>
+#include <linux/kmsan.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/mm.h>

@@ -370,7 +371,9 @@ static inline void invoke_softirq(void)
                 * it is the irq stack, because it should be near empty
                 * at this stage.
                 */
+               kmsan_context_enter();
                __do_softirq();
+               kmsan_context_exit();
 #else
                /*
                 * Otherwise, irq_exit() is called on the task stack that can

@@ -600,7 +603,9 @@ static void run_ksoftirqd(unsigned int cpu)
                 * We can safely run softirq on inline stack, as we are not deep
                 * in the task stack here.
                 */
+               kmsan_context_enter();
                __do_softirq();
+               kmsan_context_exit();
                local_irq_enable();
                cond_resched();
                return;
--

2.25.1.696.g5e7596f4ac-goog