unwind, perf: sframe user space unwinding, deferred perf callchains

Use ARCH_INIT_USER_FP_FRAME to describe how frame pointers are unwound on x86, and enable HAVE_USER_UNWIND accordinlgy so the user unwind interfaces can be used.

Signed-off-by Josh Poimboeuf <[email protected]>

---
 arch/x86/Kconfig                   |  1 +
 arch/x86/include/asm/user_unwind.h | 11 +++++++++++
 2 files changed, 12 insertions(+)
 create mode 100644 arch/x86/include/asm/user_unwind.h

arch/x86/Kconfig +1 -0

index 007bab9f2a0e..266edff59058 100644
@@ -286,6 +286,7 @@ config X86
 	select HAVE_UACCESS_VALIDATION		if HAVE_OBJTOOL
 	select HAVE_UNSTABLE_SCHED_CLOCK
 	select HAVE_USER_RETURN_NOTIFIER
+	select HAVE_USER_UNWIND
 	select HAVE_GENERIC_VDSO
 	select VDSO_GETRANDOM			if X86_64
 	select HOTPLUG_PARALLEL			if SMP && X86_64

arch/x86/include/asm/user_unwind.h new file +0 -0

new file mode 100644
index 000000000000..8c509c65cfb5

arch/x86/include/asm/user_unwind.h line 9

Quoted 5 lines

arch/x86/include/asm/user_unwind.h new file +5 -0

+#define ARCH_INIT_USER_FP_FRAME							\
+	.ra_off		= (s32)sizeof(long) * -1,				\
+	.cfa_off	= (s32)sizeof(long) * 2,				\
+	.fp_off		= (s32)sizeof(long) * -2,				\
+	.use_fp		= true,

What about compat crap?

include/linux/sframe.h new file +0 -0

new file mode 100644
index 000000000000..3a44f76929e2

include/linux/sframe.h new file +38 -0

+};
+
+struct user_unwind_frame;
+
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+
+#define INIT_MM_SFRAME .sframe_mt = MTREE_INIT(sframe_mt, 0)
+
+extern void sframe_free_mm(struct mm_struct *mm);
+
+extern int __sframe_add_section(struct sframe_file *file);
+extern int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end);
+extern int sframe_remove_section(unsigned long sframe_addr);
+extern int sframe_find(unsigned long ip, struct user_unwind_frame *frame);
+
+static inline bool current_has_sframe(void)
+{
+	struct mm_struct *mm = current->mm;
+
+	return mm && !mtree_empty(&mm->sframe_mt);
+}
+
+#else /* !CONFIG_HAVE_USER_UNWIND_SFRAME */
+
+#define INIT_MM_SFRAME
+
+static inline void sframe_free_mm(struct mm_struct *mm) {}
+
+static inline int __sframe_add_section(struct sframe_file *file) { return -EINVAL; }
+static inline int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end) { return -EINVAL; }
+static inline int sframe_remove_section(unsigned long sframe_addr) { return -EINVAL; }
+static inline int sframe_find(unsigned long ip, struct user_unwind_frame *frame) { return -EINVAL; }
+
+static inline bool current_has_sframe(void) { return false; }
+
+#endif /* CONFIG_HAVE_USER_UNWIND_SFRAME */
+
+#endif /* _LINUX_SFRAME_H */

include/linux/user_unwind.h +1 -0

index 0a19ac6c92b2..8003f9d35405 100644
@@ -7,6 +7,7 @@
 enum user_unwind_type {
 	USER_UNWIND_TYPE_AUTO,
 	USER_UNWIND_TYPE_FP,
+	USER_UNWIND_TYPE_SFRAME,
 };
 
 struct user_unwind_frame {

include/uapi/linux/elf.h +1 -0

index b54b313bcf07..b2aca31e1a49 100644
@@ -39,6 +39,7 @@ typedef __s64	Elf64_Sxword;
 #define PT_GNU_STACK	(PT_LOOS + 0x474e551)
 #define PT_GNU_RELRO	(PT_LOOS + 0x474e552)
 #define PT_GNU_PROPERTY	(PT_LOOS + 0x474e553)
+#define PT_GNU_SFRAME	(PT_LOOS + 0x474e554)
 
 
 /* ARM MTE memory tag segment type */

include/uapi/linux/prctl.h +3 -0

index 35791791a879..69511077c910 100644
@@ -328,4 +328,7 @@ struct prctl_mm_map {
 # define PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC	0x10 /* Clear the aspect on exec */
 # define PR_PPC_DEXCR_CTRL_MASK		0x1f
 
+#define PR_ADD_SFRAME			74
+#define PR_REMOVE_SFRAME		75
+
 #endif /* _LINUX_PRCTL_H */

kernel/fork.c +10 -0

index cc760491f201..a216f091edfb 100644
@@ -104,6 +104,7 @@
 #include <linux/rseq.h>
 #include <uapi/linux/pidfd.h>
 #include <linux/pidfs.h>
+#include <linux/sframe.h>
 
 #include <asm/pgalloc.h>
 #include <linux/uaccess.h>

@@ -923,6 +924,7 @@ void __mmdrop(struct mm_struct *mm)
 	mm_pasid_drop(mm);
 	mm_destroy_cid(mm);
 	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
+	sframe_free_mm(mm);
 
 	free_mm(mm);
 }

@@ -1249,6 +1251,13 @@ static void mm_init_uprobes_state(struct mm_struct *mm)
 #endif
 }
 
+static void mm_init_sframe(struct mm_struct *mm)
+{
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+	mt_init(&mm->sframe_mt);
+#endif
+}
+
 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	struct user_namespace *user_ns)
 {

@@ -1280,6 +1289,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	mm->pmd_huge_pte = NULL;
 #endif
 	mm_init_uprobes_state(mm);
+	mm_init_sframe(mm);
 	hugetlb_count_init(mm);
 
 	if (current->mm) {

kernel/sys.c +11 -0

index 3a2df1bd9f64..e4d2b64f4ae4 100644
@@ -64,6 +64,7 @@
 #include <linux/rcupdate.h>
 #include <linux/uidgid.h>
 #include <linux/cred.h>
+#include <linux/sframe.h>
 
 #include <linux/nospec.h>
 

@@ -2782,6 +2783,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
 	case PR_RISCV_SET_ICACHE_FLUSH_CTX:
 		error = RISCV_SET_ICACHE_FLUSH_CTX(arg2, arg3);
 		break;
+	case PR_ADD_SFRAME:
+		if (arg5)
+			return -EINVAL;
+		error = sframe_add_section(arg2, arg3, arg4);
+		break;
+	case PR_REMOVE_SFRAME:
+		if (arg3 || arg4 || arg5)
+			return -EINVAL;
+		error = sframe_remove_section(arg2);
+		break;
 	default:
 		error = -EINVAL;
 		break;

kernel/unwind/Makefile +1 -0

index eb466d6a3295..6f202c5840cf 100644
@@ -1 +1,2 @@
 obj-$(CONFIG_HAVE_USER_UNWIND) += user.o
+obj-$(CONFIG_HAVE_USER_UNWIND_SFRAME) += sframe.o

kernel/unwind/sframe.c new file +80 -0

new file mode 100644
index 000000000000..3e4d29e737a1
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/srcu.h>
+#include <linux/uaccess.h>
+#include <linux/mm.h>
+#include <linux/sframe.h>
+#include <linux/user_unwind.h>
+
+#include "sframe.h"
+
+#define SFRAME_FILENAME_LEN 32
+
+struct sframe_section {
+	struct rcu_head rcu;
+
+	unsigned long sframe_addr;
+	unsigned long text_addr;
+
+	unsigned long fdes_addr;
+	unsigned long fres_addr;
+	unsigned int  fdes_nr;
+	signed char ra_off, fp_off;
+};
+
+DEFINE_STATIC_SRCU(sframe_srcu);
+
+#define __SFRAME_GET_USER(out, user_ptr, type)				\
+({									\
+	type __tmp;							\
+	if (get_user(__tmp, (type *)user_ptr))				\
+		return -EFAULT;						\
+	user_ptr += sizeof(__tmp);					\
+	out = __tmp;							\
+})
+
+#define SFRAME_GET_USER_SIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, s8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, s16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, s32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+#define SFRAME_GET_USER_UNSIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, u8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, u16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, u32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+static unsigned char fre_type_to_size(unsigned char fre_type)
+{
+	if (fre_type > 2)
+		return 0;
+	return 1 << fre_type;
+}
+
+static unsigned char offset_size_enum_to_size(unsigned char off_size)
+{

Quoted 6 lines · On Sat, 14 Sep 2024 01:02:05 +0200 Josh Poimboeuf <[email protected]> wrote:

include/linux/sframe.h new file +0 -0

new file mode 100644
index 000000000000..3a44f76929e2

include/linux/sframe.h line 8

Quoted 8 lines

include/linux/sframe.h new file +8 -0

@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_SFRAME_H
+#define _LINUX_SFRAME_H
+
+#include <linux/mm_types.h>
+
+struct sframe_file {
+	unsigned long sframe_addr, text_start, text_end;

Please make each entry a separate line:

	unsigned long		sframe_addr;
	unsigned long		text_start;
	unsigned long		text_end;

It may be fine for declaring variables like this in a function, but it should not be done in a structure.

Quoted 238 lines

include/linux/sframe.h new file +38 -0

+};
+
+struct user_unwind_frame;
+
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+
+#define INIT_MM_SFRAME .sframe_mt = MTREE_INIT(sframe_mt, 0)
+
+extern void sframe_free_mm(struct mm_struct *mm);
+
+extern int __sframe_add_section(struct sframe_file *file);
+extern int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end);
+extern int sframe_remove_section(unsigned long sframe_addr);
+extern int sframe_find(unsigned long ip, struct user_unwind_frame *frame);
+
+static inline bool current_has_sframe(void)
+{
+	struct mm_struct *mm = current->mm;
+
+	return mm && !mtree_empty(&mm->sframe_mt);
+}
+
+#else /* !CONFIG_HAVE_USER_UNWIND_SFRAME */
+
+#define INIT_MM_SFRAME
+
+static inline void sframe_free_mm(struct mm_struct *mm) {}
+
+static inline int __sframe_add_section(struct sframe_file *file) { return -EINVAL; }
+static inline int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end) { return -EINVAL; }
+static inline int sframe_remove_section(unsigned long sframe_addr) { return -EINVAL; }
+static inline int sframe_find(unsigned long ip, struct user_unwind_frame *frame) { return -EINVAL; }
+
+static inline bool current_has_sframe(void) { return false; }
+
+#endif /* CONFIG_HAVE_USER_UNWIND_SFRAME */
+
+#endif /* _LINUX_SFRAME_H */

include/linux/user_unwind.h +1 -0

index 0a19ac6c92b2..8003f9d35405 100644
@@ -7,6 +7,7 @@
  enum user_unwind_type {
  	USER_UNWIND_TYPE_AUTO,
  	USER_UNWIND_TYPE_FP,
+	USER_UNWIND_TYPE_SFRAME,
  };
 
  struct user_unwind_frame {

include/uapi/linux/elf.h +1 -0

index b54b313bcf07..b2aca31e1a49 100644
@@ -39,6 +39,7 @@ typedef __s64	Elf64_Sxword;
  #define PT_GNU_STACK	(PT_LOOS + 0x474e551)
  #define PT_GNU_RELRO	(PT_LOOS + 0x474e552)
  #define PT_GNU_PROPERTY	(PT_LOOS + 0x474e553)
+#define PT_GNU_SFRAME	(PT_LOOS + 0x474e554)
 
 
  /* ARM MTE memory tag segment type */

include/uapi/linux/prctl.h +3 -0

index 35791791a879..69511077c910 100644
@@ -328,4 +328,7 @@ struct prctl_mm_map {
  # define PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC	0x10 /* Clear the aspect on exec */
  # define PR_PPC_DEXCR_CTRL_MASK		0x1f
 
+#define PR_ADD_SFRAME			74
+#define PR_REMOVE_SFRAME		75
+
  #endif /* _LINUX_PRCTL_H */

kernel/fork.c +10 -0

index cc760491f201..a216f091edfb 100644
@@ -104,6 +104,7 @@
  #include <linux/rseq.h>
  #include <uapi/linux/pidfd.h>
  #include <linux/pidfs.h>
+#include <linux/sframe.h>
 
  #include <asm/pgalloc.h>
  #include <linux/uaccess.h>

@@ -923,6 +924,7 @@ void __mmdrop(struct mm_struct *mm)
  	mm_pasid_drop(mm);
  	mm_destroy_cid(mm);
  	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
+	sframe_free_mm(mm);
 
  	free_mm(mm);
  }

@@ -1249,6 +1251,13 @@ static void mm_init_uprobes_state(struct mm_struct *mm)
  #endif
  }
 
+static void mm_init_sframe(struct mm_struct *mm)
+{
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+	mt_init(&mm->sframe_mt);
+#endif
+}
+
  static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
  	struct user_namespace *user_ns)
  {

@@ -1280,6 +1289,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
  	mm->pmd_huge_pte = NULL;
  #endif
  	mm_init_uprobes_state(mm);
+	mm_init_sframe(mm);
  	hugetlb_count_init(mm);
 
  	if (current->mm) {

kernel/sys.c +11 -0

index 3a2df1bd9f64..e4d2b64f4ae4 100644
@@ -64,6 +64,7 @@
  #include <linux/rcupdate.h>
  #include <linux/uidgid.h>
  #include <linux/cred.h>
+#include <linux/sframe.h>
 
  #include <linux/nospec.h>
 

@@ -2782,6 +2783,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
  	case PR_RISCV_SET_ICACHE_FLUSH_CTX:
  		error = RISCV_SET_ICACHE_FLUSH_CTX(arg2, arg3);
  		break;
+	case PR_ADD_SFRAME:
+		if (arg5)
+			return -EINVAL;
+		error = sframe_add_section(arg2, arg3, arg4);
+		break;
+	case PR_REMOVE_SFRAME:
+		if (arg3 || arg4 || arg5)
+			return -EINVAL;
+		error = sframe_remove_section(arg2);
+		break;
  	default:
  		error = -EINVAL;
  		break;

kernel/unwind/Makefile +1 -0

index eb466d6a3295..6f202c5840cf 100644
@@ -1 +1,2 @@
  obj-$(CONFIG_HAVE_USER_UNWIND) += user.o
+obj-$(CONFIG_HAVE_USER_UNWIND_SFRAME) += sframe.o

kernel/unwind/sframe.c new file +80 -0

new file mode 100644
index 000000000000..3e4d29e737a1
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/srcu.h>
+#include <linux/uaccess.h>
+#include <linux/mm.h>
+#include <linux/sframe.h>
+#include <linux/user_unwind.h>
+
+#include "sframe.h"
+
+#define SFRAME_FILENAME_LEN 32
+
+struct sframe_section {
+	struct rcu_head rcu;
+
+	unsigned long sframe_addr;
+	unsigned long text_addr;
+
+	unsigned long fdes_addr;
+	unsigned long fres_addr;
+	unsigned int  fdes_nr;
+	signed char ra_off, fp_off;
+};
+
+DEFINE_STATIC_SRCU(sframe_srcu);
+
+#define __SFRAME_GET_USER(out, user_ptr, type)				\
+({									\
+	type __tmp;							\
+	if (get_user(__tmp, (type *)user_ptr))				\
+		return -EFAULT;						\
+	user_ptr += sizeof(__tmp);					\
+	out = __tmp;							\
+})
+
+#define SFRAME_GET_USER_SIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, s8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, s16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, s32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+#define SFRAME_GET_USER_UNSIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, u8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, u16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, u32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+static unsigned char fre_type_to_size(unsigned char fre_type)
+{
+	if (fre_type > 2)
+		return 0;
+	return 1 << fre_type;
+}
+
+static unsigned char offset_size_enum_to_size(unsigned char off_size)
+{

kernel/unwind/sframe.c line 90

Quoted 10 lines

kernel/unwind/sframe.c new file +10 -0

+	if (off_size > 2)
+		return 0;
+	return 1 << off_size;
+}
+
+static int find_fde(struct sframe_section *sec, unsigned long ip,
+		    struct sframe_fde *fde)
+{
+	s32 func_off, ip_off;
+	struct sframe_fde __user *first, *last, *mid, *found;

Need to initialize found = NULL;

kernel/unwind/sframe.c line 96

Quoted 6 lines

kernel/unwind/sframe.c new file +6 -0

+
+	ip_off = ip - sec->sframe_addr;
+
+	first = (void *)sec->fdes_addr;
+	last = first + sec->fdes_nr;
+	while (first <= last) {

So we trust user space to have this table sorted?

kernel/unwind/sframe.c line 101

Quoted 5 lines

kernel/unwind/sframe.c new file +5 -0

+		mid = first + ((last - first) / 2);
+		if (get_user(func_off, (s32 *)mid))
+			return -EFAULT;
+		if (ip_off >= func_off) {
+			found = mid;

If it's not sorted, this can return the wrong value.

We should have some check that has something like:

	s32 low_func_off = 0, high_func_off = 0;

			if (low_func_off && low_func_off > func_off)
				return -EINVAL;	
			low_func_off = func_off;

kernel/unwind/sframe.c line 103

Quoted 2 lines

kernel/unwind/sframe.c new file +2 -0

+			first = mid + 1;
+		} else

			{ /* Note, this needs a bracket anyway, because
			     rules are, if one if block has a bracket,
			     the other needs one too */

			if (high_func_off && high_func_off < func_off)
				return -EINVAL;

			high_func_off = func_off;

kernel/unwind/sframe.c line 104

Quoted 1 line

kernel/unwind/sframe.c new file +1 -0

+			last = mid - 1;

		}

kernel/unwind/sframe.c line 114

Quoted 10 lines

kernel/unwind/sframe.c new file +10 -0

+	}
+
+	if (!found)
+		return -EINVAL;
+
+	if (copy_from_user(fde, found, sizeof(*fde)))
+		return -EFAULT;
+
+	return 0;
+}

-- Steve

Quoted 2 lines

So we trust user space to have this table sorted?

GNU ld will create this table sorted when linking .sframe sections and will set SFRAME_F_FDE_SORTED in flags in the output .sframe section. In the current patch, I see the __sframe_add_section () includes a check for SFRAME_F_FDE_SORTED for admitting SFrame sections.

So proceeding here with the assumption that the SFrame FDE list is sorted should work fine.

Quoted 8 lines · On Tue, 1 Oct 2024 11:20:35 -0700 Indu Bhagat <[email protected]> wrote:

Quoted 2 lines

So we trust user space to have this table sorted?

GNU ld will create this table sorted when linking .sframe sections and will set SFRAME_F_FDE_SORTED in flags in the output .sframe section. In the current patch, I see the __sframe_add_section () includes a check for SFRAME_F_FDE_SORTED for admitting SFrame sections.

Quoted 2 lines

So proceeding here with the assumption that the SFrame FDE list is sorted should work fine.

No not at all! We *cannot trust* user space. This could lead to a security hole if we assume it's sorted. The kernel must not trust anything it receives from user space. Because an attacker will be looking for ways to confuse the kernel to exploit it.

I don't quite understand, sorry.

Doing a binary search on an unordered table fails to find some entries that could be discovered by a linear scan. But an attacker could just as well use an incomplete table from the start. So assuming an ordered table seems rather unlikely to introduce additional problems. (Given the lack of a formal threat model, it's impossible to make more precise claims in either direction.)

Some distros have started compiling frame pointers into all their packages to enable the kernel to do system-wide profiling of user space. Unfortunately that creates a runtime performance penalty across the entire system. Using DWARF instead isn't feasible due to the complexity it would add to the kernel.

For in-kernel unwinding we solved this problem with the creation of the ORC unwinder for x86_64. Similarly, for user space the GNU assembler has created the SFrame format starting with binutils 2.41 for SFrame v2. SFrame is a simpler version of .eh_frame which gets placed in the .sframe section.

Add support for unwinding user space using SFrame.

More information about SFrame can be found here:

- https://lwn.net/Articles/932209/
- https://lwn.net/Articles/940686/
- https://sourceware.org/binutils/docs/sframe-spec.html

Signed-off-by Josh Poimboeuf <[email protected]>

---
  arch/Kconfig                |   3 +
  fs/binfmt_elf.c             |  47 +++-
  include/linux/mm_types.h    |   3 +
  include/linux/sframe.h      |  46 ++++
  include/linux/user_unwind.h |   1 +
  include/uapi/linux/elf.h    |   1 +
  include/uapi/linux/prctl.h  |   3 +
  kernel/fork.c               |  10 +
  kernel/sys.c                |  11 +
  kernel/unwind/Makefile      |   1 +
  kernel/unwind/sframe.c      | 420 ++++++++++++++++++++++++++++++++++++
  kernel/unwind/sframe.h      | 215 ++++++++++++++++++
  kernel/unwind/user.c        |  14 ++
  mm/init-mm.c                |   4 +-
  14 files changed, 774 insertions(+), 5 deletions(-)

create mode 100644 include/linux/sframe.h
create mode 100644 kernel/unwind/sframe.c
create mode 100644 kernel/unwind/sframe.h

arch/Kconfig +0 -0

index b1002b2da331..ff5d5bc5f947 100644

arch/Kconfig +1 -0

+
  config HAVE_PERF_REGS
  	bool
  	help

fs/binfmt_elf.c +43 -4

index 19fa49cd9907..923aed390f2e 100644
@@ -47,6 +47,7 @@
  #include <linux/dax.h>
  #include <linux/uaccess.h>
  #include <linux/rseq.h>
+#include <linux/sframe.h>
  #include <asm/param.h>
  #include <asm/page.h>
 

@@ -633,11 +634,13 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
  		unsigned long no_base, struct elf_phdr *interp_elf_phdata,
  		struct arch_elf_state *arch_state)
  {
-	struct elf_phdr *eppnt;
+	struct elf_phdr *eppnt, *sframe_phdr = NULL;
  	unsigned long load_addr = 0;
  	int load_addr_set = 0;
  	unsigned long error = ~0UL;
  	unsigned long total_size;
+	unsigned long start_code = ~0UL;
+	unsigned long end_code = 0;
  	int i;
 
  	/* First of all, some simple consistency checks */

@@ -659,7 +662,8 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
 
  	eppnt = interp_elf_phdata;
  	for (i = 0; i < interp_elf_ex->e_phnum; i++, eppnt++) {
-		if (eppnt->p_type == PT_LOAD) {
+		switch (eppnt->p_type) {
+		case PT_LOAD: {
  			int elf_type = MAP_PRIVATE;
  			int elf_prot = make_prot(eppnt->p_flags, arch_state,
  						 true, true);

@@ -688,7 +692,7 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
  			/*
  			 * Check to see if the section's size will overflow the
  			 * allowed task size. Note that p_filesz must always be
-			 * <= p_memsize so it's only necessary to check p_memsz.
+			 * <= p_memsz so it's only necessary to check p_memsz.
  			 */
  			k = load_addr + eppnt->p_vaddr;
  			if (BAD_ADDR(k) ||

@@ -698,7 +702,28 @@ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex,
  				error = -ENOMEM;
  				goto out;
  			}
+
+			if ((eppnt->p_flags & PF_X) && k < start_code)
+				start_code = k;
+
+			if ((eppnt->p_flags & PF_X) && k + eppnt->p_filesz > end_code)
+				end_code = k + eppnt->p_filesz;
+			break;
  		}
+		case PT_GNU_SFRAME:
+			sframe_phdr = eppnt;
+			break;
+		}
+	}
+
+	if (sframe_phdr) {
+		struct sframe_file sfile = {
+			.sframe_addr	= load_addr + sframe_phdr->p_vaddr,
+			.text_start	= start_code,
+			.text_end	= end_code,
+		};
+
+		__sframe_add_section(&sfile);
  	}
 
  	error = load_addr;

@@ -823,7 +848,7 @@ static int load_elf_binary(struct linux_binprm *bprm)
  	int first_pt_load = 1;
  	unsigned long error;
  	struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL;
-	struct elf_phdr *elf_property_phdata = NULL;
+	struct elf_phdr *elf_property_phdata = NULL, *sframe_phdr = NULL;
  	unsigned long elf_brk;
  	int retval, i;
  	unsigned long elf_entry;

@@ -931,6 +956,10 @@ static int load_elf_binary(struct linux_binprm *bprm)
  				executable_stack = EXSTACK_DISABLE_X;
  			break;
 
+		case PT_GNU_SFRAME:
+			sframe_phdr = elf_ppnt;
+			break;
+
  		case PT_LOPROC ... PT_HIPROC:
  			retval = arch_elf_pt_proc(elf_ex, elf_ppnt,
  						  bprm->file, false,

@@ -1316,6 +1345,16 @@ static int load_elf_binary(struct linux_binprm *bprm)
  				MAP_FIXED | MAP_PRIVATE, 0);
  	}
 
+	if (sframe_phdr) {
+		struct sframe_file sfile = {
+			.sframe_addr	= load_bias + sframe_phdr->p_vaddr,
+			.text_start	= start_code,
+			.text_end	= end_code,
+		};
+
+		__sframe_add_section(&sfile);
+	}
+
  	regs = current_pt_regs();
  #ifdef ELF_PLAT_INIT
  	/*

include/linux/mm_types.h +3 -0

index 485424979254..1aee78cbea33 100644
@@ -1019,6 +1019,9 @@ struct mm_struct {
  #endif
  		} lru_gen;
  #endif /* CONFIG_LRU_GEN_WALKS_MMU */
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+		struct maple_tree sframe_mt;
+#endif
  	} __randomize_layout;
 
  	/*

include/linux/sframe.h new file +46 -0

new file mode 100644
index 000000000000..3a44f76929e2
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_SFRAME_H
+#define _LINUX_SFRAME_H
+
+#include <linux/mm_types.h>
+
+struct sframe_file {
+	unsigned long sframe_addr, text_start, text_end;
+};
+
+struct user_unwind_frame;
+
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+
+#define INIT_MM_SFRAME .sframe_mt = MTREE_INIT(sframe_mt, 0)
+
+extern void sframe_free_mm(struct mm_struct *mm);
+
+extern int __sframe_add_section(struct sframe_file *file);
+extern int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end);
+extern int sframe_remove_section(unsigned long sframe_addr);
+extern int sframe_find(unsigned long ip, struct user_unwind_frame *frame);
+
+static inline bool current_has_sframe(void)
+{
+	struct mm_struct *mm = current->mm;
+
+	return mm && !mtree_empty(&mm->sframe_mt);
+}
+
+#else /* !CONFIG_HAVE_USER_UNWIND_SFRAME */
+
+#define INIT_MM_SFRAME
+
+static inline void sframe_free_mm(struct mm_struct *mm) {}
+
+static inline int __sframe_add_section(struct sframe_file *file) { return -EINVAL; }
+static inline int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end) { return -EINVAL; }
+static inline int sframe_remove_section(unsigned long sframe_addr) { return -EINVAL; }
+static inline int sframe_find(unsigned long ip, struct user_unwind_frame *frame) { return -EINVAL; }
+
+static inline bool current_has_sframe(void) { return false; }
+
+#endif /* CONFIG_HAVE_USER_UNWIND_SFRAME */
+
+#endif /* _LINUX_SFRAME_H */

include/linux/user_unwind.h +1 -0

index 0a19ac6c92b2..8003f9d35405 100644
@@ -7,6 +7,7 @@
  enum user_unwind_type {
  	USER_UNWIND_TYPE_AUTO,
  	USER_UNWIND_TYPE_FP,
+	USER_UNWIND_TYPE_SFRAME,
  };
 
  struct user_unwind_frame {

include/uapi/linux/elf.h +1 -0

index b54b313bcf07..b2aca31e1a49 100644
@@ -39,6 +39,7 @@ typedef __s64	Elf64_Sxword;
  #define PT_GNU_STACK	(PT_LOOS + 0x474e551)
  #define PT_GNU_RELRO	(PT_LOOS + 0x474e552)
  #define PT_GNU_PROPERTY	(PT_LOOS + 0x474e553)
+#define PT_GNU_SFRAME	(PT_LOOS + 0x474e554)
 
 
  /* ARM MTE memory tag segment type */

include/uapi/linux/prctl.h +3 -0

index 35791791a879..69511077c910 100644
@@ -328,4 +328,7 @@ struct prctl_mm_map {
  # define PR_PPC_DEXCR_CTRL_CLEAR_ONEXEC	0x10 /* Clear the aspect on exec */
  # define PR_PPC_DEXCR_CTRL_MASK		0x1f
 
+#define PR_ADD_SFRAME			74
+#define PR_REMOVE_SFRAME		75
+
  #endif /* _LINUX_PRCTL_H */

kernel/fork.c +10 -0

index cc760491f201..a216f091edfb 100644
@@ -104,6 +104,7 @@
  #include <linux/rseq.h>
  #include <uapi/linux/pidfd.h>
  #include <linux/pidfs.h>
+#include <linux/sframe.h>
 
  #include <asm/pgalloc.h>
  #include <linux/uaccess.h>

@@ -923,6 +924,7 @@ void __mmdrop(struct mm_struct *mm)
  	mm_pasid_drop(mm);
  	mm_destroy_cid(mm);
  	percpu_counter_destroy_many(mm->rss_stat, NR_MM_COUNTERS);
+	sframe_free_mm(mm);
 
  	free_mm(mm);
  }

@@ -1249,6 +1251,13 @@ static void mm_init_uprobes_state(struct mm_struct *mm)
  #endif
  }
 
+static void mm_init_sframe(struct mm_struct *mm)
+{
+#ifdef CONFIG_HAVE_USER_UNWIND_SFRAME
+	mt_init(&mm->sframe_mt);
+#endif
+}
+
  static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
  	struct user_namespace *user_ns)
  {

@@ -1280,6 +1289,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
  	mm->pmd_huge_pte = NULL;
  #endif
  	mm_init_uprobes_state(mm);
+	mm_init_sframe(mm);
  	hugetlb_count_init(mm);
 
  	if (current->mm) {

kernel/sys.c +11 -0

index 3a2df1bd9f64..e4d2b64f4ae4 100644
@@ -64,6 +64,7 @@
  #include <linux/rcupdate.h>
  #include <linux/uidgid.h>
  #include <linux/cred.h>
+#include <linux/sframe.h>
 
  #include <linux/nospec.h>
 

@@ -2782,6 +2783,16 @@ SYSCALL_DEFINE5(prctl, int, option, unsigned long, arg2, unsigned long, arg3,
  	case PR_RISCV_SET_ICACHE_FLUSH_CTX:
  		error = RISCV_SET_ICACHE_FLUSH_CTX(arg2, arg3);
  		break;
+	case PR_ADD_SFRAME:
+		if (arg5)
+			return -EINVAL;
+		error = sframe_add_section(arg2, arg3, arg4);
+		break;
+	case PR_REMOVE_SFRAME:
+		if (arg3 || arg4 || arg5)
+			return -EINVAL;
+		error = sframe_remove_section(arg2);
+		break;
  	default:
  		error = -EINVAL;
  		break;

kernel/unwind/Makefile +1 -0

index eb466d6a3295..6f202c5840cf 100644
@@ -1 +1,2 @@
  obj-$(CONFIG_HAVE_USER_UNWIND) += user.o
+obj-$(CONFIG_HAVE_USER_UNWIND_SFRAME) += sframe.o

kernel/unwind/sframe.c new file +111 -0

new file mode 100644
index 000000000000..3e4d29e737a1
@@ -0,0 +1,420 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/srcu.h>
+#include <linux/uaccess.h>
+#include <linux/mm.h>
+#include <linux/sframe.h>
+#include <linux/user_unwind.h>
+
+#include "sframe.h"
+
+#define SFRAME_FILENAME_LEN 32
+
+struct sframe_section {
+	struct rcu_head rcu;
+
+	unsigned long sframe_addr;
+	unsigned long text_addr;
+
+	unsigned long fdes_addr;
+	unsigned long fres_addr;
+	unsigned int  fdes_nr;
+	signed char ra_off, fp_off;
+};
+
+DEFINE_STATIC_SRCU(sframe_srcu);
+
+#define __SFRAME_GET_USER(out, user_ptr, type)				\
+({									\
+	type __tmp;							\
+	if (get_user(__tmp, (type *)user_ptr))				\
+		return -EFAULT;						\
+	user_ptr += sizeof(__tmp);					\
+	out = __tmp;							\
+})
+
+#define SFRAME_GET_USER_SIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, s8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, s16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, s32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+#define SFRAME_GET_USER_UNSIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, u8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, u16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, u32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+static unsigned char fre_type_to_size(unsigned char fre_type)
+{
+	if (fre_type > 2)
+		return 0;
+	return 1 << fre_type;
+}
+
+static unsigned char offset_size_enum_to_size(unsigned char off_size)
+{
+	if (off_size > 2)
+		return 0;
+	return 1 << off_size;
+}
+
+static int find_fde(struct sframe_section *sec, unsigned long ip,
+		    struct sframe_fde *fde)
+{
+	s32 func_off, ip_off;
+	struct sframe_fde __user *first, *last, *mid, *found;
+
+	ip_off = ip - sec->sframe_addr;
+
+	first = (void *)sec->fdes_addr;
+	last = first + sec->fdes_nr;
+	while (first <= last) {
+		mid = first + ((last - first) / 2);
+		if (get_user(func_off, (s32 *)mid))
+			return -EFAULT;
+		if (ip_off >= func_off) {
+			found = mid;
+			first = mid + 1;
+		} else
+			last = mid - 1;
+	}
+
+	if (!found)
+		return -EINVAL;
+
+	if (copy_from_user(fde, found, sizeof(*fde)))
+		return -EFAULT;

kernel/unwind/sframe.c new file +7 -0

+	unsigned char offset_count, offset_size;
+	unsigned char addr_size;
+	void __user *f, *last_f;
+	u8 fre_info;
+	int i;
+
+	addr_size = fre_type_to_size(fre_type);

kernel/unwind/sframe.c new file +18 -0

+
+		if (fde_type == SFRAME_FDE_TYPE_PCINC) {
+			if (fre_ip_off > ip_off)
+				break;
+		} else {
+			/* SFRAME_FDE_TYPE_PCMASK */
+			if (ip_off % fde->rep_size < fre_ip_off)
+				break;
+		}
+
+		SFRAME_GET_USER_UNSIGNED(fre_info, f, 1);
+
+		offset_count = SFRAME_FRE_OFFSET_COUNT(fre_info);
+		offset_size  = offset_size_enum_to_size(SFRAME_FRE_OFFSET_SIZE(fre_info));
+
+		if (!offset_count || !offset_size)
+			return -EINVAL;
+

kernel/unwind/sframe.c new file +248 -0

+		SFRAME_GET_USER_SIGNED(ra_off, f, offset_size);
+	}
+
+	fp_off = sec->fp_off;
+	if (!fp_off && offset_count) {
+		offset_count--;
+		SFRAME_GET_USER_SIGNED(fp_off, f, offset_size);
+	}
+
+	if (offset_count)
+		return -EINVAL;
+
+	frame->cfa_off = cfa_off;
+	frame->ra_off = ra_off;
+	frame->fp_off = fp_off;
+	frame->use_fp = SFRAME_FRE_CFA_BASE_REG_ID(fre_info) == SFRAME_BASE_REG_FP;
+
+	return 0;
+}
+
+int sframe_find(unsigned long ip, struct user_unwind_frame *frame)
+{
+	struct mm_struct *mm = current->mm;
+	struct sframe_section *sec;
+	struct sframe_fde fde;
+	int srcu_idx;
+	int ret = -EINVAL;
+
+	srcu_idx = srcu_read_lock(&sframe_srcu);
+
+	sec = mtree_load(&mm->sframe_mt, ip);
+	if (!sec) {
+		srcu_read_unlock(&sframe_srcu, srcu_idx);
+		return -EINVAL;
+	}
+
+
+	ret = find_fde(sec, ip, &fde);
+	if (ret)
+		goto err_unlock;
+
+	ret = find_fre(sec, &fde, ip, frame);
+	if (ret)
+		goto err_unlock;
+
+	srcu_read_unlock(&sframe_srcu, srcu_idx);
+	return 0;
+
+err_unlock:
+	srcu_read_unlock(&sframe_srcu, srcu_idx);
+	return ret;
+}
+
+static int get_sframe_file(unsigned long sframe_addr, struct sframe_file *file)
+{
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *sframe_vma, *text_vma, *vma;
+	VMA_ITERATOR(vmi, mm, 0);
+
+	mmap_read_lock(mm);
+
+	sframe_vma = vma_lookup(mm, sframe_addr);
+	if (!sframe_vma || !sframe_vma->vm_file)
+		goto err_unlock;
+
+	text_vma = NULL;
+
+	for_each_vma(vmi, vma) {
+		if (vma->vm_file != sframe_vma->vm_file)
+			continue;
+		if (vma->vm_flags & VM_EXEC) {
+			if (text_vma) {
+				/*
+				 * Multiple EXEC segments in a single file
+				 * aren't currently supported, is that a thing?
+				 */
+				mmap_read_unlock(mm);
+				pr_warn_once("unsupported multiple EXEC segments in task %s[%d]\n",
+					     current->comm, current->pid);
+				return -EINVAL;
+			}
+			text_vma = vma;
+		}
+	}
+
+	file->sframe_addr	= sframe_addr;
+	file->text_start	= text_vma->vm_start;
+	file->text_end		= text_vma->vm_end;
+
+	mmap_read_unlock(mm);
+	return 0;
+
+err_unlock:
+	mmap_read_unlock(mm);
+	return -EINVAL;
+}
+
+static int validate_sframe_addrs(struct sframe_file *file)
+{
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *text_vma;
+
+	mmap_read_lock(mm);
+
+	if (!vma_lookup(mm, file->sframe_addr))
+		goto err_unlock;
+
+	text_vma = vma_lookup(mm, file->text_start);
+	if (!(text_vma->vm_flags & VM_EXEC))
+		goto err_unlock;
+
+	if (vma_lookup(mm, file->text_end-1) != text_vma)
+		goto err_unlock;
+
+	mmap_read_unlock(mm);
+	return 0;
+
+err_unlock:
+	mmap_read_unlock(mm);
+	return -EINVAL;
+}
+
+int __sframe_add_section(struct sframe_file *file)
+{
+	struct maple_tree *sframe_mt = &current->mm->sframe_mt;
+	struct sframe_section *sec;
+	struct sframe_header shdr;
+	unsigned long header_end;
+	int ret;
+
+	if (copy_from_user(&shdr, (void *)file->sframe_addr, sizeof(shdr)))
+		return -EFAULT;
+
+	if (shdr.preamble.magic != SFRAME_MAGIC ||
+	    shdr.preamble.version != SFRAME_VERSION_2 ||
+	    !(shdr.preamble.flags & SFRAME_F_FDE_SORTED) ||
+	    shdr.auxhdr_len || !shdr.num_fdes || !shdr.num_fres ||
+	    shdr.fdes_off > shdr.fres_off) {
+		/*
+		 * Either binutils < 2.41, corrupt sframe header, or
+		 * unsupported feature.
+		 * */
+		pr_warn_once("bad sframe header in task %s[%d]\n",
+			     current->comm, current->pid);
+		return -EINVAL;
+	}
+
+	header_end = file->sframe_addr + SFRAME_HDR_SIZE(shdr);
+
+	sec = kmalloc(sizeof(*sec), GFP_KERNEL);
+	if (!sec)
+		return -ENOMEM;
+
+	sec->sframe_addr	= file->sframe_addr;
+	sec->text_addr		= file->text_start;
+	sec->fdes_addr		= header_end + shdr.fdes_off;
+	sec->fres_addr		= header_end + shdr.fres_off;
+	sec->fdes_nr		= shdr.num_fdes;
+	sec->ra_off		= shdr.cfa_fixed_ra_offset;
+	sec->fp_off		= shdr.cfa_fixed_fp_offset;
+
+	ret = mtree_insert_range(sframe_mt, file->text_start, file->text_end,
+				 sec, GFP_KERNEL);
+	if (ret) {
+		kfree(sec);
+		return ret;
+	}
+
+	return 0;
+}
+
+int sframe_add_section(unsigned long sframe_addr, unsigned long text_start, unsigned long text_end)
+{
+	struct sframe_file file;
+	int ret;
+
+	if (!text_start || !text_end) {
+		ret = get_sframe_file(sframe_addr, &file);
+		if (ret)
+			return ret;
+	} else {
+		/*
+		 * This is mainly for generated code, for which the text isn't
+		 * file-backed so the user has to give the text bounds.
+		 */
+		file.sframe_addr	= sframe_addr;
+		file.text_start		= text_start;
+		file.text_end		= text_end;
+		ret = validate_sframe_addrs(&file);
+		if (ret)
+			return ret;
+	}
+
+	return __sframe_add_section(&file);
+}
+
+static void sframe_free_rcu(struct rcu_head *rcu)
+{
+	struct sframe_section *sec = container_of(rcu, struct sframe_section, rcu);
+
+	kfree(sec);
+}
+
+static int __sframe_remove_section(struct mm_struct *mm,
+				   struct sframe_section *sec)
+{
+	struct sframe_section *s;
+
+	s = mtree_erase(&mm->sframe_mt, sec->text_addr);
+	if (!s || WARN_ON_ONCE(s != sec))
+		return -EINVAL;
+
+	call_srcu(&sframe_srcu, &sec->rcu, sframe_free_rcu);
+
+	return 0;
+}
+
+int sframe_remove_section(unsigned long sframe_addr)
+{
+	struct mm_struct *mm = current->mm;
+	struct sframe_section *sec;
+	unsigned long index = 0;
+
+	sec = mtree_load(&mm->sframe_mt, sframe_addr);
+	if (!sec)
+		return -EINVAL;
+
+	mt_for_each(&mm->sframe_mt, sec, index, ULONG_MAX) {
+		if (sec->sframe_addr == sframe_addr)
+			return __sframe_remove_section(mm, sec);
+	}
+
+	return -EINVAL;
+}
+
+void sframe_free_mm(struct mm_struct *mm)
+{
+	struct sframe_section *sec;
+	unsigned long index = 0;
+
+	if (!mm)
+		return;
+
+	mt_for_each(&mm->sframe_mt, sec, index, ULONG_MAX)
+		kfree(sec);
+
+	mtree_destroy(&mm->sframe_mt);
+}

kernel/unwind/sframe.h new file +59 -0

new file mode 100644
index 000000000000..aa468d6f1f4a
@@ -0,0 +1,215 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright (C) 2023, Oracle and/or its affiliates.
+ *
+ * This file contains definitions for the SFrame stack tracing format, which is
+ * documented at https://sourceware.org/binutils/docs
+ */
+#ifndef _SFRAME_H
+#define _SFRAME_H
+
+#include <linux/types.h>
+
+#define SFRAME_VERSION_1	1
+#define SFRAME_VERSION_2	2
+#define SFRAME_MAGIC		0xdee2
+
+/* Function Descriptor Entries are sorted on PC. */
+#define SFRAME_F_FDE_SORTED	0x1
+/* Frame-pointer based stack tracing. Defined, but not set. */
+#define SFRAME_F_FRAME_POINTER	0x2
+
+#define SFRAME_CFA_FIXED_FP_INVALID 0
+#define SFRAME_CFA_FIXED_RA_INVALID 0
+
+/* Supported ABIs/Arch. */
+#define SFRAME_ABI_AARCH64_ENDIAN_BIG	    1 /* AARCH64 big endian. */
+#define SFRAME_ABI_AARCH64_ENDIAN_LITTLE    2 /* AARCH64 little endian. */
+#define SFRAME_ABI_AMD64_ENDIAN_LITTLE	    3 /* AMD64 little endian. */
+
+/* SFrame FRE types. */
+#define SFRAME_FRE_TYPE_ADDR1	0
+#define SFRAME_FRE_TYPE_ADDR2	1
+#define SFRAME_FRE_TYPE_ADDR4	2
+
+/*
+ * SFrame Function Descriptor Entry types.
+ *
+ * The SFrame format has two possible representations for functions. The
+ * choice of which type to use is made according to the instruction patterns
+ * in the relevant program stub.
+ */
+
+/* Unwinders perform a (PC >= FRE_START_ADDR) to look up a matching FRE. */
+#define SFRAME_FDE_TYPE_PCINC	0
+/*
+ * Unwinders perform a (PC & FRE_START_ADDR_AS_MASK >= FRE_START_ADDR_AS_MASK)
+ * to look up a matching FRE. Typical usecases are pltN entries, trampolines
+ * etc.
+ */
+#define SFRAME_FDE_TYPE_PCMASK	1
+
+/**
+ * struct sframe_preamble - SFrame Preamble.
+ * @magic: Magic number (SFRAME_MAGIC).
+ * @version: Format version number (SFRAME_VERSION).
+ * @flags: Various flags.
+ */
+struct sframe_preamble {
+	u16 magic;

kernel/unwind/sframe.h new file +146 -0

+ *	  used). When fixed, this field specifies the fixed stack frame offset
+ *	  and the individual FREs do not need to track it. When not fixed, it
+ *	  is set to SFRAME_CFA_FIXED_FP_INVALID, and the individual FREs may
+ *	  provide the applicable stack frame offset, if any.
+ * @cfa_fixed_ra_offset: Offset for the Return Address from CFA is fixed for
+ *	  some ABIs. When fixed, this field specifies the fixed stack frame
+ *	  offset and the individual FREs do not need to track it. When not
+ *	  fixed, it is set to SFRAME_CFA_FIXED_FP_INVALID.
+ * @auxhdr_len: Number of bytes making up the auxiliary header, if any.
+ *	  Some ABI/arch, in the future, may use this space for extending the
+ *	  information in SFrame header. Auxiliary header is contained in bytes
+ *	  sequentially following the sframe_header.
+ * @num_fdes: Number of SFrame FDEs in this SFrame section.
+ * @num_fres: Number of SFrame Frame Row Entries.
+ * @fre_len:  Number of bytes in the SFrame Frame Row Entry section.
+ * @fdes_off: Offset of SFrame Function Descriptor Entry section.
+ * @fres_off: Offset of SFrame Frame Row Entry section.
+ */
+struct sframe_header {
+	struct sframe_preamble preamble;
+	u8  abi_arch;
+	s8  cfa_fixed_fp_offset;
+	s8  cfa_fixed_ra_offset;
+	u8  auxhdr_len;
+	u32 num_fdes;
+	u32 num_fres;
+	u32 fre_len;
+	u32 fdes_off;
+	u32 fres_off;
+} __packed;
+
+#define SFRAME_HDR_SIZE(sframe_hdr)	\
+	((sizeof(struct sframe_header) + (sframe_hdr).auxhdr_len))
+
+/* Two possible keys for executable (instruction) pointers signing. */
+#define SFRAME_AARCH64_PAUTH_KEY_A    0 /* Key A. */
+#define SFRAME_AARCH64_PAUTH_KEY_B    1 /* Key B. */
+
+/**
+ * struct sframe_fde - SFrame Function Descriptor Entry.
+ * @start_addr: Function start address. Encoded as a signed offset,
+ *	  relative to the current FDE.
+ * @size: Size of the function in bytes.
+ * @fres_off: Offset of the first SFrame Frame Row Entry of the function,
+ *	  relative to the beginning of the SFrame Frame Row Entry sub-section.
+ * @fres_num: Number of frame row entries for the function.
+ * @info: Additional information for deciphering the stack trace
+ *	  information for the function. Contains information about SFrame FRE
+ *	  type, SFrame FDE type, PAC authorization A/B key, etc.
+ * @rep_size: Block size for SFRAME_FDE_TYPE_PCMASK
+ * @padding: Unused
+ */
+struct sframe_fde {
+	s32 start_addr;
+	u32 size;
+	u32 fres_off;
+	u32 fres_num;
+	u8  info;
+	u8  rep_size;
+	u16 padding;
+} __packed;
+
+/*
+ * 'func_info' in SFrame FDE contains additional information for deciphering
+ * the stack trace information for the function. In V1, the information is
+ * organized as follows:
+ *   - 4-bits: Identify the FRE type used for the function.
+ *   - 1-bit: Identify the FDE type of the function - mask or inc.
+ *   - 1-bit: PAC authorization A/B key (aarch64).
+ *   - 2-bits: Unused.
+ * ---------------------------------------------------------------------
+ * |  Unused  |  PAC auth A/B key (aarch64) |  FDE type |   FRE type   |
+ * |          |        Unused (amd64)       |           |              |
+ * ---------------------------------------------------------------------
+ * 8          6                             5           4              0
+ */
+
+/* Note: Set PAC auth key to SFRAME_AARCH64_PAUTH_KEY_A by default.  */
+#define SFRAME_FUNC_INFO(fde_type, fre_enc_type) \
+	(((SFRAME_AARCH64_PAUTH_KEY_A & 0x1) << 5) | \
+	 (((fde_type) & 0x1) << 4) | ((fre_enc_type) & 0xf))
+
+#define SFRAME_FUNC_FRE_TYPE(data)	  ((data) & 0xf)
+#define SFRAME_FUNC_FDE_TYPE(data)	  (((data) >> 4) & 0x1)
+#define SFRAME_FUNC_PAUTH_KEY(data)	  (((data) >> 5) & 0x1)
+
+/*
+ * Size of stack frame offsets in an SFrame Frame Row Entry. A single
+ * SFrame FRE has all offsets of the same size. Offset size may vary
+ * across frame row entries.
+ */
+#define SFRAME_FRE_OFFSET_1B	  0
+#define SFRAME_FRE_OFFSET_2B	  1
+#define SFRAME_FRE_OFFSET_4B	  2
+
+/* An SFrame Frame Row Entry can be SP or FP based.  */
+#define SFRAME_BASE_REG_FP	0
+#define SFRAME_BASE_REG_SP	1
+
+/*
+ * The index at which a specific offset is presented in the variable length
+ * bytes of an FRE.
+ */
+#define SFRAME_FRE_CFA_OFFSET_IDX   0
+/*
+ * The RA stack offset, if present, will always be at index 1 in the variable
+ * length bytes of the FRE.
+ */
+#define SFRAME_FRE_RA_OFFSET_IDX    1
+/*
+ * The FP stack offset may appear at offset 1 or 2, depending on the ABI as RA
+ * may or may not be tracked.
+ */
+#define SFRAME_FRE_FP_OFFSET_IDX    2
+
+/*
+ * 'fre_info' in SFrame FRE contains information about:
+ *   - 1 bit: base reg for CFA
+ *   - 4 bits: Number of offsets (N). A value of up to 3 is allowed to track
+ *   all three of CFA, FP and RA (fixed implicit order).
+ *   - 2 bits: information about size of the offsets (S) in bytes.
+ *     Valid values are SFRAME_FRE_OFFSET_1B, SFRAME_FRE_OFFSET_2B,
+ *     SFRAME_FRE_OFFSET_4B
+ *   - 1 bit: Mangled RA state bit (aarch64 only).
+ * ---------------------------------------------------------------
+ * | Mangled-RA (aarch64) |  Size of   |   Number of  | base_reg |
+ * |  Unused (amd64)      |  offsets   |    offsets   |          |
+ * ---------------------------------------------------------------
+ * 8                      7             5             1          0
+ */
+
+/* Note: Set mangled_ra_p to zero by default. */
+#define SFRAME_FRE_INFO(base_reg_id, offset_num, offset_size) \
+	(((0 & 0x1) << 7) | (((offset_size) & 0x3) << 5) | \
+	 (((offset_num) & 0xf) << 1) | ((base_reg_id) & 0x1))
+
+/* Set the mangled_ra_p bit as indicated. */
+#define SFRAME_FRE_INFO_UPDATE_MANGLED_RA_P(mangled_ra_p, fre_info) \
+	((((mangled_ra_p) & 0x1) << 7) | ((fre_info) & 0x7f))
+
+#define SFRAME_FRE_CFA_BASE_REG_ID(data)	  ((data) & 0x1)
+#define SFRAME_FRE_OFFSET_COUNT(data)		  (((data) >> 1) & 0xf)
+#define SFRAME_FRE_OFFSET_SIZE(data)		  (((data) >> 5) & 0x3)
+#define SFRAME_FRE_MANGLED_RA_P(data)		  (((data) >> 7) & 0x1)
+
+#endif /* _SFRAME_H */

kernel/unwind/user.c +14 -0

index 5d16f9604a61..3a7b14cf522b 100644
@@ -8,6 +8,7 @@
  #include <linux/sched.h>
  #include <linux/sched/task_stack.h>
  #include <linux/user_unwind.h>
+#include <linux/sframe.h>
  #include <linux/uaccess.h>
  #include <asm/user_unwind.h>
 

@@ -29,6 +30,11 @@ int user_unwind_next(struct user_unwind_state *state)
  	case USER_UNWIND_TYPE_FP:
  		frame = &fp_frame;
  		break;
+	case USER_UNWIND_TYPE_SFRAME:
+		ret = sframe_find(state->ip, frame);
+		if (ret)
+			goto the_end;
+		break;
  	default:
  		BUG();
  	}

@@ -57,6 +63,7 @@ int user_unwind_start(struct user_unwind_state *state,
  		      enum user_unwind_type type)
  {
  	struct pt_regs *regs = task_pt_regs(current);
+	bool sframe_possible = current_has_sframe();
 
  	memset(state, 0, sizeof(*state));
 

@@ -67,6 +74,13 @@ int user_unwind_start(struct user_unwind_state *state,
 
  	switch (type) {
  	case USER_UNWIND_TYPE_AUTO:
+		state->type = sframe_possible ? USER_UNWIND_TYPE_SFRAME :
+						USER_UNWIND_TYPE_FP;
+		break;
+	case USER_UNWIND_TYPE_SFRAME:
+		if (!sframe_possible)
+			return -EINVAL;
+		break;
  	case USER_UNWIND_TYPE_FP:
  		break;
  	default:

mm/init-mm.c +1 -0

index 24c809379274..c4c6af046778 100644
@@ -11,6 +11,7 @@
  #include <linux/atomic.h>
  #include <linux/user_namespace.h>
  #include <linux/iommu.h>
+#include <linux/sframe.h>
  #include <asm/mmu.h>
 
  #ifndef INIT_MM_CONTEXT

arch/Kconfig config HAVE_HARDLOCKUP_DETECTOR_ARCH · line 431

Quoted 6 lines

+++ b/arch/Kconfig

arch/Kconfig +2 -0

@@ -428,6 +428,9 @@ config HAVE_HARDLOCKUP_DETECTOR_ARCH
  config HAVE_USER_UNWIND
  	bool
+config HAVE_USER_UNWIND_SFRAME
+	bool

I found this unwinder of userspace using SFrame to depend on your generic unwinder of userspace framework. Does this warrant to add:

kernel/unwind/sframe.c line 121

Quoted 6 lines

kernel/unwind/sframe.c new file +6 -0

+static int find_fre(struct sframe_section *sec, struct sframe_fde *fde,
+		    unsigned long ip, struct user_unwind_frame *frame)
+{
+	unsigned char fde_type = SFRAME_FUNC_FDE_TYPE(fde->info);
+	unsigned char fre_type = SFRAME_FUNC_FRE_TYPE(fde->info);
+	s32 fre_ip_off, cfa_off, ra_off, fp_off, ip_off;

Doesn't fre_ip_off need to be u32 (see also below)? The SFrame format specification states the FRE sfre_start_address is either u8, u16, or u32:
https://sourceware.org/binutils/docs/sframe-spec.html#The-SFrame-FRE-Types

kernel/unwind/sframe.c line 166

Quoted 1 line

kernel/unwind/sframe.c new file +1 -0

+	SFRAME_GET_USER_UNSIGNED(cfa_off, f, offset_size);

As far as I know the CFA offset from CFA base register is signed in the SFrame file format. See Binutils include/sframe-api.h, sframe_fre_get_cfa_offset(). Therefore use SFRAME_GET_USER_SIGNED(). Both cfa_off and struct user_unwind_frame cfa_off are already defined as s32.

Quoted 13 lines

+++ b/mm/init-mm.c

mm/init-mm.c +1 -1

@@ -11,6 +11,7 @@
  #include <linux/atomic.h>
  #include <linux/user_namespace.h>
  #include <linux/iommu.h>
+#include <linux/sframe.h>
  #include <asm/mmu.h>
  #ifndef INIT_MM_CONTEXT

@@ -44,7 +45,8 @@ struct mm_struct init_mm = {
  #endif
  	.user_ns	= &init_user_ns,
  	.cpu_bitmap	= CPU_BITS_NONE,
-	INIT_MM_CONTEXT(init_mm)

mm/init-mm.c struct mm_struct init_mm = { · line 49

Quoted 2 lines

mm/init-mm.c +2 -0

+	INIT_MM_CONTEXT(init_mm),
+	INIT_MM_SFRAME,

This does not compile on s390, as INIT_MM_CONTEXT() is defined with a trailing comma, leading to two consecutive commas. Already reported by the kernel test robot for other architectures.
Same if INIT_MM_SFRAME expands into nothing there would be two consecutive commas.

Hello,

kernel test robot noticed "kernel_BUG_at_kernel/unwind/user.c" on:

commit: 164c5ae4072303c9eb4e263115a5e70d5a3cc052 ("[PATCH v2 05/11] perf/x86: Use user_unwind interface")
url: https://github.com/intel-lab-lkp/linux/commits/Josh-Poimboeuf/unwind-Introduce-generic-user-space-unwinding-interface/20240914-070619
base: https://git.kernel.org/cgit/linux/kernel/git/perf/perf-tools-next.git perf-tools-next
patch link: https://lore.kernel.org/all/daf3f59e0d14ee11b45ad6735b8a211a3c7534dc.1726268190.git.jpoimboe@kernel.org/
patch subject: [PATCH v2 05/11] perf/x86: Use user_unwind interface

in testcase: fsmark
version: fsmark-x86_64-2628be5-1_20240224
with following parameters:

	iterations: 1x
	nr_threads: 64t
	disk: 1BRD_48G
	fs: xfs
	filesize: 4M
	test_size: 24G
	sync_method: NoSync
	cpufreq_governor: performance

compiler: gcc-12
test machine: 96 threads 2 sockets Intel(R) Xeon(R) Platinum 8260L CPU @ 2.40GHz (Cascade Lake) with 128G memory

(please refer to attached dmesg/kmsg for entire log/backtrace)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-lkp/[email protected]

[   53.036370][    C3] ------------[ cut here ]------------
[   53.036374][    C3] kernel BUG at kernel/unwind/user.c:39!

Instead of attempting to unwind user space from the NMI handler, defer it to run in task context by sending a self-IPI and then scheduling the unwind to run in the IRQ's exit task work before returning to user space.

This allows the user stack page to be paged in if needed, avoids duplicate unwinds for kernel-bound workloads, and prepares for SFrame unwinding (so .sframe sections can be paged in on demand).

Suggested-by Steven Rostedt <[email protected]>

Suggested-by Peter Zijlstra <[email protected]>

Signed-off-by Josh Poimboeuf <[email protected]>

---

kernel/events/core.c +27 -0

index 19fd7bd38ecf..5fc7c5156287 100644
@@ -6854,11 +6860,70 @@ static void perf_pending_irq(struct irq_work *entry)
 		perf_swevent_put_recursion_context(rctx);
 }
 
+struct perf_callchain_deferred_event {
+	struct perf_event_header	header;
+	struct perf_callchain_entry	callchain;
+};
+
+#define PERF_CALLCHAIN_DEFERRED_EVENT_SIZE				\
+	sizeof(struct perf_callchain_deferred_event) +			\
+	(sizeof(__u64) * 1) + /* PERF_CONTEXT_USER */			\
+	(sizeof(__u64) * PERF_MAX_STACK_DEPTH)
+
+static void perf_event_callchain_deferred(struct perf_event *event)
+{
+	struct pt_regs *regs = task_pt_regs(current);
+	struct perf_callchain_entry *callchain;
+	struct perf_output_handle handle;
+	struct perf_sample_data data;
+	unsigned char buf[PERF_CALLCHAIN_DEFERRED_EVENT_SIZE];
+	struct perf_callchain_entry_ctx ctx;
+	struct perf_callchain_deferred_event *deferred_event;
+
+	deferred_event = (void *)&buf;
+
+	callchain = &deferred_event->callchain;
+	callchain->nr = 0;
+
+	ctx.entry		= callchain;
+	ctx.max_stack		= MIN(event->attr.sample_max_stack,

kernel/events/core.c +22 -1

+	perf_event__output_id_sample(event, &handle, &data);
+	perf_output_end(&handle);
+}
+
 static void perf_pending_task(struct callback_head *head)
 {
 	struct perf_event *event = container_of(head, struct perf_event, pending_task);
 	int rctx;
 
+	if (!is_software_event(event)) {
+		if (event->pending_callchain) {
+			perf_event_callchain_deferred(event);
+			event->pending_callchain = 0;
+		}
+		return;
+	}
+
 	/*
 	 * All accesses to the event must belong to the same implicit RCU read-side
 	 * critical section as the ->pending_work reset. See comment in

@@ -7688,6 +7753,8 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
 	bool user   = !event->attr.exclude_callchain_user;
 	const u32 max_stack = event->attr.sample_max_stack;
 	struct perf_callchain_entry *callchain;
+	bool defer_user = IS_ENABLED(CONFIG_HAVE_PERF_CALLCHAIN_DEFERRED) &&
+			  event->attr.defer_callchain;
 
 	/* Disallow cross-task user callchains. */
 	user &= !event->ctx->task || event->ctx->task == current;

@@ -7695,7 +7762,14 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
 	if (!kernel && !user)
 		return &__empty_callchain;
 
-	callchain = get_perf_callchain(regs, kernel, user, max_stack, true);
+	callchain = get_perf_callchain(regs, kernel, user, max_stack, true,
+				       defer_user);
+
+	if (user && defer_user && !event->pending_callchain) {
+		event->pending_callchain = 1;
+		irq_work_queue(&event->pending_irq);
+	}
+
 	return callchain ?: &__empty_callchain;
 }
 

Signature · 1 line

2.46.0

This is a new version of sframe user space unwinding + perf deferred callchains. Better late than never.

Unfortunately I didn't get a chance to do any testing with this one yet as I'm rushing it out the door before GNU Cauldron starts. Conference-driven development ;-)

Plus I don't have the perf tool piece working yet so I don't have a way of doing end-to-end testing at the moment anyway.

In other words, please don't merge this yet.

Namhyung, if you're still available to write a perf tool patch which integrates with this, that would be great. Otherwise I could give it a try.

Steven, let me know if this would interface ok with your anticipated tracing usage.

I think the unwinder should have an interface itself that provides the deferred unwinding, instead of having all the tracers to implement their own.

The user space unwinder (regardless of sframe or not) should provide a way to say "I want a user space stack trace for here, but don't do it yet. Just give me a cookie and then call my callback function with the stack and cookie before going back to user space".

That is, we should have an interface like:

typedef void (unwinder_callback_t)(struct user_space_stack *, u64 cookie);

struct unwinder_client {
	struct list_head	list;
	unwinder_callback_t	callback;

Quoted 8 lines · Steven Rostedt

I think the unwinder should have an interface itself that provides the deferred unwinding, instead of having all the tracers to implement their own.

The user space unwinder (regardless of sframe or not) should provide a way to say "I want a user space stack trace for here, but don't do it yet. Just give me a cookie and then call my callback function with the stack and cookie before going back to user space".

We (Steven, Mathieu and I) have been discussing this at GNU Cauldron and I think we're in basic agreement on this.

I think the biggest tweak we decided on is that the context id (aka "cookie") would be percpu. Its initial value is (cpuid << 48). It gets incremented for every entry from user space.

Quoted 8 lines

That is, we should have an interface like:

typedef void (unwinder_callback_t)(struct user_space_stack *, u64 cookie);

struct unwinder_client {
	struct list_head	list;
	unwinder_callback_t	callback;  

I assume we want to allow tracers to pick sframes or FP (or auto). If so we would need to add a user_unwind_type enum to this struct.

Then the question is, what to do if tracer A wants sframe and tracer B wants FP? I'm thinking it's fine to allow that. I assume the "multiple tracers unwinding user space" case isn't realistic so any extra overhead from these cases is the user's fault?

The unwinder would need two sets of callbacks and task work functions:
one for sframe, one for FP.

The tracer would need to pass its &my_unwinder struct to unwinder_trigger() so the unwinder knows which task work function to activate.

I thinking we also need a 'max_entries' field. No need to keep unwinding if we've already reached the tracer's max. If there are multiple callbacks then we'd have to get the max of the maxes, but that's easy enough.

Mathieu had requested passing an opaque void * from the NMI to the callback, but I don't think that's possible with this scheme since the unwinder wouldn't know which callback to give it to. Instead the tracer would have to keep track of its own data associated with the given cookie.

Quoted 8 lines · Steven Rostedt

I think the unwinder should have an interface itself that provides the deferred unwinding, instead of having all the tracers to implement their own.

The user space unwinder (regardless of sframe or not) should provide a way to say "I want a user space stack trace for here, but don't do it yet. Just give me a cookie and then call my callback function with the stack and cookie before going back to user space".

We (Steven, Mathieu and I) have been discussing this at GNU Cauldron and I think we're in basic agreement on this.

I think the biggest tweak we decided on is that the context id (aka "cookie") would be percpu. Its initial value is (cpuid << 48). It gets incremented for every entry from user space.

Quoted 3 lines

That is, we should have an interface like:

typedef void (unwinder_callback_t)(struct user_space_stack *, u64 cookie);

Quoted 10 lines · Josh Poimboeuf

Quoted 8 lines · Steven Rostedt

I think the unwinder should have an interface itself that provides the deferred unwinding, instead of having all the tracers to implement their own.

The user space unwinder (regardless of sframe or not) should provide a way to say "I want a user space stack trace for here, but don't do it yet. Just give me a cookie and then call my callback function with the stack and cookie before going back to user space".

Quoted 6 lines

We (Steven, Mathieu and I) have been discussing this at GNU Cauldron and I think we're in basic agreement on this.

I think the biggest tweak we decided on is that the context id (aka "cookie") would be percpu. Its initial value is (cpuid << 48). It gets incremented for every entry from user space.

Why? What's the purpose of the cookie?

This scheme seems unsound

pin yourself on CPU0 and trigger 1<<48 unwinds while keeping CPU1 idle.

Quoted 3 lines

Quoted 3 lines

That is, we should have an interface like:

typedef void (unwinder_callback_t)(struct user_space_stack *, u64 cookie);

Just make it a void* and let the consumer figure it out.

The cookie is incremented once per entry from userspace, when needed.

It's a unique id used by the tracers which is emitted with both kernel and user stacktrace samples so the userspace tool can stitch them back together. Even if you have multiple tracers triggering at the same time they can all share a single user trace.

Quoted 3 lines

This scheme seems unsound

pin yourself on CPU0 and trigger 1<<48 unwinds while keeping CPU1 idle.

Hm???

Quoted 6 lines · Josh Poimboeuf

The cookie is incremented once per entry from userspace, when needed.

It's a unique id used by the tracers which is emitted with both kernel and user stacktrace samples so the userspace tool can stitch them back together. Even if you have multiple tracers triggering at the same time they can all share a single user trace.

But perf don't need this at all, right? It knows that the next deferred trace for that tid will be the one.

Quoted 6 lines

Quoted 4 lines

This scheme seems unsound

pin yourself on CPU0 and trigger 1<<48 unwinds while keeping CPU1 idle.

Hm???

The point being that it is possible to wrap one CPU into the id space of another CPU. It is not trivial, but someone who wants to can make it happen.

Combined I don't see the need to force this into this scheme, carry an opaque (void*) pointer and let the user do whatever it wants, perf for instance can pass NULL and not do anything like this.

Quoted 4 lines · On Mon, 16 Sep 2024 20:15:45 +0200 Peter Zijlstra <[email protected]> wrote:

Quoted 2 lines · Josh Poimboeuf

The cookie is incremented once per entry from userspace, when needed.

Quoted 7 lines

Quoted 4 lines

It's a unique id used by the tracers which is emitted with both kernel and user stacktrace samples so the userspace tool can stitch them back together. Even if you have multiple tracers triggering at the same time they can all share a single user trace.

But perf don't need this at all, right? It knows that the next deferred trace for that tid will be the one.

Is that because perf uses per task buffers? Will perf know this if it uses a global buffer? What does perf do with "-a"?

Hello,

Quoted 7 lines · Steven Rostedt

Quoted 4 lines · On Mon, 16 Sep 2024 20:15:45 +0200 Peter Zijlstra <[email protected]> wrote:

Quoted 2 lines · Josh Poimboeuf

The cookie is incremented once per entry from userspace, when needed.

Quoted 7 lines

Quoted 7 lines

Quoted 4 lines

It's a unique id used by the tracers which is emitted with both kernel and user stacktrace samples so the userspace tool can stitch them back together. Even if you have multiple tracers triggering at the same time they can all share a single user trace.

But perf don't need this at all, right? It knows that the next deferred trace for that tid will be the one.

Well.. technically you can sample without tid. But I'm not sure how much it'd be useful if you collect callchains without tid.

Quoted 1 line

Quoted 2 lines

Is that because perf uses per task buffers? Will perf know this if it uses a global buffer? What does perf do with "-a"?

Then it'd use per-cpu ring buffers. But each sample would have pid/tid pair and time so perf tools can match it with a deferred callchian.

The point being that it is possible to wrap one CPU into the id space of another CPU. It is not trivial, but someone who wants to can make it happen.

On 2024-09-16 20:15, Peter Zijlstra wrote:
[...]

Quoted 3 lines

The point being that it is possible to wrap one CPU into the id space of another CPU. It is not trivial, but someone who wants to can make it happen.

I agree that the overflow of the free-running counter bleeding into the CPU numbers is something we want to prevent. We don't care if this counter overflows after day/months/years for sake of correlation within a system call, but we do care about the fact that this free-running counter could be made to overflow after a very long time while the system runs, and then we reach a state where the CPU numbers are mixed up, which leads to short-term correlation issues.

I would recommend this layout for this 64-bit value instead:

low-bits: cpu number
high-bits: free-running counter

This way, we eliminate any carry from overflow into the cpu number bits.

Thanks,

Mathieu

Quoted 15 lines · Mathieu Desnoyers

On 2024-09-16 20:15, Peter Zijlstra wrote:
[...]

Quoted 3 lines

The point being that it is possible to wrap one CPU into the id space of another CPU. It is not trivial, but someone who wants to can make it happen.

I agree that the overflow of the free-running counter bleeding into the CPU numbers is something we want to prevent. We don't care if this counter overflows after day/months/years for sake of correlation within a system call, but we do care about the fact that this free-running counter could be made to overflow after a very long time while the system runs, and then we reach a state where the CPU numbers are mixed up, which leads to short-term correlation issues.

Quoted 6 lines

I would recommend this layout for this 64-bit value instead:

low-bits: cpu number
high-bits: free-running counter

This way, we eliminate any carry from overflow into the cpu number bits.

Even better: AFAIR from the discussion I had with Steven and Josh, we intend to have the cookie stored to/read from the task struct with interrupts off, we can simply do:

struct stackwalk_cookie {
    uint64_t counter;    /* free running per-cpu counter value */ int cpu;        /* cpu on which the counter was snapshot. */
};

So we don't have to deal with any overflow trickiness, there is no need for bit-shifting, and we get a full 64-bit free-running counter independently of the number of CPUs.

Thanks,

Mathieu

Quoted 6 lines

Thanks,

Mathieu

Completely untested patch here that I hacked up today:

git://git.kernel.org/pub/scm/linux/kernel/git/jpoimboe/linux.git sframe-2.1

To avoid races, unwind_user_deferred() can't be called from NMI. The tracers will need to trigger irq_work to call it.

+++ b/kernel/unwind/user.c

kernel/unwind/user.c +2 -1

@@ -177,7 +181,8 @@ int unwind_user_deferred(struct unwind_callback *callback, u64 *ctx_cookie)
 
 		cookie = __this_cpu_read(ctx_ctr);
 		cookie &= ((1UL << 48) - 1);
-		cookie |= ((cpu << 48) + 1);
+		cookie |= cpu << 48;
+		cookie++;
 		__this_cpu_write(ctx_ctr, cookie);
 
 		current->unwind_ctx_cookie = cookie;

As the cookie never got incremented.

That was just one issue. Things are still not working but I'll debug the rest later.