perf: user space sframe unwinding

The 'init_nr' argument has double duty: it's used to initialize both the number of contexts and the number of stack entries. That's confusing and the callers always pass zero anyway. Hard code the zero.

Signed-off-by Josh Poimboeuf <[email protected]>

---
 include/linux/perf_event.h |  2 +-
 kernel/bpf/stackmap.c      |  4 ++--
 kernel/events/callchain.c  | 12 ++++++------
 kernel/events/core.c       |  2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

include/linux/perf_event.h +1 -1

index afb028c54f33..f4b05954076c 100644
@@ -1533,7 +1533,7 @@ DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
 extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
 extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
 extern struct perf_callchain_entry *
-get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
+get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                   u32 max_stack, bool crosstask, bool add_mark);
 extern int get_callchain_buffers(int max_stack);
 extern void put_callchain_buffers(void);

kernel/bpf/stackmap.c +2 -2

index d6b277482085..b0b0fbff7c18 100644
@@ -294,7 +294,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
        if (max_depth > sysctl_perf_event_max_stack)
                max_depth = sysctl_perf_event_max_stack;
 
-       trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
+       trace = get_perf_callchain(regs, kernel, user, max_depth,
                                   false, false);
 
        if (unlikely(!trace))

@@ -420,7 +420,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
        else if (kernel && task)
                trace = get_callchain_entry_for_task(task, max_depth);
        else
-               trace = get_perf_callchain(regs, 0, kernel, user, max_depth,
+               trace = get_perf_callchain(regs, kernel, user, max_depth,
                                           false, false);
        if (unlikely(!trace))
                goto err_fault;

kernel/events/callchain.c +6 -6

index 1273be84392c..1e135195250c 100644
@@ -177,7 +177,7 @@ put_callchain_entry(int rctx)
 }
 
 struct perf_callchain_entry *
-get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
+get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                   u32 max_stack, bool crosstask, bool add_mark)
 {
        struct perf_callchain_entry *entry;

@@ -188,11 +188,11 @@ get_perf_callchain(struct pt_regs *regs, u32 init_nr, bool kernel, bool user,
        if (!entry)
                return NULL;
 
-       ctx.entry     = entry;
-       ctx.max_stack = max_stack;
-       ctx.nr        = entry->nr = init_nr;
-       ctx.contexts       = 0;
-       ctx.contexts_maxed = false;
+       ctx.entry               = entry;
+       ctx.max_stack           = max_stack;
+       ctx.nr                  = entry->nr = 0;
+       ctx.contexts            = 0;
+       ctx.contexts_maxed      = false;
 
        if (kernel && !user_mode(regs)) {
                if (add_mark)

kernel/events/core.c +1 -2

index 683dc086ef10..b0d62df7df4e 100644
@@ -7600,7 +7600,7 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
        if (!kernel && !user)
                return &__empty_callchain;
 
-       callchain = get_perf_callchain(regs, 0, kernel, user,
+       callchain = get_perf_callchain(regs, kernel, user,
                                       max_stack, crosstask, true);
        return callchain ?: &__empty_callchain;
 }
--

2.41.0

get_perf_callchain() doesn't support cross-task unwinding, so it doesn't

make much sense to have 'crosstask' as an argument. Instead, have perf_callchain() adjust 'user' accordingly.

Signed-off-by Josh Poimboeuf <[email protected]>

---
 include/linux/perf_event.h | 2 +-
 kernel/bpf/stackmap.c      | 5 ++---
 kernel/events/callchain.c  | 6 +-----
 kernel/events/core.c       | 8 ++++----
 4 files changed, 8 insertions(+), 13 deletions(-)

include/linux/perf_event.h +1 -1

index f4b05954076c..2d8fa253b9df 100644
@@ -1534,7 +1534,7 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct p
 extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
 extern struct perf_callchain_entry *
 get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool crosstask, bool add_mark);
+                  u32 max_stack, bool add_mark);
 extern int get_callchain_buffers(int max_stack);
 extern void put_callchain_buffers(void);
 extern struct perf_callchain_entry *get_callchain_entry(int *rctx);

kernel/bpf/stackmap.c +2 -3

index b0b0fbff7c18..e4827ca5378d 100644
@@ -294,8 +294,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
        if (max_depth > sysctl_perf_event_max_stack)
                max_depth = sysctl_perf_event_max_stack;
 
-       trace = get_perf_callchain(regs, kernel, user, max_depth,
-                                  false, false);
+       trace = get_perf_callchain(regs, kernel, user, max_depth, false);
 
        if (unlikely(!trace))
                /* couldn't fetch the stack trace */

@@ -421,7 +420,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                trace = get_callchain_entry_for_task(task, max_depth);
        else
                trace = get_perf_callchain(regs, kernel, user, max_depth,
-                                          false, false);
+                                          false);
        if (unlikely(!trace))
                goto err_fault;
 

kernel/events/callchain.c +1 -5

index 1e135195250c..aa5f9d11c28d 100644
@@ -178,7 +178,7 @@ put_callchain_entry(int rctx)
 
 struct perf_callchain_entry *
 get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool crosstask, bool add_mark)
+                  u32 max_stack, bool add_mark)
 {
        struct perf_callchain_entry *entry;
        struct perf_callchain_entry_ctx ctx;

@@ -209,9 +209,6 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                }
 
                if (regs) {
-                       if (crosstask)
-                               goto exit_put;
-
                        if (add_mark)
                                perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 

@@ -219,7 +216,6 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                }
        }
 
-exit_put:
        put_callchain_entry(rctx);
 
        return entry;

kernel/events/core.c +4 -5

index b0d62df7df4e..5e41a3b70bcd 100644
@@ -7592,16 +7592,16 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
 {
        bool kernel = !event->attr.exclude_callchain_kernel;
        bool user   = !event->attr.exclude_callchain_user;
-       /* Disallow cross-task user callchains. */
-       bool crosstask = event->ctx->task && event->ctx->task != current;
        const u32 max_stack = event->attr.sample_max_stack;
        struct perf_callchain_entry *callchain;
 
+       /* Disallow cross-task user callchains. */
+       user &= !event->ctx->task || event->ctx->task == current;
+
        if (!kernel && !user)
                return &__empty_callchain;
 
-       callchain = get_perf_callchain(regs, kernel, user,
-                                      max_stack, crosstask, true);
+       callchain = get_perf_callchain(regs, kernel, user, max_stack, true);
        return callchain ?: &__empty_callchain;
 }
 
--

2.41.0

Quoted 2 lines · Josh Poimboeuf

get_perf_callchain() doesn't support cross-task unwinding, so it doesn't

For only user stacks, but it seems there's no place to get cross-task kernel stacks too.

Quoted 4 lines

make much sense to have 'crosstask' as an argument. Instead, have perf_callchain() adjust 'user' accordingly.

Signed-off-by Josh Poimboeuf <[email protected]>

Acked-by Namhyung Kim <[email protected]>

Thanks,
Namhyung

Quoted 102 lines

---
  include/linux/perf_event.h | 2 +-
  kernel/bpf/stackmap.c      | 5 ++---
  kernel/events/callchain.c  | 6 +-----
  kernel/events/core.c       | 8 ++++----
  4 files changed, 8 insertions(+), 13 deletions(-)

include/linux/perf_event.h +1 -1

index f4b05954076c..2d8fa253b9df 100644
@@ -1534,7 +1534,7 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct p
  extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
  extern struct perf_callchain_entry *
  get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool crosstask, bool add_mark);
+                  u32 max_stack, bool add_mark);
  extern int get_callchain_buffers(int max_stack);
  extern void put_callchain_buffers(void);
  extern struct perf_callchain_entry *get_callchain_entry(int *rctx);

kernel/bpf/stackmap.c +2 -3

index b0b0fbff7c18..e4827ca5378d 100644
@@ -294,8 +294,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
         if (max_depth > sysctl_perf_event_max_stack)
                 max_depth = sysctl_perf_event_max_stack;
 
-       trace = get_perf_callchain(regs, kernel, user, max_depth,
-                                  false, false);
+       trace = get_perf_callchain(regs, kernel, user, max_depth, false);
 
         if (unlikely(!trace))
                 /* couldn't fetch the stack trace */

@@ -421,7 +420,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                 trace = get_callchain_entry_for_task(task, max_depth);
         else
                 trace = get_perf_callchain(regs, kernel, user, max_depth,
-                                          false, false);
+                                          false);
         if (unlikely(!trace))
                 goto err_fault;
 

kernel/events/callchain.c +1 -5

index 1e135195250c..aa5f9d11c28d 100644
@@ -178,7 +178,7 @@ put_callchain_entry(int rctx)
 
  struct perf_callchain_entry *
  get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool crosstask, bool add_mark)
+                  u32 max_stack, bool add_mark)
  {
         struct perf_callchain_entry *entry;
         struct perf_callchain_entry_ctx ctx;

@@ -209,9 +209,6 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                 }
 
                 if (regs) {
-                       if (crosstask)
-                               goto exit_put;
-
                         if (add_mark)
                                 perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 

@@ -219,7 +216,6 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                 }
         }
 
-exit_put:
         put_callchain_entry(rctx);
 
         return entry;

kernel/events/core.c +4 -5

index b0d62df7df4e..5e41a3b70bcd 100644
@@ -7592,16 +7592,16 @@ perf_callchain(struct perf_event *event, struct pt_regs *regs)
  {
         bool kernel = !event->attr.exclude_callchain_kernel;
         bool user   = !event->attr.exclude_callchain_user;
-       /* Disallow cross-task user callchains. */
-       bool crosstask = event->ctx->task && event->ctx->task != current;
         const u32 max_stack = event->attr.sample_max_stack;
         struct perf_callchain_entry *callchain;
 
+       /* Disallow cross-task user callchains. */
+       user &= !event->ctx->task || event->ctx->task == current;
+
         if (!kernel && !user)
                 return &__empty_callchain;
 
-       callchain = get_perf_callchain(regs, kernel, user,
-                                      max_stack, crosstask, true);
+       callchain = get_perf_callchain(regs, kernel, user, max_stack, true);
         return callchain ?: &__empty_callchain;
  }
 
--

2.41.0

Simplify the get_perf_callchain() user logic a bit. task_pt_regs() should never be NULL.

Signed-off-by Josh Poimboeuf <[email protected]>

---
 kernel/events/callchain.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

kernel/events/callchain.c +7 -10

index aa5f9d11c28d..2bee8b6fda0e 100644
@@ -202,20 +202,18 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
 
        if (user) {
                if (!user_mode(regs)) {
-                       if  (current->mm)
-                               regs = task_pt_regs(current);
-                       else
-                               regs = NULL;
+                       if (!current->mm)
+                               goto exit_put;
+                       regs = task_pt_regs(current);
                }
 
-               if (regs) {
-                       if (add_mark)
-                               perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
+               if (add_mark)
+                       perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 
-                       perf_callchain_user(&ctx, regs);
-               }
+               perf_callchain_user(&ctx, regs);
        }
 
+exit_put:
        put_callchain_entry(rctx);
 
        return entry;
--

2.41.0

Instead of attempting to unwind user space from the NMI handler, defer it to run in task context by sending a self-IPI and then scheduling the unwind to run in the IRQ's exit task work before returning to user space.

This allows the user stack page to be paged in if needed, avoids duplicate unwinds for kernel-bound workloads, and prepares for SFrame unwinding (so .sframe sections can be paged in on demand).

Suggested-by Steven Rostedt <[email protected]>

Suggested-by Peter Zijlstra <[email protected]>

Signed-off-by Josh Poimboeuf <[email protected]>

---
 arch/Kconfig                    |  3 ++
 include/linux/perf_event.h      | 22 ++++++--
 include/uapi/linux/perf_event.h |  1 +
 kernel/bpf/stackmap.c           |  5 +-
 kernel/events/callchain.c       |  7 ++-
 kernel/events/core.c            | 90 ++++++++++++++++++++++++++++++---
 6 files changed, 115 insertions(+), 13 deletions(-)

arch/Kconfig +3 -0

index f4b210ab0612..690c82212224 100644
@@ -425,6 +425,9 @@ config HAVE_HARDLOCKUP_DETECTOR_ARCH
          It uses the same command line parameters, and sysctl interface,
          as the generic hardlockup detectors.
 
+config HAVE_PERF_CALLCHAIN_DEFERRED
+       bool
+
 config HAVE_PERF_REGS
        bool
        help

include/linux/perf_event.h +19 -3

index 2d8fa253b9df..2f232111dff2 100644
@@ -786,6 +786,7 @@ struct perf_event {
        struct irq_work                 pending_irq;
        struct callback_head            pending_task;
        unsigned int                    pending_work;
+       unsigned int                    pending_unwind;
 
        atomic_t                        event_limit;
 

@@ -1113,7 +1114,10 @@ int perf_event_read_local(struct perf_event *event, u64 *value,
 extern u64 perf_event_read_value(struct perf_event *event,
                                 u64 *enabled, u64 *running);
 
-extern struct perf_callchain_entry *perf_callchain(struct perf_event *event, struct pt_regs *regs);
+extern void perf_callchain(struct perf_sample_data *data,
+                          struct perf_event *event, struct pt_regs *regs);
+extern void perf_callchain_deferred(struct perf_sample_data *data,
+                                   struct perf_event *event, struct pt_regs *regs);
 
 static inline bool branch_sample_no_flags(const struct perf_event *event)
 {

@@ -1189,6 +1193,7 @@ struct perf_sample_data {
        u64                             data_page_size;
        u64                             code_page_size;
        u64                             aux_size;
+       bool                            deferred;
 } ____cacheline_aligned;
 
 /* default value for data source */

@@ -1206,6 +1211,7 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
        data->sample_flags = PERF_SAMPLE_PERIOD;
        data->period = period;
        data->dyn_size = 0;
+       data->deferred = false;
 
        if (addr) {
                data->addr = addr;

@@ -1219,7 +1225,11 @@ static inline void perf_sample_save_callchain(struct perf_sample_data *data,
 {
        int size = 1;
 
-       data->callchain = perf_callchain(event, regs);
+       if (data->deferred)
+               perf_callchain_deferred(data, event, regs);
+       else
+               perf_callchain(data, event, regs);
+
        size += data->callchain->nr;
 
        data->dyn_size += size * sizeof(u64);

@@ -1534,12 +1544,18 @@ extern void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct p
 extern void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
 extern struct perf_callchain_entry *
 get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool add_mark);
+                  u32 max_stack, bool add_mark, bool defer_user);
 extern int get_callchain_buffers(int max_stack);
 extern void put_callchain_buffers(void);
 extern struct perf_callchain_entry *get_callchain_entry(int *rctx);
 extern void put_callchain_entry(int rctx);
 
+#ifdef CONFIG_HAVE_PERF_CALLCHAIN_DEFERRED
+extern void perf_callchain_user_deferred(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs);
+#else
+static inline void perf_callchain_user_deferred(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) {}
+#endif
+
 extern int sysctl_perf_event_max_stack;
 extern int sysctl_perf_event_max_contexts_per_stack;
 

include/uapi/linux/perf_event.h +1 -0

index 39c6a250dd1b..9a1127af4cda 100644
@@ -1237,6 +1237,7 @@ enum perf_callchain_context {
        PERF_CONTEXT_HV                 = (__u64)-32,
        PERF_CONTEXT_KERNEL             = (__u64)-128,
        PERF_CONTEXT_USER               = (__u64)-512,
+       PERF_CONTEXT_USER_DEFERRED      = (__u64)-640,
 
        PERF_CONTEXT_GUEST              = (__u64)-2048,
        PERF_CONTEXT_GUEST_KERNEL       = (__u64)-2176,

kernel/bpf/stackmap.c +1 -2

index e4827ca5378d..fcdd26715b12 100644
@@ -294,8 +294,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
        if (max_depth > sysctl_perf_event_max_stack)
                max_depth = sysctl_perf_event_max_stack;
 
-       trace = get_perf_callchain(regs, kernel, user, max_depth, false);
-
+       trace = get_perf_callchain(regs, kernel, user, max_depth, false, false);
        if (unlikely(!trace))
                /* couldn't fetch the stack trace */
                return -EFAULT;

kernel/bpf/stackmap.c +0 -0

        if (unlikely(!trace))
                goto err_fault;
 

kernel/events/callchain.c +1 -1

index 2bee8b6fda0e..16571c8d6771 100644
@@ -178,7 +178,7 @@ put_callchain_entry(int rctx)
 
 struct perf_callchain_entry *
 get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool add_mark)
+                  u32 max_stack, bool add_mark, bool defer_user)
 {
        struct perf_callchain_entry *entry;
        struct perf_callchain_entry_ctx ctx;

kernel/events/callchain.c +2 -0

+               }
+
                if (add_mark)
                        perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 

kernel/events/core.c +0 -0

index 5e41a3b70bcd..290e06b0071c 100644

kernel/events/core.c +5 -0

+               return;
+       }
+
        /*
         * If we 'fail' here, that's OK, it means recursion is already disabled
         * and we won't recurse 'further'.

@@ -6772,11 +6778,57 @@ static void perf_pending_irq(struct irq_work *entry)
                perf_swevent_put_recursion_context(rctx);
 }
 
+static void perf_pending_task_unwind(struct perf_event *event)
+{

kernel/events/core.c +23 -0

+
+       perf_prepare_header(&header, &data, event, regs);
+
+       if (perf_output_begin(&handle, &data, event, header.size))
+               goto done;
+
+       perf_output_sample(&handle, &header, &data, event);
+
+       perf_output_end(&handle);
+
+done:
+       kfree(callchain);
+}
+
+
 static void perf_pending_task(struct callback_head *head)
 {
        struct perf_event *event = container_of(head, struct perf_event, pending_task);
        int rctx;
 
+       if (!is_software_event(event)) {
+               if (event->pending_unwind) {
+                       perf_pending_task_unwind(event);
+                       event->pending_unwind = 0;
+               }
+               return;
+       }
+
        /*
         * If we 'fail' here, that's OK, it means recursion is already disabled
         * and we won't recurse 'further'.

@@ -7587,22 +7639,48 @@ static u64 perf_get_page_size(unsigned long addr)
 
 static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
 

kernel/events/core.c +29 -4

        /* Disallow cross-task user callchains. */
        user &= !event->ctx->task || event->ctx->task == current;
 
        if (!kernel && !user)
-               return &__empty_callchain;
+               goto empty;
 
-       callchain = get_perf_callchain(regs, kernel, user, max_stack, true);
-       return callchain ?: &__empty_callchain;
+       data->callchain = get_perf_callchain(regs, kernel, user, max_stack, true, defer_user);
+       if (!data->callchain)
+               goto empty;
+
+       if (user && defer_user && !event->pending_unwind) {
+               event->pending_unwind = 1;
+               irq_work_queue(&event->pending_irq);
+       }
+
+       return;
+
+empty:
+       data->callchain = &__empty_callchain;
+}
+
+void perf_callchain_deferred(struct perf_sample_data *data,
+                            struct perf_event *event, struct pt_regs *regs)
+{
+       struct perf_callchain_entry_ctx ctx;
+
+       ctx.entry               = data->callchain;
+       ctx.max_stack           = event->attr.sample_max_stack;
+       ctx.nr                  = 0;
+       ctx.contexts            = 0;
+       ctx.contexts_maxed      = false;
+
+       perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
+       perf_callchain_user_deferred(&ctx, regs);
 }
 
 static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d)
--

2.41.0

Quoted 5 lines · Josh Poimboeuf

+++ b/kernel/bpf/stackmap.c

kernel/bpf/stackmap.c +0 -0

@@ -294,8 +294,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
        if (max_depth > sysctl_perf_event_max_stack)
                max_depth = sysctl_perf_event_max_stack;
 

kernel/bpf/stackmap.c static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · line 422

Quoted 12 lines

kernel/bpf/stackmap.c +2 -3

-       trace = get_perf_callchain(regs, kernel, user, max_depth, false);
-
+       trace = get_perf_callchain(regs, kernel, user, max_depth, false, false);
        if (unlikely(!trace))
                /* couldn't fetch the stack trace */
                return -EFAULT;

@@ -420,7 +419,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                trace = get_callchain_entry_for_task(task, max_depth);
        else
                trace = get_perf_callchain(regs, kernel, user, max_depth,
-                                          false);
+                                          false, false);

Hmm.. BPF is not supported. Any plans?

Quoted 14 lines

kernel/events/callchain.c +2 -1

+++ b/kernel/events/callchain.c

@@ -178,7 +178,7 @@ put_callchain_entry(int rctx)
 
 struct perf_callchain_entry *
 get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool add_mark)
+                  u32 max_stack, bool add_mark, bool defer_user)
 {
        struct perf_callchain_entry *entry;
        struct perf_callchain_entry_ctx ctx;

@@ -207,6 +207,11 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                        regs = task_pt_regs(current);
                }
 

kernel/events/callchain.c get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, · line 212

Quoted 3 lines

kernel/events/callchain.c +3 -0

+               if (defer_user) {
+                       perf_callchain_store_context(&ctx, PERF_CONTEXT_USER_DEFERRED);
+                       goto exit_put;

Hmm.. ok. Then now perf tools need to stitch the call stacks in two (or more?) samples.

Quoted 13 lines

kernel/events/callchain.c +2 -0

+               }
+
                if (add_mark)
                        perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 

kernel/events/core.c +0 -0

index 5e41a3b70bcd..290e06b0071c 100644
@@ -6751,6 +6751,12 @@ static void perf_pending_irq(struct irq_work *entry)
        struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
        int rctx;
 

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6756

Quoted 3 lines

kernel/events/core.c +3 -0

+       if (!is_software_event(event)) {
+               if (event->pending_unwind)
+                       task_work_add(current, &event->pending_task, TWA_RESUME);

I'm not familiar with this code. Is it possible to the current task is preempted before returning to user space (and run the perf_pending_task_unwind) ?

Quoted 7 lines

kernel/events/core.c +7 -0

+static void perf_pending_task_unwind(struct perf_event *event)
+{
+       struct pt_regs *regs = task_pt_regs(current);
+       struct perf_output_handle handle;
+       struct perf_event_header header;
+       struct perf_sample_data data;
+       struct perf_callchain_entry *callchain;

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6792

Quoted 5 lines

kernel/events/core.c +5 -0

+
+       callchain = kmalloc(sizeof(struct perf_callchain_entry) +
+                           (sizeof(__u64) * event->attr.sample_max_stack) +
+                           (sizeof(__u64) * 1) /* one context */,
+                           GFP_KERNEL);

Any chance it can reuse get_perf_callchain() instead of allocating the callchains every time?

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6799

Quoted 7 lines

kernel/events/core.c +7 -0

+       if (!callchain)
+               return;
+
+       callchain->nr = 0;
+       data.callchain = callchain;
+
+       perf_sample_data_init(&data, 0, event->hw.last_period);

It would double count the period for a sample.
I guess we want to set the period to 0.

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6803

Quoted 4 lines

kernel/events/core.c +4 -0

+
+       data.deferred = true;
+
+       perf_prepare_sample(&data, event, regs);

I don't think this would work properly. Not to mention it duplicates sample data unnecessarily, some data needs special handling to avoid inefficient (full) data copy like for (user) stack, regs and aux.

Anyway I'm not sure it can support these additional samples for deferred callchains without breaking the existing perf tools. Anyway it doesn't know PERF_CONTEXT_USER_DEFERRED at least. I think this should be controlled by a new feature bit in the perf_event_attr.

Then we might add a breaking change to have a special sample record for the deferred callchains and sample ID only.

Quoted 5 lines

kernel/events/core.c +2 -2

-struct perf_callchain_entry *
-perf_callchain(struct perf_event *event, struct pt_regs *regs)
+void perf_callchain(struct perf_sample_data *data, struct perf_event *event,
+                   struct pt_regs *regs)
 {

kernel/events/core.c static u64 perf_get_page_size(unsigned long addr) · line 7648

Quoted 5 lines

        bool kernel = !event->attr.exclude_callchain_kernel;
        bool user   = !event->attr.exclude_callchain_user;
        const u32 max_stack = event->attr.sample_max_stack;

kernel/events/core.c +1 -1

-       struct perf_callchain_entry *callchain;
+       bool defer_user = IS_ENABLED(CONFIG_HAVE_PERF_CALLCHAIN_DEFERRED);

Is it always enabled depending on the kernel config? It could be controlled by event->attr.something..

Quoted 7 lines · Namhyung Kim

Quoted 7 lines

kernel/events/core.c +7 -0

+static void perf_pending_task_unwind(struct perf_event *event)
+{
+       struct pt_regs *regs = task_pt_regs(current);
+       struct perf_output_handle handle;
+       struct perf_event_header header;
+       struct perf_sample_data data;
+       struct perf_callchain_entry *callchain;

Quoted 8 lines

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6792

Quoted 5 lines

kernel/events/core.c +5 -0

+
+       callchain = kmalloc(sizeof(struct perf_callchain_entry) +
+                           (sizeof(__u64) * event->attr.sample_max_stack) +
+                           (sizeof(__u64) * 1) /* one context */,
+                           GFP_KERNEL);

Any chance it can reuse get_perf_callchain() instead of allocating the callchains every time?

I don't think so, because if it gets preempted, the new task might also need to do an unwind. But there's only one task-context callchain per CPU.

Quoted 6 lines · Namhyung Kim

Quoted 5 lines · Josh Poimboeuf

+++ b/kernel/bpf/stackmap.c

kernel/bpf/stackmap.c +0 -0

@@ -294,8 +294,7 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map,
        if (max_depth > sysctl_perf_event_max_stack)
                max_depth = sysctl_perf_event_max_stack;
 

Quoted 14 lines

kernel/bpf/stackmap.c static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, · line 422

Quoted 12 lines

kernel/bpf/stackmap.c +2 -3

-       trace = get_perf_callchain(regs, kernel, user, max_depth, false);
-
+       trace = get_perf_callchain(regs, kernel, user, max_depth, false, false);
        if (unlikely(!trace))
                /* couldn't fetch the stack trace */
                return -EFAULT;

@@ -420,7 +419,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task,
                trace = get_callchain_entry_for_task(task, max_depth);
        else
                trace = get_perf_callchain(regs, kernel, user, max_depth,
-                                          false);
+                                          false, false);

Hmm.. BPF is not supported. Any plans?

I'm not sure whether this feature makes sense for BPF. If the BPF program runs in atomic context then it would have to defer the unwind until right before exiting to user space. But then the program is no longer running.

Quoted 15 lines

Quoted 15 lines

kernel/events/callchain.c +2 -1

+++ b/kernel/events/callchain.c

@@ -178,7 +178,7 @@ put_callchain_entry(int rctx)
 
 struct perf_callchain_entry *
 get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
-                  u32 max_stack, bool add_mark)
+                  u32 max_stack, bool add_mark, bool defer_user)
 {
        struct perf_callchain_entry *entry;
        struct perf_callchain_entry_ctx ctx;

@@ -207,6 +207,11 @@ get_perf_callchain(struct pt_regs *regs, bool kernel, bool user,
                        regs = task_pt_regs(current);
                }
 

Quoted 6 lines

kernel/events/callchain.c get_perf_callchain(struct pt_regs *regs, bool kernel, bool user, · line 212

Quoted 3 lines

kernel/events/callchain.c +3 -0

+               if (defer_user) {
+                       perf_callchain_store_context(&ctx, PERF_CONTEXT_USER_DEFERRED);
+                       goto exit_put;

Hmm.. ok. Then now perf tools need to stitch the call stacks in two (or more?) samples.

Yes. And it could be more than two samples if there were multiple NMIs before returning to user space. In that case there would be a single user sample which needs to be stitched to each of the kernel samples.

Quoted 14 lines

Quoted 14 lines

kernel/events/callchain.c +2 -0

+               }
+
                if (add_mark)
                        perf_callchain_store_context(&ctx, PERF_CONTEXT_USER);
 

kernel/events/core.c +0 -0

index 5e41a3b70bcd..290e06b0071c 100644
@@ -6751,6 +6751,12 @@ static void perf_pending_irq(struct irq_work *entry)
        struct perf_event *event = container_of(entry, struct perf_event, pending_irq);
        int rctx;
 

Quoted 7 lines

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6756

Quoted 3 lines

kernel/events/core.c +3 -0

+       if (!is_software_event(event)) {
+               if (event->pending_unwind)
+                       task_work_add(current, &event->pending_task, TWA_RESUME);

I'm not familiar with this code. Is it possible to the current task is preempted before returning to user space (and run the perf_pending_task_unwind) ?

Yes, the task work (perf_pending_task_unwind) is preemptible.

Quoted 16 lines

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6792

Quoted 13 lines

kernel/events/core.c +12 -0

+static void perf_pending_task_unwind(struct perf_event *event)
+{
+       struct pt_regs *regs = task_pt_regs(current);
+       struct perf_output_handle handle;
+       struct perf_event_header header;
+       struct perf_sample_data data;
+       struct perf_callchain_entry *callchain;
+
+       callchain = kmalloc(sizeof(struct perf_callchain_entry) +
+                           (sizeof(__u64) * event->attr.sample_max_stack) +
+                           (sizeof(__u64) * 1) /* one context */,
+                           GFP_KERNEL);

Any chance it can reuse get_perf_callchain() instead of allocating the callchains every time?

I don't think so, because if it gets preempted, the new task might also need to do an unwind. But there's only one task-context callchain per CPU.

The allocation is less than ideal. I could just allocate it on the stack, and keep the number of entries bounded to 128 entries or so.

Quoted 11 lines

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6799

Quoted 8 lines

kernel/events/core.c +7 -0

+       if (!callchain)
+               return;
+
+       callchain->nr = 0;
+       data.callchain = callchain;
+
+       perf_sample_data_init(&data, 0, event->hw.last_period);

It would double count the period for a sample.
I guess we want to set the period to 0.

Ok.

Quoted 8 lines

kernel/events/core.c static void perf_pending_irq(struct irq_work *entry) · line 6803

Quoted 4 lines

kernel/events/core.c +4 -0

+
+       data.deferred = true;
+
+       perf_prepare_sample(&data, event, regs);

I don't think this would work properly. Not to mention it duplicates sample data unnecessarily, some data needs special handling to avoid inefficient (full) data copy like for (user) stack, regs and aux.

Yeah. I tried sending only the sample ID and callchain, but perf tool didn't appreciate that ;-) So for the RFC I gave up and did this.

Quoted 7 lines

Anyway I'm not sure it can support these additional samples for deferred callchains without breaking the existing perf tools. Anyway it doesn't know PERF_CONTEXT_USER_DEFERRED at least. I think this should be controlled by a new feature bit in the perf_event_attr.

Quoted 2 lines

Then we might add a breaking change to have a special sample record for the deferred callchains and sample ID only.

Sounds like a good idea. I'll need to study the code to figure out how to do that on the perf tool side. Or would you care to write a patch?

Quoted 14 lines

kernel/events/core.c static u64 perf_get_page_size(unsigned long addr) · line 7648

Quoted 11 lines

kernel/events/core.c +3 -3

-struct perf_callchain_entry *
-perf_callchain(struct perf_event *event, struct pt_regs *regs)
+void perf_callchain(struct perf_sample_data *data, struct perf_event *event,
+                   struct pt_regs *regs)
 {
        bool kernel = !event->attr.exclude_callchain_kernel;
        bool user   = !event->attr.exclude_callchain_user;
        const u32 max_stack = event->attr.sample_max_stack;
-       struct perf_callchain_entry *callchain;
+       bool defer_user = IS_ENABLED(CONFIG_HAVE_PERF_CALLCHAIN_DEFERRED);

Is it always enabled depending on the kernel config? It could be controlled by event->attr.something..

Possibly, depending on what y'all think. Also, fp vs sframe could be an attr (though sframe implies deferred).

Thanks for the review!

Ok, then BPF gets no user call stacks even with sframes.

Ok, but ...

If the task is preempted, the call stack would be from another task (if it also has the pending call stacks) and we need to check which user call stack matches which kernel call stack. So there's no guarantee we can just use adjacent samples.

Quoted 1 line · Namhyung Kim

Ok, then BPF gets no user call stacks even with sframes.

Well, you could obviously run another BPF program at return to user and stitch there.

Quoted 7 lines

Ok, but ...

If the task is preempted, the call stack would be from another task (if it also has the pending call stacks) and we need to check which user call stack matches which kernel call stack. So there's no guarantee we can just use adjacent samples.

So upon requesting a user backtrace for the task it could request a token (from eg a global atomic u64 in the most crude case) and place this in the task_struct. The backtrace will emit this as forward reference for the user trace.

Once the task_work generates the user stacktrace, it will again dump this token along with the user unwind, after which it will reset the token for this this task (to 0).

Yeah, I thought something like this first, but then I thought "can we just use PID for this?"

Quoted 2 lines · Namhyung Kim

Yeah, I thought something like this first, but then I thought "can we just use PID for this?"

TID, and assuming things are otherwise time ordered, yes.

Quoted 8 lines · Josh Poimboeuf

Quoted 6 lines · Namhyung Kim

Anyway I'm not sure it can support these additional samples for deferred callchains without breaking the existing perf tools. Anyway it doesn't know PERF_CONTEXT_USER_DEFERRED at least. I think this should be controlled by a new feature bit in the perf_event_attr.

Quoted 5 lines

Quoted 2 lines

Then we might add a breaking change to have a special sample record for the deferred callchains and sample ID only.

Sounds like a good idea. I'll need to study the code to figure out how to do that on the perf tool side. Or would you care to write a patch?

Sure, I'd be happy to write one.

---8<---

include/uapi/linux/perf_event.h +2 -1

index 39c6a250dd1b..a3765ff59798 100644
@@ -456,7 +456,8 @@ struct perf_event_attr {
 				inherit_thread :  1, /* children only inherit if cloned with CLONE_THREAD */
 				remove_on_exec :  1, /* event is removed from task on exec */
 				sigtrap        :  1, /* send synchronous SIGTRAP on event */
-				__reserved_1   : 26;
+				defer_callchain:  1, /* generate DEFERRED_CALLCHAINS records for userspace */
+				__reserved_1   : 25;
 
 	union {
 		__u32		wakeup_events;	  /* wakeup every n events */

include/uapi/linux/perf_event.h +5 -0

+	 * have kernel callchains only and they need to be stitched with this
+	 * to make full callchains.
+	 *
+	 * struct {
+	 *	struct perf_event_header	header;

Quoted 19 lines · Namhyung Kim

---8<---

include/uapi/linux/perf_event.h +2 -1

index 39c6a250dd1b..a3765ff59798 100644
@@ -456,7 +456,8 @@ struct perf_event_attr {
 				inherit_thread :  1, /* children only inherit if cloned with CLONE_THREAD */
 				remove_on_exec :  1, /* event is removed from task on exec */
 				sigtrap        :  1, /* send synchronous SIGTRAP on event */
-				__reserved_1   : 26;
+				defer_callchain:  1, /* generate DEFERRED_CALLCHAINS records for userspace */
+				__reserved_1   : 25;
 
 	union {
 		__u32		wakeup_events;	  /* wakeup every n events */

@@ -1207,6 +1208,20 @@ enum perf_event_type {
 	 */
 	PERF_RECORD_AUX_OUTPUT_HW_ID		= 21,
 

include/uapi/linux/perf_event.h enum perf_event_type { · line 1212

Quoted 2 lines

include/uapi/linux/perf_event.h +2 -0

+	/*
+	 * Deferred user stack callchains (for SFrame).  Previous samples would

Possibly also useful for ShadowStack based unwinders. And by virtue of it possibly saving work when multiple consecutive samples hit the same kernel section, for everything.

Quoted 21 lines · Peter Zijlstra

Quoted 19 lines · Namhyung Kim

---8<---

include/uapi/linux/perf_event.h +2 -1

index 39c6a250dd1b..a3765ff59798 100644
@@ -456,7 +456,8 @@ struct perf_event_attr {
 				inherit_thread :  1, /* children only inherit if cloned with CLONE_THREAD */
 				remove_on_exec :  1, /* event is removed from task on exec */
 				sigtrap        :  1, /* send synchronous SIGTRAP on event */
-				__reserved_1   : 26;
+				defer_callchain:  1, /* generate DEFERRED_CALLCHAINS records for userspace */
+				__reserved_1   : 25;
 
 	union {
 		__u32		wakeup_events;	  /* wakeup every n events */

@@ -1207,6 +1208,20 @@ enum perf_event_type {
 	 */
 	PERF_RECORD_AUX_OUTPUT_HW_ID		= 21,
 

Quoted 6 lines

include/uapi/linux/perf_event.h enum perf_event_type { · line 1212

Quoted 2 lines

include/uapi/linux/perf_event.h +2 -0

+	/*
+	 * Deferred user stack callchains (for SFrame).  Previous samples would

Possibly also useful for ShadowStack based unwinders. And by virtue of it possibly saving work when multiple consecutive samples hit the same kernel section, for everything.

[ necroing old thread as I'm finally working on a v2 ]

Peter, can you elaborate? What did you mean by "same kernel section"?

Like if there's a duplicate kernel callchain? Or something else?

Quoted 29 lines · Josh Poimboeuf

Quoted 27 lines · Peter Zijlstra

Quoted 16 lines · Namhyung Kim

---8<---

include/uapi/linux/perf_event.h +2 -1

index 39c6a250dd1b..a3765ff59798 100644
@@ -456,7 +456,8 @@ struct perf_event_attr {
 				inherit_thread :  1, /* children only inherit if cloned with CLONE_THREAD */
 				remove_on_exec :  1, /* event is removed from task on exec */
 				sigtrap        :  1, /* send synchronous SIGTRAP on event */
-				__reserved_1   : 26;
+				defer_callchain:  1, /* generate DEFERRED_CALLCHAINS records for userspace */
+				__reserved_1   : 25;
 
 	union {
 		__u32		wakeup_events;	  /* wakeup every n events */

include/uapi/linux/perf_event.h enum perf_event_type { · line 1212

Quoted 5 lines

include/uapi/linux/perf_event.h +2 -0

@@ -1207,6 +1208,20 @@ enum perf_event_type {
 	 */
 	PERF_RECORD_AUX_OUTPUT_HW_ID		= 21,
 
+	/*
+	 * Deferred user stack callchains (for SFrame).  Previous samples would

Possibly also useful for ShadowStack based unwinders. And by virtue of it possibly saving work when multiple consecutive samples hit the same kernel section, for everything.

Quoted 5 lines

[ necroing old thread as I'm finally working on a v2 ]

Peter, can you elaborate? What did you mean by "same kernel section"?

Like if there's a duplicate kernel callchain? Or something else?

Yeah, multiple samples hitting the same syscall invocation will, by necessity, have the same user callchain.

Quoted 10 lines · Peter Zijlstra

Quoted 9 lines · Josh Poimboeuf

Quoted 7 lines · Peter Zijlstra

include/uapi/linux/perf_event.h enum perf_event_type { · line 1212

Quoted 2 lines · Namhyung Kim

include/uapi/linux/perf_event.h +2 -0

+	/*
+	 * Deferred user stack callchains (for SFrame).  Previous samples would

Possibly also useful for ShadowStack based unwinders. And by virtue of it possibly saving work when multiple consecutive samples hit the same kernel section, for everything.

Quoted 8 lines

Quoted 5 lines

[ necroing old thread as I'm finally working on a v2 ]

Peter, can you elaborate? What did you mean by "same kernel section"?

Like if there's a duplicate kernel callchain? Or something else?

Yeah, multiple samples hitting the same syscall invocation will, by necessity, have the same user callchain.

Or if it would instead make sense to combine all the deferred user options into a single variable length PERF_RECORD_USER_DEFERRED event type?

Some distros have started compiling frame pointers into all their packages to enable the kernel to do system-wide profiling of user space. Unfortunately that creates a runtime performance penalty across the entire system. Using DWARF instead isn't feasible due to complexity and slowness.

For in-kernel unwinding we solved this problem with the creation of the ORC unwinder for x86_64. Similarly, for user space the GNU assembler has created the SFrame format starting with binutils 2.40. SFrame is a simpler version of .eh_frame which gets placed in the .sframe section.

Add support for unwinding user space using SFrame.

More information about SFrame can be found here:

- https://lwn.net/Articles/932209/
- https://lwn.net/Articles/940686/
- https://sourceware.org/binutils/docs/sframe-spec.html

Signed-off-by Josh Poimboeuf <[email protected]>

---
  arch/Kconfig                |   3 +
  arch/x86/include/asm/mmu.h  |   2 +-
  fs/binfmt_elf.c             |  46 +++-
  include/linux/mm_types.h    |   3 +
  include/linux/sframe.h      |  46 ++++
  include/linux/user_unwind.h |   1 +
  include/uapi/linux/elf.h    |   1 +
  include/uapi/linux/prctl.h  |   3 +
  kernel/fork.c               |  10 +
  kernel/sys.c                |  11 +
  kernel/unwind/Makefile      |   1 +
  kernel/unwind/sframe.c      | 414 ++++++++++++++++++++++++++++++++++++
  kernel/unwind/sframe.h      | 217 +++++++++++++++++++
  kernel/unwind/user.c        |  15 +-
  mm/init-mm.c                |   2 +
  15 files changed, 768 insertions(+), 7 deletions(-)

create mode 100644 include/linux/sframe.h
create mode 100644 kernel/unwind/sframe.c
create mode 100644 kernel/unwind/sframe.h

kernel/unwind/sframe.c new file +303 -0

new file mode 100644
index 000000000000..b167c19497e5
@@ -0,0 +1,414 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/srcu.h>
+#include <linux/uaccess.h>
+#include <linux/mm.h>
+#include <linux/sframe.h>
+#include <linux/user_unwind.h>
+
+#include "sframe.h"
+
+#define SFRAME_FILENAME_LEN 32
+
+struct sframe_section {
+	struct rcu_head rcu;
+
+	unsigned long sframe_addr;
+	unsigned long text_addr;
+
+	unsigned long fdes_addr;
+	unsigned long fres_addr;
+	unsigned int  fdes_num;
+	signed char ra_off, fp_off;
+};
+
+DEFINE_STATIC_SRCU(sframe_srcu);
+
+#define __SFRAME_GET_USER(out, user_ptr, type)				\
+({									\
+	type __tmp;							\
+	if (get_user(__tmp, (type *)user_ptr))				\
+		return -EFAULT;						\
+	user_ptr += sizeof(__tmp);					\
+	out = __tmp;							\
+})
+
+#define SFRAME_GET_USER_SIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, s8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, s16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, s32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+#define SFRAME_GET_USER_UNSIGNED(out, user_ptr, size)			\
+({									\
+	switch (size) {							\
+	case 1:								\
+		__SFRAME_GET_USER(out, user_ptr, u8);			\
+		break;							\
+	case 2:								\
+		__SFRAME_GET_USER(out, user_ptr, u16);			\
+		break;							\
+	case 4:								\
+		__SFRAME_GET_USER(out, user_ptr, u32);			\
+		break;							\
+	default:							\
+		return -EINVAL;						\
+	}								\
+})
+
+static unsigned char fre_type_to_size(unsigned char fre_type)
+{
+	if (fre_type > 2)
+		return 0;
+	return 1 << fre_type;
+}
+
+static unsigned char offset_size_enum_to_size(unsigned char off_size)
+{
+	if (off_size > 2)
+		return 0;
+	return 1 << off_size;
+}
+
+static int find_fde(struct sframe_section *sec, unsigned long ip,
+		    struct sframe_fde *fde)
+{
+	s32 func_off, ip_off;
+	struct sframe_fde __user *first, *last, *mid, *found;
+
+	ip_off = ip - sec->sframe_addr;
+
+	first = (void *)sec->fdes_addr;
+	last = first + sec->fdes_num;
+	while (first <= last) {
+		mid = first + ((last - first) / 2);
+		if (get_user(func_off, (s32 *)mid))
+			return -EFAULT;
+		if (ip_off >= func_off) {
+			found = mid;
+			first = mid + 1;
+		} else
+			last = mid - 1;
+	}
+
+	if (!found)
+		return -EINVAL;
+
+	if (copy_from_user(fde, found, sizeof(*fde)))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int find_fre(struct sframe_section *sec, struct sframe_fde *fde,
+		    unsigned long ip, struct user_unwind_frame *frame)
+{
+	unsigned char fde_type = SFRAME_FUNC_FDE_TYPE(fde->info);
+	unsigned char fre_type = SFRAME_FUNC_FRE_TYPE(fde->info);
+	s32 fre_ip_off, cfa_off, ra_off, fp_off, ip_off;
+	unsigned char offset_count, offset_size;
+	unsigned char addr_size;
+	void __user *f, *last_f;
+	u8 fre_info;
+	int i;
+
+	addr_size = fre_type_to_size(fre_type);
+	if (!addr_size)
+		return -EINVAL;
+
+	ip_off = ip - sec->sframe_addr - fde->start_addr;
+
+	f = (void *)sec->fres_addr + fde->fres_off;
+
+	for (i = 0; i < fde->fres_num; i++) {
+
+		SFRAME_GET_USER_UNSIGNED(fre_ip_off, f, addr_size);
+
+		if (fde_type == SFRAME_FDE_TYPE_PCINC) {
+			if (fre_ip_off > ip_off)
+				break;
+		} else {
+			/* SFRAME_FDE_TYPE_PCMASK */
+#if 0 /* sframe v2 */
+			if (ip_off % fde->rep_size < fre_ip_off)
+				break;
+#endif
+		}
+
+		SFRAME_GET_USER_UNSIGNED(fre_info, f, 1);
+
+		offset_count = SFRAME_FRE_OFFSET_COUNT(fre_info);
+		offset_size  = offset_size_enum_to_size(SFRAME_FRE_OFFSET_SIZE(fre_info));
+
+		if (!offset_count || !offset_size)
+			return -EINVAL;
+
+		last_f = f;
+		f += offset_count * offset_size;
+	}
+
+	if (!last_f)
+		return -EINVAL;
+
+	f = last_f;
+
+	SFRAME_GET_USER_UNSIGNED(cfa_off, f, offset_size);
+	offset_count--;
+
+	ra_off = sec->ra_off;
+	if (!ra_off) {
+		if (!offset_count--)
+			return -EINVAL;
+		SFRAME_GET_USER_SIGNED(ra_off, f, offset_size);
+	}
+
+	fp_off = sec->fp_off;
+	if (!fp_off && offset_count) {
+		offset_count--;
+		SFRAME_GET_USER_SIGNED(fp_off, f, offset_size);
+	}
+
+	if (offset_count)
+		return -EINVAL;
+
+	frame->cfa_off = cfa_off;
+	frame->ra_off = ra_off;
+	frame->fp_off = fp_off;
+	frame->use_fp = SFRAME_FRE_CFA_BASE_REG_ID(fre_info) == SFRAME_BASE_REG_FP;
+
+	return 0;
+}
+
+int sframe_find(unsigned long ip, struct user_unwind_frame *frame)
+{
+	struct mm_struct *mm = current->mm;
+	struct sframe_section *sec;
+	struct sframe_fde fde;
+	int srcu_idx;
+	int ret = -EINVAL;
+
+	srcu_idx = srcu_read_lock(&sframe_srcu);
+
+	sec = mtree_load(&mm->sframe_mt, ip);
+	if (!sec) {
+		srcu_read_unlock(&sframe_srcu, srcu_idx);
+		return -EINVAL;
+	}
+
+
+	ret = find_fde(sec, ip, &fde);
+	if (ret)
+		goto err_unlock;
+
+	ret = find_fre(sec, &fde, ip, frame);
+	if (ret)
+		goto err_unlock;
+
+	srcu_read_unlock(&sframe_srcu, srcu_idx);
+	return 0;
+
+err_unlock:
+	srcu_read_unlock(&sframe_srcu, srcu_idx);
+	return ret;
+}
+
+static int get_sframe_file(unsigned long sframe_addr, struct sframe_file *file)
+{
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *sframe_vma, *text_vma, *vma;
+	VMA_ITERATOR(vmi, mm, 0);
+
+	mmap_read_lock(mm);
+
+	sframe_vma = vma_lookup(mm, sframe_addr);
+	if (!sframe_vma || !sframe_vma->vm_file)
+		goto err_unlock;
+
+	text_vma = NULL;
+
+	for_each_vma(vmi, vma) {
+		if (vma->vm_file != sframe_vma->vm_file)
+			continue;
+		if (vma->vm_flags & VM_EXEC) {
+			if (text_vma) {
+				/*
+				 * Multiple EXEC segments in a single file
+				 * aren't currently supported, is that a thing?
+				 */
+				WARN_ON_ONCE(1);
+				goto err_unlock;
+			}
+			text_vma = vma;
+		}
+	}
+
+	file->sframe_addr	= sframe_addr;
+	file->text_start	= text_vma->vm_start;
+	file->text_end		= text_vma->vm_end;
+
+	mmap_read_unlock(mm);
+	return 0;
+
+err_unlock:
+	mmap_read_unlock(mm);
+	return -EINVAL;
+}
+
+static int validate_sframe_addrs(struct sframe_file *file)
+{
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *text_vma;
+
+	mmap_read_lock(mm);
+
+	if (!vma_lookup(mm, file->sframe_addr))
+		goto err_unlock;
+
+	text_vma = vma_lookup(mm, file->text_start);
+	if (!(text_vma->vm_flags & VM_EXEC))
+		goto err_unlock;
+
+	if (vma_lookup(mm, file->text_end-1) != text_vma)
+		goto err_unlock;
+
+	mmap_read_unlock(mm);
+	return 0;
+
+err_unlock:
+	mmap_read_unlock(mm);
+	return -EINVAL;
+}
+
+int __sframe_add_section(struct sframe_file *file)
+{
+	struct maple_tree *sframe_mt = &current->mm->sframe_mt;
+	struct sframe_section *sec;
+	struct sframe_header shdr;
+	unsigned long header_end;
+	int ret;
+
+	if (copy_from_user(&shdr, (void *)file->sframe_addr, sizeof(shdr)))

kernel/unwind/sframe.c line 313

Quoted 8 lines

kernel/unwind/sframe.c new file +8 -0

+	if (shdr.preamble.magic != SFRAME_MAGIC ||
+	    shdr.preamble.version != SFRAME_VERSION_1 ||
+	    (!shdr.preamble.flags & SFRAME_F_FDE_SORTED) ||
+	    shdr.auxhdr_len || !shdr.num_fdes || !shdr.num_fres ||
+	    shdr.fdes_off > shdr.fres_off) {
+		return -EINVAL;
+	}
+

I would say that it will be ideal to start the support with SFRAME_VERSION_2 onwards, if we have a choice.

The structure SFrame FDE in SFRAME_VERSION_1 was unaligned on-disk. We fixed that in SFRAME_VERSION_2 (Binutils 2.41) by adding some padding as you have already noted. For x86_64, its not an issue though, yes.

Quoted 3 lines

The structure SFrame FDE in SFRAME_VERSION_1 was unaligned on-disk. We fixed that in SFRAME_VERSION_2 (Binutils 2.41) by adding some padding as you have already noted. For x86_64, its not an issue though, yes.

Agreed. I actually had v2 implemented but then realized my binutils was older so I changed to v1 for testing. But yeah, we can make v2 the minimum.

Quoted 4 lines · On Thu, 9 Nov 2023 11:37:59 -0800 Josh Poimboeuf <[email protected]> wrote:

Quoted 3 lines

The structure SFrame FDE in SFRAME_VERSION_1 was unaligned on-disk. We fixed that in SFRAME_VERSION_2 (Binutils 2.41) by adding some padding as you have already noted. For x86_64, its not an issue though, yes.

Quoted 3 lines

Agreed. I actually had v2 implemented but then realized my binutils was older so I changed to v1 for testing. But yeah, we can make v2 the minimum.

Cool, my test box has binutils 2.41, so it should be good to go ;-)

Some distros have started compiling frame pointers into all their packages to enable the kernel to do system-wide profiling of user space. Unfortunately that creates a runtime performance penalty across the entire system. Using DWARF (or .eh_frame) instead isn't feasible because of complexity and slowness.

For in-kernel unwinding we solved this problem with the creation of the ORC unwinder for x86_64. Similarly, for user space the GNU assembler has created the SFrame ("Simple Frame") format starting with binutils 2.40.

These patches add support for unwinding user space from the kernel using SFrame with perf. It should be easy to add user unwinding support for other components like ftrace.

I tested it on Gentoo by recompiling everything with -Wa,-gsframe and using a custom glibc patch (which I'll send in a reply to this email).