perf: add ability to sample interrupted machine state

(quoted diff) +2 -1

@@ -595,7 +595,8 @@ struct perf_sample_data {
 	struct perf_callchain_entry	*callchain;
 	struct perf_raw_record		*raw;
 	struct perf_branch_stack	*br_stack;
-	struct perf_regs_user		regs_user;
+	struct perf_regs		regs_user;
+	struct perf_regs		regs_intr;
 	u64				stack_user_size;
 	u64				weight;
 	/*

Quoted 11 lines · Stephane Eranian

include/linux/perf_event.h +2 -1

@@ -595,7 +595,8 @@ struct perf_sample_data {
      struct perf_callchain_entry     *callchain;
      struct perf_raw_record          *raw;
      struct perf_branch_stack        *br_stack;
-     struct perf_regs_user           regs_user;
+     struct perf_regs                regs_user;
+     struct perf_regs                regs_intr;
      u64                             stack_user_size;
      u64                             weight;
      /*

include/linux/perf_event.h static inline void perf_sample_data_init(struct perf_sample_data *data, · line 624

Quoted 6 lines

include/linux/perf_event.h +2 -0

@@ -618,6 +619,8 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
      data->weight = 0;
      data->data_src.val = 0;
      data->txn = 0;
+     data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE;
+     data->regs_intr.regs = NULL;
 }

I don't think we've been very careful here; does the below make sense?

AFAICT we don't need to set stack_user_size at all, perf_prepare_sample() will set it when required, and with the change to perf_sample_regs_user() the same is true for the regs_user thing.

This again reduces the cost of perf_sample_data_init() to touching a single cacheline.

I'm not entirely sure the ____cacheline_aligned makes sense though, the previous stack line is probably touched already so any next cacheline is the one, and one avg we'd gain 0.5 cachelines worth of data.

---

include/linux/perf_event.h +18 -15

index 707617a8c0f6..d27fec8118b1 100644
@@ -575,34 +575,40 @@ extern u64 perf_event_read_value(struct perf_event *event,
 
 
 struct perf_sample_data {
-       u64                             type;
+       /*
+        * Fields set by perf_sample_data_init(), group so as to
+        * minimize the cachelines touched.
+        */
+       u64                             addr;
+       struct perf_raw_record          *raw;
+       struct perf_branch_stack        *br_stack;
+       u64                             period;
+       u64                             weight;
+       u64                             txn;
+       union  perf_mem_data_src        data_src;
+
 
+       /*
+        * The other fields, optionally {set,used} by
+        * perf_{prepare,output}_sample().
+        */
+       u64                             type;
        u64                             ip;
        struct {
                u32     pid;
                u32     tid;
        }                               tid_entry;
        u64                             time;
-       u64                             addr;
        u64                             id;
        u64                             stream_id;
        struct {
                u32     cpu;
                u32     reserved;
        }                               cpu_entry;
-       u64                             period;
-       union  perf_mem_data_src        data_src;
        struct perf_callchain_entry     *callchain;
-       struct perf_raw_record          *raw;
-       struct perf_branch_stack        *br_stack;
        struct perf_regs_user           regs_user;
        u64                             stack_user_size;
-       u64                             weight;
-       /*
-        * Transaction flags for abort events:
-        */
-       u64                             txn;
-};
+} ____cacheline_aligned;
 
 static inline void perf_sample_data_init(struct perf_sample_data *data,
                                         u64 addr, u64 period)

@@ -612,9 +618,6 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
        data->raw  = NULL;
        data->br_stack = NULL;
        data->period = period;
-       data->regs_user.abi = PERF_SAMPLE_REGS_ABI_NONE;
-       data->regs_user.regs = NULL;
-       data->stack_user_size = 0;
        data->weight = 0;
        data->data_src.val = 0;
        data->txn = 0;

kernel/events/core.c +2 -0

index c8b53c94d41d..926cd7aafc14 100644
@@ -4299,6 +4299,8 @@ perf_output_sample_regs(struct perf_output_handle *handle,
 static void perf_sample_regs_user(struct perf_regs_user *regs_user,
                                  struct pt_regs *regs)
 {
+       regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
+
        if (!user_mode(regs)) {
                if (current->mm)
                        regs = task_pt_regs(current);

include/linux/perf_event.h +12 -0

@@ -618,6 +619,8 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
 	data->weight = 0;
 	data->data_src.val = 0;
 	data->txn = 0;
+	data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE;
+	data->regs_intr.regs = NULL;
 }
 
+static void perf_sample_regs_intr(struct perf_regs *regs_intr,
+				  struct pt_regs *regs)
+{
+	regs_intr->regs = regs;
+	regs_intr->abi  = perf_reg_abi(current);
+}

@@ -4800,6 +4824,20 @@ void perf_prepare_sample(struct perf_event_header *header,
 		data->stack_user_size = stack_size;
 		header->size += size;
 	}
+
+	if (sample_type & PERF_SAMPLE_REGS_INTR) {
+		/* regs dump ABI info */
+		int size = sizeof(u64);

Quoted 28 lines · Stephane Eranian

include/linux/perf_event.h +17 -0

@@ -618,6 +619,8 @@ static inline void perf_sample_data_init(struct perf_sample_data *data,
      data->weight = 0;
      data->data_src.val = 0;
      data->txn = 0;
+     data->regs_intr.abi = PERF_SAMPLE_REGS_ABI_NONE;
+     data->regs_intr.regs = NULL;
 }
 
+static void perf_sample_regs_intr(struct perf_regs *regs_intr,
+                               struct pt_regs *regs)
+{
+     regs_intr->regs = regs;
+     regs_intr->abi  = perf_reg_abi(current);
+}

@@ -4800,6 +4824,20 @@ void perf_prepare_sample(struct perf_event_header *header,
              data->stack_user_size = stack_size;
              header->size += size;
      }
+
+     if (sample_type & PERF_SAMPLE_REGS_INTR) {
+             /* regs dump ABI info */
+             int size = sizeof(u64);
+
+             perf_sample_regs_intr(&data->regs_intr, regs);
+
+             if (data->regs_intr.regs) {
+                     u64 mask = event->attr.sample_regs_intr;

kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 4840

Quoted 5 lines

kernel/events/core.c +5 -0

+                     size += hweight64(mask) * sizeof(u64);
+             }
+
+             header->size += size;
+     }

Given that the prepare_sample hunk sets both regs_intr fields, the addition to perf_sample_data_init() is entirely superfluous, no?

PEBS can capture machine state regs at retiremnt of the sampled instructions. When precise sampling is enabled on an event, PEBS is used, so substitute the interrupted state with the PEBS state. Note that not all registers are captured by PEBS. Those missing are replaced by the interrupt state counter-parts.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event_intel_ds.c |   19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

arch/x86/kernel/cpu/perf_event_intel_ds.c +7 -0

index 980970c..d612bcd 100644
@@ -925,6 +925,25 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
 	regs.bp = pebs->bp;
 	regs.sp = pebs->sp;
 
+	if (sample_type & PERF_SAMPLE_REGS_INTR) {
+		regs.ax = pebs->ax;
+		regs.bx = pebs->bx;
+		regs.cx = pebs->cx;
+		regs.si = pebs->si;
+		regs.di = pebs->di;
+

Quoted 33 lines · Stephane Eranian

PEBS can capture machine state regs at retiremnt of the sampled instructions. When precise sampling is enabled on an event, PEBS is used, so substitute the interrupted state with the PEBS state. Note that not all registers are captured by PEBS. Those missing are replaced by the interrupt state counter-parts.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event_intel_ds.c |   19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

arch/x86/kernel/cpu/perf_event_intel_ds.c +14 -0

index 980970c..d612bcd 100644
@@ -925,6 +925,25 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
      regs.bp = pebs->bp;
      regs.sp = pebs->sp;
 
+     if (sample_type & PERF_SAMPLE_REGS_INTR) {
+             regs.ax = pebs->ax;
+             regs.bx = pebs->bx;
+             regs.cx = pebs->cx;
+             regs.si = pebs->si;
+             regs.di = pebs->di;
+
+             regs.r8 = pebs->r8;
+             regs.r9 = pebs->r9;
+             regs.r10 = pebs->r10;
+             regs.r11 = pebs->r11;
+             regs.r12 = pebs->r12;
+             regs.r13 = pebs->r13;
+             regs.r14 = pebs->r14;

arch/x86/kernel/cpu/perf_event_intel_ds.c static void __intel_pmu_pebs_event(struct perf_event *event, · line 944

Quoted 3 lines

arch/x86/kernel/cpu/perf_event_intel_ds.c +3 -0

+             regs.r14 = pebs->r15;
+
+             data.regs_intr.regs = &regs;

This last assignment is superfluous. Note how you previous patch's perf_sample_regs_intr() is unconditionally writing its .regs value, and in turn, perf_sample_regs_intr() is called unconditionally from perf_prepare_sample().