perf_events: add support for sampling taken branches (v3)

This patch adds the ability to sample taken branches to the perf_event interface.

The ability to capture taken branches is very useful for all sorts of analysis. For instance, basic block profiling, call counts, statistical call graph.

This new capability requires hardware assist and as such may not be available on all HW platforms. On Intel X86, it is implemented on top of the Last Branch Record (LBR) facility.

To enable taken branches sampling, the PERF_SAMPLE_BRANCH_STACK bit must be set in attr->sample_type.

Sampled taken branches may be filtered by type and/or priv levels.

The patch adds a new field, called branch_sample_type, to the perf_event_attr structure. It contains a bitmask of filters to apply to the sampled taken branches.

Filters may be implemented in HW. If the HW filter does not exist or is not good enough, some arch may also implement a SW filter.

The following generic filters are currently defined:
- PERF_SAMPLE_USER
only branches whose targets are at the user level

- PERF_SAMPLE_KERNEL
only branches whose targets are at the kernel level

- PERF_SAMPLE_ANY
any type of branches (subject to priv levels filters)

- PERF_SAMPLE_ANY_CALL
any call branches (may incl. syscall on some arch)

- PERF_SAMPLE_ANY_RET
any return branches (may incl. syscall returns on some arch)

- PERF_SAMPLE_IND_CALL
indirect call branches

Obviously filter may be combined. The priv level bits are optional. If not provided, the priv level of the associated event are used. It is possible to collect branches at a priv level different from the associated event.

The number of taken branch records present in each sample may vary based on HW, the type of sampled branches, the executed code. Therefore each sample contains the number of taken branches it contains.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   21 +++++---
 include/linux/perf_event.h                 |   66 ++++++++++++++++++++++++++--
 kernel/events/core.c                       |   58 ++++++++++++++++++++++++
 3 files changed, 133 insertions(+), 12 deletions(-)

arch/x86/kernel/cpu/perf_event_intel_lbr.c +13 -8

index 3fab3de..c3f8100 100644
@@ -144,9 +144,11 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 
 		rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 
-		cpuc->lbr_entries[i].from  = msr_lastbranch.from;
-		cpuc->lbr_entries[i].to    = msr_lastbranch.to;
-		cpuc->lbr_entries[i].flags = 0;
+		cpuc->lbr_entries[i].from	= msr_lastbranch.from;
+		cpuc->lbr_entries[i].to		= msr_lastbranch.to;
+		cpuc->lbr_entries[i].mispred	= 0;
+		cpuc->lbr_entries[i].predicted	= 0;
+		cpuc->lbr_entries[i].reserved	= 0;
 	}
 	cpuc->lbr_stack.nr = i;
 }

@@ -167,19 +169,22 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 
 	for (i = 0; i < x86_pmu.lbr_nr; i++) {
 		unsigned long lbr_idx = (tos - i) & mask;
-		u64 from, to, flags = 0;
+		u64 from, to, mis = 0, pred = 0;
 
 		rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
 		rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);
 
 		if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
-			flags = !!(from & LBR_FROM_FLAG_MISPRED);
+			mis = !!(from & LBR_FROM_FLAG_MISPRED);
+			pred = !mis;
 			from = (u64)((((s64)from) << 1) >> 1);
 		}
 
-		cpuc->lbr_entries[i].from  = from;
-		cpuc->lbr_entries[i].to    = to;
-		cpuc->lbr_entries[i].flags = flags;
+		cpuc->lbr_entries[i].from	= from;
+		cpuc->lbr_entries[i].to		= to;
+		cpuc->lbr_entries[i].mispred	= mis;
+		cpuc->lbr_entries[i].predicted	= pred;
+		cpuc->lbr_entries[i].reserved	= 0;
 	}
 	cpuc->lbr_stack.nr = i;
 }

include/linux/perf_event.h +32 -1

index 0b91db2..17751b1 100644
@@ -129,11 +129,38 @@ enum perf_event_sample_format {
 	PERF_SAMPLE_PERIOD			= 1U << 8,
 	PERF_SAMPLE_STREAM_ID			= 1U << 9,
 	PERF_SAMPLE_RAW				= 1U << 10,
+	PERF_SAMPLE_BRANCH_STACK		= 1U << 11,
 
-	PERF_SAMPLE_MAX = 1U << 11,		/* non-ABI */
+	PERF_SAMPLE_MAX = 1U << 12,		/* non-ABI */
 };
 
 /*
+ * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set
+ *
+ * If the user does not pass priv level information via branch_sample_type,
+ * the kernel uses the event's priv level. Branch and event priv levels do
+ * not have to match. Branch priv level is checked for permissions.
+ *
+ * The branch types can be combined, however BRANCH_ANY covers all types
+ * of branches and therefore it supersedes all the other types.
+ */
+enum perf_branch_sample_type {
+	PERF_SAMPLE_BRANCH_USER		= 1U << 0, /* user level branches */
+	PERF_SAMPLE_BRANCH_KERNEL	= 1U << 1, /* kernel level branches */
+
+	PERF_SAMPLE_BRANCH_ANY		= 1U << 2, /* any branch types */
+	PERF_SAMPLE_BRANCH_ANY_CALL	= 1U << 3, /* any call branch */
+	PERF_SAMPLE_BRANCH_ANY_RETURN	= 1U << 4, /* any return branch */
+	PERF_SAMPLE_BRANCH_IND_CALL	= 1U << 5, /* indirect calls */
+
+	PERF_SAMPLE_BRANCH_MAX		= 1U << 6,/* non-ABI */
+};
+
+#define PERF_SAMPLE_BRANCH_PLM_ALL \
+	(PERF_SAMPLE_BRANCH_USER|\
+	 PERF_SAMPLE_BRANCH_KERNEL)
+
+/*
  * The format of the data returned by read() on a perf event fd,
  * as specified by attr.read_format:
  *

@@ -240,6 +267,7 @@ struct perf_event_attr {
 		__u64		bp_len;
 		__u64		config2; /* extension of config1 */
 	};
+	__u64	branch_sample_type; /* enum branch_sample_type */
 };
 
 /*

@@ -458,6 +486,8 @@ enum perf_event_type {
 	 *
 	 *	{ u32			size;
 	 *	  char                  data[size];}&& PERF_SAMPLE_RAW
+	 *
+	 *	{ u64 from, to, flags } lbr[nr];} && PERF_SAMPLE_BRANCH_STACK
 	 * };
 	 */
 	PERF_RECORD_SAMPLE			= 9,

@@ -530,12 +560,31 @@ struct perf_raw_record {
 	void				*data;
 };
 
+/*

include/linux/perf_event.h +20 -3

 struct perf_branch_entry {
-	__u64				from;
-	__u64				to;
-	__u64				flags;
+	__u64	from;
+	__u64	to;
+	__u64	mispred:1,  /* target mispredicted */
+		predicted:1,/* target predicted */
+		reserved:62;
 };
 
+/*
+ * branch stack layout:
+ *  nr: number of taken branches stored in entries[]
+ *
+ * Note that nr can vary from sample to sample
+ */
 struct perf_branch_stack {
 	__u64				nr;
 	struct perf_branch_entry	entries[0];

@@ -566,7 +615,9 @@ struct hw_perf_event {
 			unsigned long	event_base;
 			int		idx;
 			int		last_cpu;
+
 			struct hw_perf_event_extra extra_reg;
+			struct hw_perf_event_extra branch_reg;
 		};
 		struct { /* software */
 			struct hrtimer	hrtimer;

@@ -1003,12 +1054,14 @@ struct perf_sample_data {
 	u64				period;
 	struct perf_callchain_entry	*callchain;
 	struct perf_raw_record		*raw;
+	struct perf_branch_stack	*br_stack;
 };
 
 static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
 {
 	data->addr = addr;
 	data->raw  = NULL;
+	data->br_stack = NULL;
 }
 
 extern void perf_output_sample(struct perf_output_handle *handle,

@@ -1147,6 +1200,11 @@ extern void perf_bp_event(struct perf_event *event, void *data);
 # define perf_instruction_pointer(regs)	instruction_pointer(regs)
 #endif
 
+static inline bool has_branch_stack(struct perf_event *event)
+{
+	return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
+}
+
 extern int perf_output_begin(struct perf_output_handle *handle,
 			     struct perf_event *event, unsigned int size);
 extern void perf_output_end(struct perf_output_handle *handle);

kernel/events/core.c +42 -0

index 91fb68a..ed39225 100644
@@ -3877,6 +3877,24 @@ void perf_output_sample(struct perf_output_handle *handle,
 			}
 		}
 	}
+
+	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
+		if (data->br_stack) {
+			size_t size;
+
+			size = data->br_stack->nr
+			     * sizeof(struct perf_branch_entry);
+
+			perf_output_put(handle, data->br_stack->nr);
+			perf_output_copy(handle, data->br_stack->entries, size);
+		} else {
+			/*
+			 * we always store at least the value of nr
+			 */
+			u64 nr = 0;
+			perf_output_put(handle, nr);
+		}
+	}
 }
 
 void perf_prepare_sample(struct perf_event_header *header,

@@ -3919,6 +3937,15 @@ void perf_prepare_sample(struct perf_event_header *header,
 		WARN_ON_ONCE(size & (sizeof(u64)-1));
 		header->size += size;
 	}
+
+	if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
+		int size = sizeof(u64); /* nr */
+		if (data->br_stack) {
+			size += data->br_stack->nr
+			      * sizeof(struct perf_branch_entry);
+		}
+		header->size += size;
+	}
 }
 
 static void perf_event_output(struct perf_event *event,

@@ -5898,6 +5925,37 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
 	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
 		return -EINVAL;
 
+	if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
+		u64 mask = attr->branch_sample_type;
+
+		/* only using defined bits */
+		if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
+			return -EINVAL;
+
+		/* at least one branch bit must be set */
+		if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
+			return -EINVAL;
+
+		/* kernel level capture */
+		if ((mask & PERF_SAMPLE_BRANCH_KERNEL)
+		    && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
+			return -EACCES;

kernel/events/core.c +6 -0

+			/*
+			 * adjust user setting (for HW filter setup)
+			 */
+			attr->branch_sample_type = mask;
+		}
+	}
 out:
 	return ret;
 

Quoted 48 lines · Stephane Eranian

This patch adds the ability to sample taken branches to the perf_event interface.

The ability to capture taken branches is very useful for all sorts of analysis. For instance, basic block profiling, call counts, statistical call graph.

This new capability requires hardware assist and as such may not be available on all HW platforms. On Intel X86, it is implemented on top of the Last Branch Record (LBR) facility.

To enable taken branches sampling, the PERF_SAMPLE_BRANCH_STACK bit must be set in attr->sample_type.

Sampled taken branches may be filtered by type and/or priv levels.

The patch adds a new field, called branch_sample_type, to the perf_event_attr structure. It contains a bitmask of filters to apply to the sampled taken branches.

Filters may be implemented in HW. If the HW filter does not exist or is not good enough, some arch may also implement a SW filter.

The following generic filters are currently defined:
- PERF_SAMPLE_USER
  only branches whose targets are at the user level

- PERF_SAMPLE_KERNEL
  only branches whose targets are at the kernel level

- PERF_SAMPLE_ANY
  any type of branches (subject to priv levels filters)

- PERF_SAMPLE_ANY_CALL
  any call branches (may incl. syscall on some arch)

- PERF_SAMPLE_ANY_RET
  any return branches (may incl. syscall returns on some arch)

- PERF_SAMPLE_IND_CALL
  indirect call branches

Obviously filter may be combined. The priv level bits are optional. If not provided, the priv level of the associated event are used. It is possible to collect branches at a priv level different from the associated event.

Quoted 5 lines

The number of taken branch records present in each sample may vary based on HW, the type of sampled branches, the executed code. Therefore each sample contains the number of taken branches it contains.

Signed-off-by Stephane Eranian <[email protected]>

 Reviewed by: Anshuman Khandual <[email protected]>

Quoted 119 lines

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   21 +++++--- include/linux/perf_event.h                 |   66 ++++++++++++++++++++++++++-- kernel/events/core.c                       |   58 ++++++++++++++++++++++++

 3 files changed, 133 insertions(+), 12 deletions(-)

arch/x86/kernel/cpu/perf_event_intel_lbr.c +13 -8

index 3fab3de..c3f8100 100644
@@ -144,9 +144,11 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 
               rdmsrl(x86_pmu.lbr_from + lbr_idx, msr_lastbranch.lbr);
 
-             cpuc->lbr_entries[i].from  = msr_lastbranch.from;
-             cpuc->lbr_entries[i].to    = msr_lastbranch.to;
-             cpuc->lbr_entries[i].flags = 0;
+             cpuc->lbr_entries[i].from       = msr_lastbranch.from;
+             cpuc->lbr_entries[i].to         = msr_lastbranch.to;
+             cpuc->lbr_entries[i].mispred    = 0;
+             cpuc->lbr_entries[i].predicted  = 0;
+             cpuc->lbr_entries[i].reserved   = 0;
       }
       cpuc->lbr_stack.nr = i;
  }

@@ -167,19 +169,22 @@ static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)
 
       for (i = 0; i < x86_pmu.lbr_nr; i++) {
               unsigned long lbr_idx = (tos - i) & mask;
-             u64 from, to, flags = 0;
+             u64 from, to, mis = 0, pred = 0;
 
               rdmsrl(x86_pmu.lbr_from + lbr_idx, from);
               rdmsrl(x86_pmu.lbr_to   + lbr_idx, to);
 
               if (lbr_format == LBR_FORMAT_EIP_FLAGS) {
-                     flags = !!(from & LBR_FROM_FLAG_MISPRED);
+                     mis = !!(from & LBR_FROM_FLAG_MISPRED);
+                     pred = !mis;
                       from = (u64)((((s64)from) << 1) >> 1);
               }
 
-             cpuc->lbr_entries[i].from  = from;
-             cpuc->lbr_entries[i].to    = to;
-             cpuc->lbr_entries[i].flags = flags;
+             cpuc->lbr_entries[i].from       = from;
+             cpuc->lbr_entries[i].to         = to;
+             cpuc->lbr_entries[i].mispred    = mis;
+             cpuc->lbr_entries[i].predicted  = pred;
+             cpuc->lbr_entries[i].reserved   = 0;
       }
       cpuc->lbr_stack.nr = i;
  }

include/linux/perf_event.h +28 -1

index 0b91db2..17751b1 100644
@@ -129,11 +129,38 @@ enum perf_event_sample_format {
       PERF_SAMPLE_PERIOD                      = 1U << 8,
       PERF_SAMPLE_STREAM_ID                   = 1U << 9,
       PERF_SAMPLE_RAW                         = 1U << 10,
+     PERF_SAMPLE_BRANCH_STACK                = 1U << 11,
 
-     PERF_SAMPLE_MAX = 1U << 11,             /* non-ABI */
+     PERF_SAMPLE_MAX = 1U << 12,             /* non-ABI */
  };
 
  /*
+ * values to program into branch_sample_type when PERF_SAMPLE_BRANCH is set
+ *
+ * If the user does not pass priv level information via branch_sample_type,
+ * the kernel uses the event's priv level. Branch and event priv levels do
+ * not have to match. Branch priv level is checked for permissions.
+ *
+ * The branch types can be combined, however BRANCH_ANY covers all types
+ * of branches and therefore it supersedes all the other types.
+ */
+enum perf_branch_sample_type {
+     PERF_SAMPLE_BRANCH_USER         = 1U << 0, /* user level branches */
+     PERF_SAMPLE_BRANCH_KERNEL       = 1U << 1, /* kernel level branches */
+
+     PERF_SAMPLE_BRANCH_ANY          = 1U << 2, /* any branch types */
+     PERF_SAMPLE_BRANCH_ANY_CALL     = 1U << 3, /* any call branch */
+     PERF_SAMPLE_BRANCH_ANY_RETURN   = 1U << 4, /* any return branch */
+     PERF_SAMPLE_BRANCH_IND_CALL     = 1U << 5, /* indirect calls */
+
+     PERF_SAMPLE_BRANCH_MAX          = 1U << 6,/* non-ABI */
+};
+
+#define PERF_SAMPLE_BRANCH_PLM_ALL \
+     (PERF_SAMPLE_BRANCH_USER|\
+      PERF_SAMPLE_BRANCH_KERNEL)
+
+/*
   * The format of the data returned by read() on a perf event fd,
   * as specified by attr.read_format:
   *

@@ -240,6 +267,7 @@ struct perf_event_attr {
               __u64           bp_len;

              __u64           config2; /* extension of config1 */ };
+     __u64   branch_sample_type; /* enum branch_sample_type */ };

 /*

include/linux/perf_event.h +3 -0

@@ -458,6 +486,8 @@ enum perf_event_type {
        *
        *      { u32                   size;
        *        char                  data[size];}&& PERF_SAMPLE_RAW
+      *
+      *      { u64 from, to, flags } lbr[nr];} && PERF_SAMPLE_BRANCH_STACK
        * };
        */
       PERF_RECORD_SAMPLE                      = 9,

@@ -530,12 +560,31 @@ struct perf_raw_record {
       void                            *data;
  };
 
+/*

include/linux/perf_event.h struct perf_raw_record { · line 573

Quoted 10 lines

include/linux/perf_event.h +10 -0

+ * single taken branch record layout:
+ *
+ *      from: source instruction (may not always be a branch insn)
+ *        to: branch target
+ *   mispred: branch target was mispredicted
+ * predicted: branch target was predicted
+ *
+ * support for mispred, predicted is optional. In case it
+ * is not supported mispred = predicted = 0.
+ */

So the user level perf tools would check for ((mispred = 0) && (predicted = 0)) in a sample and report that its not supported by the HW PMU ? Point here is that if its not supported we should say  "No HW support" rather than displaying mispred = 0 and predicted = 0 (As this could be misleading)

Quoted 125 lines

include/linux/perf_event.h +13 -3

  struct perf_branch_entry {
-     __u64                           from;
-     __u64                           to;
-     __u64                           flags;
+     __u64   from;
+     __u64   to;
+     __u64   mispred:1,  /* target mispredicted */
+             predicted:1,/* target predicted */
+             reserved:62;
  };
 
+/*
+ * branch stack layout:
+ *  nr: number of taken branches stored in entries[]
+ *
+ * Note that nr can vary from sample to sample
+ */
  struct perf_branch_stack {
       __u64                           nr;
       struct perf_branch_entry        entries[0];

@@ -566,7 +615,9 @@ struct hw_perf_event {
                       unsigned long   event_base;
                       int             idx;
                       int             last_cpu;
+
                       struct hw_perf_event_extra extra_reg;
+                     struct hw_perf_event_extra branch_reg;
               };

              struct { /* software */
                      struct hrtimer  hrtimer;

include/linux/perf_event.h +7 -0

@@ -1003,12 +1054,14 @@ struct perf_sample_data {
       u64                             period;
       struct perf_callchain_entry     *callchain;
       struct perf_raw_record          *raw;
+     struct perf_branch_stack        *br_stack;
  };
 
  static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
  {
       data->addr = addr;
       data->raw  = NULL;
+     data->br_stack = NULL;
  }
 
  extern void perf_output_sample(struct perf_output_handle *handle,

@@ -1147,6 +1200,11 @@ extern void perf_bp_event(struct perf_event *event, void *data);
  # define perf_instruction_pointer(regs)      instruction_pointer(regs)
  #endif
 
+static inline bool has_branch_stack(struct perf_event *event)
+{
+     return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;
+}
+
  extern int perf_output_begin(struct perf_output_handle *handle,
                            struct perf_event *event, unsigned int size);
  extern void perf_output_end(struct perf_output_handle *handle);

kernel/events/core.c +46 -0

index 91fb68a..ed39225 100644
@@ -3877,6 +3877,24 @@ void perf_output_sample(struct perf_output_handle *handle,
                       }
               }
       }
+
+     if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
+             if (data->br_stack) {
+                     size_t size;
+
+                     size = data->br_stack->nr
+                          * sizeof(struct perf_branch_entry);
+
+                     perf_output_put(handle, data->br_stack->nr);
+                     perf_output_copy(handle, data->br_stack->entries, size);
+             } else {
+                     /*
+                      * we always store at least the value of nr
+                      */
+                     u64 nr = 0;
+                     perf_output_put(handle, nr);
+             }
+     }
  }
 
  void perf_prepare_sample(struct perf_event_header *header,

@@ -3919,6 +3937,15 @@ void perf_prepare_sample(struct perf_event_header *header,
               WARN_ON_ONCE(size & (sizeof(u64)-1));
               header->size += size;
       }
+
+     if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
+             int size = sizeof(u64); /* nr */
+             if (data->br_stack) {
+                     size += data->br_stack->nr
+                           * sizeof(struct perf_branch_entry);
+             }
+             header->size += size;
+     }
  }
 
  static void perf_event_output(struct perf_event *event,

@@ -5898,6 +5925,37 @@ static int perf_copy_attr(struct perf_event_attr __user *uattr,
       if (attr->read_format & ~(PERF_FORMAT_MAX-1))
               return -EINVAL;
 
+     if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
+             u64 mask = attr->branch_sample_type;
+
+             /* only using defined bits */
+             if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
+                     return -EINVAL;
+
+             /* at least one branch bit must be set */
+             if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
+                     return -EINVAL;
+
+             /* kernel level capture */
+             if ((mask & PERF_SAMPLE_BRANCH_KERNEL)
+                 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
+                     return -EACCES;
+
+             /* propagate priv level, when not set for branch */
+             if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
+

kernel/events/core.c static int perf_copy_attr(struct perf_event_attr __user *uattr, · line 5952

Quoted 6 lines

kernel/events/core.c +6 -0

+                     /* exclude_kernel checked on syscall entry */
+                     if (!attr->exclude_kernel)
+                             mask |= PERF_SAMPLE_BRANCH_KERNEL;
+
+                     if (!attr->exclude_user)
+                             mask |= PERF_SAMPLE_BRANCH_USER;

Why we are not taking care for attr->exclude_hv ? Should not we define PERF_SAMPLE_BRANCH_HV for hyper-visor level branches ?

Quoted 9 lines

kernel/events/core.c +6 -0

+                     /*
+                      * adjust user setting (for HW filter setup)
+                      */
+                     attr->branch_sample_type = mask;
+             }
+     }

 out:
      return ret;

--
Linux Technology Centre
IBM Systems and Technology Group
Bangalore India

This patch adds the LBR definitions for NHM/WSM/SNB and Core. It also adds the definitions for the architected LBR MSR:
LBR_SELECT, LBRT_TOS.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/include/asm/msr-index.h           |    7 +++++++
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   18 +++++++++---------
 2 files changed, 16 insertions(+), 9 deletions(-)

arch/x86/include/asm/msr-index.h +7 -0

index a6962d9..ccb8059 100644
@@ -56,6 +56,13 @@
 #define MSR_OFFCORE_RSP_0		0x000001a6
 #define MSR_OFFCORE_RSP_1		0x000001a7
 
+#define MSR_LBR_SELECT			0x000001c8
+#define MSR_LBR_TOS			0x000001c9
+#define MSR_LBR_NHM_FROM		0x00000680
+#define MSR_LBR_NHM_TO			0x000006c0
+#define MSR_LBR_CORE_FROM		0x00000040
+#define MSR_LBR_CORE_TO			0x00000060
+
 #define MSR_IA32_PEBS_ENABLE		0x000003f1
 #define MSR_IA32_DS_AREA		0x00000600
 #define MSR_IA32_PERF_CAPABILITIES	0x00000345

arch/x86/kernel/cpu/perf_event_intel_lbr.c +9 -9

index c3f8100..e14431f 100644
@@ -205,23 +205,23 @@ void intel_pmu_lbr_read(void)
 void intel_pmu_lbr_init_core(void)
 {
 	x86_pmu.lbr_nr     = 4;
-	x86_pmu.lbr_tos    = 0x01c9;
-	x86_pmu.lbr_from   = 0x40;
-	x86_pmu.lbr_to     = 0x60;
+	x86_pmu.lbr_tos    = MSR_LBR_TOS;
+	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
+	x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
 }
 
 void intel_pmu_lbr_init_nhm(void)
 {
 	x86_pmu.lbr_nr     = 16;
-	x86_pmu.lbr_tos    = 0x01c9;
-	x86_pmu.lbr_from   = 0x680;
-	x86_pmu.lbr_to     = 0x6c0;
+	x86_pmu.lbr_tos    = MSR_LBR_TOS;
+	x86_pmu.lbr_from   = MSR_LBR_NHM_FROM;
+	x86_pmu.lbr_to     = MSR_LBR_NHM_TO;
 }
 
 void intel_pmu_lbr_init_atom(void)
 {
 	x86_pmu.lbr_nr	   = 8;
-	x86_pmu.lbr_tos    = 0x01c9;
-	x86_pmu.lbr_from   = 0x40;
-	x86_pmu.lbr_to     = 0x60;
+	x86_pmu.lbr_tos    = MSR_LBR_TOS;
+	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
+	x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
 }

If precise sampling is enabled on Intel X86, then perf_event uses PEBS. To correct for the off-by-one error of PEBS, perf_event uses LBR when precise_sample > 1.

On Intel X86 PERF_SAMPLE_BRANCH_STACK is implemented using LBR, therefore both features must be coordinated as they may not configure LBR the same way.

For PEBS, LBR needs to capture all branches at all priv levels. This patch sets this up.

The configuration of PERF_SAMPLE_BRANCH_STACK may not be compatible in which case an error must be returned.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event.c |   22 ++++++++++++++++++++++
 1 files changed, 22 insertions(+), 0 deletions(-)

arch/x86/kernel/cpu/perf_event.c +22 -0

index 3779313..710ec93 100644
@@ -356,6 +356,7 @@ int x86_setup_perfctr(struct perf_event *event)
 int x86_pmu_hw_config(struct perf_event *event)
 {
 	if (event->attr.precise_ip) {
+		u64 *br_type, br_sel;
 		int precise = 0;
 
 		/* Support for constant skid */

@@ -369,6 +370,27 @@ int x86_pmu_hw_config(struct perf_event *event)
 
 		if (event->attr.precise_ip > precise)
 			return -EOPNOTSUPP;
+		/*
+		 * check that PEBS LBR correction does not conflict with
+		 * whatever the user is asking with attr->branch_sample_type
+		 */
+		if (event->attr.precise_ip > 1) {
+
+			br_type = &event->attr.branch_sample_type;
+
+			if (has_branch_stack(event)) {
+				br_sel = *br_type & PERF_SAMPLE_BRANCH_ANY;
+				if (br_sel != PERF_SAMPLE_BRANCH_ANY)
+					return -EOPNOTSUPP;
+			} else {
+				/*
+				 * For PEBS fixups, we capture all
+				 * the branches at all priv levels
+				 */
+				*br_type = PERF_SAMPLE_BRANCH_ANY
+					 | PERF_SAMPLE_BRANCH_PLM_ALL;
+			}
+		}
 	}
 
 	/*

This patch adds the mappings from the generic PERF_SAMPLE_BRANCH_* filters to the actual Intel X86 LBR filters, whenever they exist.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event.h           |    2 +
 arch/x86/kernel/cpu/perf_event_intel.c     |    2 +-
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   99 +++++++++++++++++++++++++++-
 3 files changed, 100 insertions(+), 3 deletions(-)

arch/x86/kernel/cpu/perf_event.h +2 -0

index 4535ada..776fb5a 100644
@@ -535,6 +535,8 @@ void intel_pmu_lbr_init_nhm(void);
 
 void intel_pmu_lbr_init_atom(void);
 
+void intel_pmu_lbr_init_snb(void);
+
 int p4_pmu_init(void);
 
 int p6_pmu_init(void);

arch/x86/kernel/cpu/perf_event_intel.c +1 -1

index 97f7bb5..b0db016 100644
@@ -1757,7 +1757,7 @@ __init int intel_pmu_init(void)
 		memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
 		       sizeof(hw_cache_event_ids));
 
-		intel_pmu_lbr_init_nhm();
+		intel_pmu_lbr_init_snb();
 
 		x86_pmu.event_constraints = intel_snb_event_constraints;
 		x86_pmu.pebs_constraints = intel_snb_pebs_event_constraints;

arch/x86/kernel/cpu/perf_event_intel_lbr.c +97 -2

index e14431f..8a1eb6c 100644
@@ -14,6 +14,47 @@ enum {
 };
 
 /*
+ * Intel LBR_SELECT bits
+ * Intel Vol3a, April 2011, Section 16.7 Table 16-10
+ *
+ * Hardware branch filter (not available on all CPUs)
+ */
+#define LBR_KERNEL_BIT		0 /* do not capture at ring0 */
+#define LBR_USER_BIT		1 /* do not capture at ring > 0 */
+#define LBR_JCC_BIT		2 /* do not capture conditional branches */
+#define LBR_REL_CALL_BIT	3 /* do not capture relative calls */
+#define LBR_IND_CALL_BIT	4 /* do not capture indirect calls */
+#define LBR_RETURN_BIT		5 /* do not capture near returns */
+#define LBR_IND_JMP_BIT		6 /* do not capture indirect jumps */
+#define LBR_REL_JMP_BIT		7 /* do not capture relative jumps */
+#define LBR_FAR_BIT		8 /* do not capture far branches */
+
+#define LBR_KERNEL	(1 << LBR_KERNEL_BIT)
+#define LBR_USER	(1 << LBR_USER_BIT)
+#define LBR_JCC		(1 << LBR_JCC_BIT)
+#define LBR_REL_CALL	(1 << LBR_REL_CALL_BIT)
+#define LBR_IND_CALL	(1 << LBR_IND_CALL_BIT)
+#define LBR_RETURN	(1 << LBR_RETURN_BIT)
+#define LBR_REL_JMP	(1 << LBR_REL_JMP_BIT)
+#define LBR_IND_JMP	(1 << LBR_IND_JMP_BIT)
+#define LBR_FAR		(1 << LBR_FAR_BIT)
+
+#define LBR_PLM (LBR_KERNEL | LBR_USER)
+
+#define LBR_SEL_MASK	0x1ff /* valid bits in LBR_SELECT */
+
+#define LBR_ANY		 \
+	(LBR_JCC	|\
+	 LBR_REL_CALL	|\
+	 LBR_IND_CALL	|\
+	 LBR_RETURN	|\
+	 LBR_REL_JMP	|\
+	 LBR_IND_JMP	|\
+	 LBR_FAR)
+
+#define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
+
+/*
  * We only support LBR implementations that have FREEZE_LBRS_ON_PMI
  * otherwise it becomes near impossible to get a reliable stack.
  */

@@ -153,8 +194,6 @@ static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)
 	cpuc->lbr_stack.nr = i;
 }
 
-#define LBR_FROM_FLAG_MISPRED  (1ULL << 63)
-
 /*
  * Due to lack of segmentation in Linux the effective address (offset)
  * is the same as the linear address, allowing us to merge the LIP and EIP

@@ -202,26 +241,82 @@ void intel_pmu_lbr_read(void)
 		intel_pmu_lbr_read_64(cpuc);
 }
 
+/*
+ * Map interface branch filters onto LBR filters
+ */
+static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
+	[PERF_SAMPLE_BRANCH_ANY]        = LBR_ANY,
+	[PERF_SAMPLE_BRANCH_USER]       = LBR_USER,
+	[PERF_SAMPLE_BRANCH_KERNEL]     = LBR_KERNEL,
+	[PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_REL_JMP
+					| LBR_IND_JMP | LBR_FAR,
+	/*
+	 * NHM/WSM erratum: must include REL_JMP+IND_JMP to get CALL branches
+	 */
+	[PERF_SAMPLE_BRANCH_ANY_CALL] =
+	 LBR_REL_CALL | LBR_IND_CALL | LBR_REL_JMP | LBR_IND_JMP | LBR_FAR,
+	/*
+	 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
+	 */
+	[PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP,
+};
+
+static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
+	[PERF_SAMPLE_BRANCH_ANY]        = LBR_ANY,
+	[PERF_SAMPLE_BRANCH_USER]       = LBR_USER,
+	[PERF_SAMPLE_BRANCH_KERNEL]     = LBR_KERNEL,
+	[PERF_SAMPLE_BRANCH_ANY_RETURN] = LBR_RETURN | LBR_FAR,
+	[PERF_SAMPLE_BRANCH_ANY_CALL]   = LBR_REL_CALL | LBR_IND_CALL
+					| LBR_FAR,
+	[PERF_SAMPLE_BRANCH_IND_CALL]   = LBR_IND_CALL,
+};
+
+/* core */
 void intel_pmu_lbr_init_core(void)
 {
 	x86_pmu.lbr_nr     = 4;
 	x86_pmu.lbr_tos    = MSR_LBR_TOS;
 	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
 	x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
+
+	pr_cont("4-deep LBR, ");
 }
 
+/* nehalem/westmere */
 void intel_pmu_lbr_init_nhm(void)
 {
 	x86_pmu.lbr_nr     = 16;
 	x86_pmu.lbr_tos    = MSR_LBR_TOS;
 	x86_pmu.lbr_from   = MSR_LBR_NHM_FROM;
 	x86_pmu.lbr_to     = MSR_LBR_NHM_TO;
+
+	x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
+	x86_pmu.lbr_sel_map  = nhm_lbr_sel_map;
+
+	pr_cont("16-deep LBR, ");
 }
 
+/* sandy bridge */
+void intel_pmu_lbr_init_snb(void)
+{
+	x86_pmu.lbr_nr	 = 16;
+	x86_pmu.lbr_tos	 = MSR_LBR_TOS;
+	x86_pmu.lbr_from = MSR_LBR_NHM_FROM;
+	x86_pmu.lbr_to   = MSR_LBR_NHM_TO;
+
+	x86_pmu.lbr_sel_mask = LBR_SEL_MASK;
+	x86_pmu.lbr_sel_map  = snb_lbr_sel_map;
+
+	pr_cont("16-deep LBR, ");
+}
+
+/* atom */
 void intel_pmu_lbr_init_atom(void)
 {
 	x86_pmu.lbr_nr	   = 8;
 	x86_pmu.lbr_tos    = MSR_LBR_TOS;
 	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;
 	x86_pmu.lbr_to     = MSR_LBR_CORE_TO;
+
+	pr_cont("8-deep LBR, ");
 }

The patch adds a restriction for Intel Atom LBR support. Only steppings 10 (PineView) and more recent are supported. Older models, do not have a functional LBR. Their LBR does not freeze on PMU interrupt which makes LBR unusable in the context of perf_events.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)

arch/x86/kernel/cpu/perf_event_intel_lbr.c +10 -0

index 8a1eb6c..e2b7094 100644
@@ -313,6 +313,16 @@ void intel_pmu_lbr_init_snb(void)
 /* atom */
 void intel_pmu_lbr_init_atom(void)
 {
+	/*
+	 * only models starting at stepping 10 seems
+	 * to have an operational LBR which can freeze
+	 * on PMU interrupt
+	 */
+	if (boot_cpu_data.x86_mask < 10) {
+		pr_cont("LBR disabled due to erratum");
+		return;
+	}
+
 	x86_pmu.lbr_nr	   = 8;
 	x86_pmu.lbr_tos    = MSR_LBR_TOS;
 	x86_pmu.lbr_from   = MSR_LBR_CORE_FROM;

This patch implements PERF_SAMPLE_BRANCH support for Intel X86 processors. It connects PERF_SAMPLE_BRANCH to the actual LBR.

The patch adds the hooks in the PMU irq handler to save the LBR on counter overflow for both regular and PEBS modes.

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/x86/kernel/cpu/perf_event_intel.c     |   35 +++++++++++++
 arch/x86/kernel/cpu/perf_event_intel_ds.c  |   10 ++--
 arch/x86/kernel/cpu/perf_event_intel_lbr.c |   73 +++++++++++++++++++++++++++-
 include/linux/perf_event.h                 |    3 +
 4 files changed, 113 insertions(+), 8 deletions(-)

PERF_SAMPLE_BRANCH_* is disabled for:
- SW events (sw counters, tracepoints)
- HW breakpoints
- ALL but Intel X86 architecture
- AMD64 processors

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/alpha/kernel/perf_event.c       |    4 ++++
 arch/arm/kernel/perf_event.c         |    4 ++++
 arch/mips/kernel/perf_event_mipsxx.c |    4 ++++
 arch/powerpc/kernel/perf_event.c     |    4 ++++
 arch/sh/kernel/perf_event.c          |    4 ++++
 arch/sparc/kernel/perf_event.c       |    4 ++++
 arch/x86/kernel/cpu/perf_event_amd.c |    3 +++
 kernel/events/core.c                 |   24 ++++++++++++++++++++++++
 kernel/events/hw_breakpoint.c        |    6 ++++++
 9 files changed, 57 insertions(+), 0 deletions(-)

arch/alpha/kernel/perf_event.c +4 -0

index 8143cd7..0dae252 100644
@@ -685,6 +685,10 @@ static int alpha_pmu_event_init(struct perf_event *event)
 {
 	int err;
 
+	/* does not support taken branch sampling */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	switch (event->attr.type) {
 	case PERF_TYPE_RAW:
 	case PERF_TYPE_HARDWARE:

arch/arm/kernel/perf_event.c +0 -0

index 88b0941..42262ff 100644

arch/arm/kernel/perf_event.c +2 -0

+		return -EOPNOTSUPP;
+
 	if (armpmu->map_event(event) == -ENOENT)
 		return -ENOENT;
 

arch/mips/kernel/perf_event_mipsxx.c +4 -0

index 315fc0b..7070f8c 100644
@@ -606,6 +606,10 @@ static int mipspmu_event_init(struct perf_event *event)
 {
 	int err = 0;
 
+	/* does not support taken branch sampling */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	switch (event->attr.type) {
 	case PERF_TYPE_RAW:
 	case PERF_TYPE_HARDWARE:

arch/powerpc/kernel/perf_event.c +4 -0

index d614ab5..4e0b265 100644
@@ -1078,6 +1078,10 @@ static int power_pmu_event_init(struct perf_event *event)
 	if (!ppmu)
 		return -ENOENT;
 
+	/* does not support taken branch sampling */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	switch (event->attr.type) {
 	case PERF_TYPE_HARDWARE:
 		ev = event->attr.config;

arch/sh/kernel/perf_event.c +4 -0

index 10b14e3..068b8a2 100644
@@ -310,6 +310,10 @@ static int sh_pmu_event_init(struct perf_event *event)
 {
 	int err;
 
+	/* does not support taken branch sampling */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	switch (event->attr.type) {
 	case PERF_TYPE_RAW:
 	case PERF_TYPE_HW_CACHE:

arch/sparc/kernel/perf_event.c +4 -0

index 614da62..8e16a4a 100644
@@ -1105,6 +1105,10 @@ static int sparc_pmu_event_init(struct perf_event *event)
 	if (atomic_read(&nmi_active) < 0)
 		return -ENODEV;
 
+	/* does not support taken branch sampling */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	switch (attr->type) {
 	case PERF_TYPE_HARDWARE:
 		if (attr->config >= sparc_pmu->max_events)

arch/x86/kernel/cpu/perf_event_amd.c +3 -0

index 0397b23..0d8da03 100644
@@ -138,6 +138,9 @@ static int amd_pmu_hw_config(struct perf_event *event)
 	if (ret)
 		return ret;
 
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	if (event->attr.exclude_host && event->attr.exclude_guest)
 		/*
 		 * When HO == GO == 1 the hardware treats that as GO == HO == 0

kernel/events/core.c +24 -0

index ed39225..36d1a63 100644
@@ -5000,6 +5000,12 @@ static int perf_swevent_init(struct perf_event *event)
 	if (event->attr.type != PERF_TYPE_SOFTWARE)
 		return -ENOENT;
 
+	/*
+	 * no branch sampling for software events
+	 */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	switch (event_id) {
 	case PERF_COUNT_SW_CPU_CLOCK:
 	case PERF_COUNT_SW_TASK_CLOCK:

@@ -5110,6 +5116,12 @@ static int perf_tp_event_init(struct perf_event *event)
 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
 		return -ENOENT;
 
+	/*
+	 * no branch sampling for tracepoint events
+	 */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	err = perf_trace_init(event);
 	if (err)
 		return err;

@@ -5335,6 +5347,12 @@ static int cpu_clock_event_init(struct perf_event *event)
 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
 		return -ENOENT;
 
+	/*
+	 * no branch sampling for software events
+	 */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	perf_swevent_init_hrtimer(event);
 
 	return 0;

@@ -5409,6 +5427,12 @@ static int task_clock_event_init(struct perf_event *event)
 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
 		return -ENOENT;
 
+	/*
+	 * no branch sampling for software events
+	 */
+	if (has_branch_stack(event))
+		return -EOPNOTSUPP;
+
 	perf_swevent_init_hrtimer(event);
 
 	return 0;

kernel/events/hw_breakpoint.c +6 -0

index b0309f7..cee5423 100644
@@ -581,6 +581,12 @@ static int hw_breakpoint_event_init(struct perf_event *bp)
 	if (bp->attr.type != PERF_TYPE_BREAKPOINT)
 		return -ENOENT;
 
+	/*
+	 * no branch sampling for breakpoint events
+	 */
+	if (has_branch_stack(bp))
+		return -EOPNOTSUPP;
+
 	err = register_perf_hw_breakpoint(bp);
 	if (err)
 		return err;

Quoted 42 lines · Stephane Eranian

PERF_SAMPLE_BRANCH_* is disabled for:
- SW events (sw counters, tracepoints)
- HW breakpoints
- ALL but Intel X86 architecture
- AMD64 processors

Signed-off-by Stephane Eranian <[email protected]>

---
 arch/alpha/kernel/perf_event.c       |    4 ++++ arch/arm/kernel/perf_event.c         |    4 ++++ arch/mips/kernel/perf_event_mipsxx.c |    4 ++++ arch/powerpc/kernel/perf_event.c     |    4 ++++ arch/sh/kernel/perf_event.c          |    4 ++++ arch/sparc/kernel/perf_event.c       |    4 ++++ arch/x86/kernel/cpu/perf_event_amd.c |    3 +++ kernel/events/core.c                 |   24 ++++++++++++++++++++++++ kernel/events/hw_breakpoint.c        |    6 ++++++

 9 files changed, 57 insertions(+), 0 deletions(-)

arch/alpha/kernel/perf_event.c +4 -0

index 8143cd7..0dae252 100644
@@ -685,6 +685,10 @@ static int alpha_pmu_event_init(struct perf_event *event)
  {
       int err;
 
+     /* does not support taken branch sampling */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       switch (event->attr.type) {

      case PERF_TYPE_RAW:
      case PERF_TYPE_HARDWARE:

arch/arm/kernel/perf_event.c +0 -0

index 88b0941..42262ff 100644
@@ -540,6 +540,10 @@ static int armpmu_event_init(struct perf_event *event)
       int err = 0;
       atomic_t *active_events = &armpmu->active_events;
 

arch/arm/kernel/perf_event.c static int armpmu_event_init(struct perf_event *event) · line 544

Quoted 2 lines

arch/arm/kernel/perf_event.c +2 -0

+     /* does not support taken branch sampling */
+     if (has_branch_smpl(event))

I guess this would be 'has_branch_stack' instead of 'has_branch_smpl'. 'has_branch_smpl' has not been defined any where but getting called here.

Quoted 142 lines

arch/arm/kernel/perf_event.c +2 -0

+             return -EOPNOTSUPP;
+
       if (armpmu->map_event(event) == -ENOENT)
               return -ENOENT;
 

arch/mips/kernel/perf_event_mipsxx.c +4 -0

index 315fc0b..7070f8c 100644
@@ -606,6 +606,10 @@ static int mipspmu_event_init(struct perf_event *event)
  {
       int err = 0;
 
+     /* does not support taken branch sampling */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       switch (event->attr.type) {

      case PERF_TYPE_RAW:
      case PERF_TYPE_HARDWARE:

arch/powerpc/kernel/perf_event.c +4 -0

index d614ab5..4e0b265 100644
@@ -1078,6 +1078,10 @@ static int power_pmu_event_init(struct perf_event *event)
       if (!ppmu)
               return -ENOENT;
 
+     /* does not support taken branch sampling */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       switch (event->attr.type) {

      case PERF_TYPE_HARDWARE:
              ev = event->attr.config;

arch/sh/kernel/perf_event.c +4 -0

index 10b14e3..068b8a2 100644
@@ -310,6 +310,10 @@ static int sh_pmu_event_init(struct perf_event *event)
  {
       int err;
 
+     /* does not support taken branch sampling */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       switch (event->attr.type) {

      case PERF_TYPE_RAW:
      case PERF_TYPE_HW_CACHE:

arch/sparc/kernel/perf_event.c +4 -0

index 614da62..8e16a4a 100644
@@ -1105,6 +1105,10 @@ static int sparc_pmu_event_init(struct perf_event *event)
       if (atomic_read(&nmi_active) < 0)
               return -ENODEV;
 
+     /* does not support taken branch sampling */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       switch (attr->type) {

      case PERF_TYPE_HARDWARE:
              if (attr->config >= sparc_pmu->max_events)

arch/x86/kernel/cpu/perf_event_amd.c +3 -0

index 0397b23..0d8da03 100644
@@ -138,6 +138,9 @@ static int amd_pmu_hw_config(struct perf_event *event)
       if (ret)
               return ret;
 
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       if (event->attr.exclude_host && event->attr.exclude_guest)
               /*
                * When HO == GO == 1 the hardware treats that as GO == HO == 0

kernel/events/core.c +6 -0

index ed39225..36d1a63 100644
@@ -5000,6 +5000,12 @@ static int perf_swevent_init(struct perf_event *event)
       if (event->attr.type != PERF_TYPE_SOFTWARE)
               return -ENOENT;
 
+     /*
+      * no branch sampling for software events
+      */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       switch (event_id) {

      case PERF_COUNT_SW_CPU_CLOCK:
      case PERF_COUNT_SW_TASK_CLOCK:

kernel/events/core.c +18 -0

@@ -5110,6 +5116,12 @@ static int perf_tp_event_init(struct perf_event *event)
       if (event->attr.type != PERF_TYPE_TRACEPOINT)
               return -ENOENT;
 
+     /*
+      * no branch sampling for tracepoint events
+      */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       err = perf_trace_init(event);
       if (err)
               return err;

@@ -5335,6 +5347,12 @@ static int cpu_clock_event_init(struct perf_event *event)
       if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
               return -ENOENT;
 
+     /*
+      * no branch sampling for software events
+      */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       perf_swevent_init_hrtimer(event);
 
       return 0;

@@ -5409,6 +5427,12 @@ static int task_clock_event_init(struct perf_event *event)
       if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
               return -ENOENT;
 
+     /*
+      * no branch sampling for software events
+      */
+     if (has_branch_stack(event))
+             return -EOPNOTSUPP;
+
       perf_swevent_init_hrtimer(event);
 
       return 0;

kernel/events/hw_breakpoint.c +0 -0

index b0309f7..cee5423 100644
@@ -581,6 +581,12 @@ static int hw_breakpoint_event_init(struct perf_event *bp)
       if (bp->attr.type != PERF_TYPE_BREAKPOINT)
               return -ENOENT;

kernel/events/hw_breakpoint.c static int hw_breakpoint_event_init(struct perf_event *bp) · line 592

Quoted 10 lines

kernel/events/hw_breakpoint.c +6 -0

 
+     /*
+      * no branch sampling for breakpoint events
+      */
+     if (has_branch_stack(bp))
+             return -EOPNOTSUPP;
+
       err = register_perf_hw_breakpoint(bp);
       if (err)
               return err;

--
Linux Technology Centre
IBM Systems and Technology Group
Bangalore India

From: Roberto Agostino Vitillo<[email protected]>

This patch adds:
- ability to parse samples with PERF_SAMPLE_BRANCH_STACK
- sort on branches
- build histograms on branches

  static struct sort_dimension sort_dimensions[] = {

(quoted diff) +2 -6

-	{ .name = "pid",	.entry =&sort_thread,	},
-	{ .name = "comm",	.entry =&sort_comm,	},
-	{ .name = "dso",	.entry =&sort_dso,	},
-	{ .name = "symbol",	.entry =&sort_sym,	},
-	{ .name = "parent",	.entry =&sort_parent,	},
-	{ .name = "cpu",	.entry =&sort_cpu,	},
+	{ .name = "pid",	.entry =&sort_thread,			},
+	{ .name = "comm",	.entry =&sort_comm,			},

Quoted 8 lines · Stephane Eranian

From: Roberto Agostino Vitillo<[email protected]>

This patch adds:
- ability to parse samples with PERF_SAMPLE_BRANCH_STACK
- sort on branches
- build histograms on branches

[..]

Quoted 19 lines

 static struct sort_dimension sort_dimensions[] = {
-       { .name = "pid",        .entry =&sort_thread,   },
-       { .name = "comm",       .entry =&sort_comm,     },
-       { .name = "dso",        .entry =&sort_dso,      },
-       { .name = "symbol",     .entry =&sort_sym,      },
-       { .name = "parent",     .entry =&sort_parent,   },
-       { .name = "cpu",        .entry =&sort_cpu,      },
+       { .name = "pid",        .entry =&sort_thread,                   },
+       { .name = "comm",       .entry =&sort_comm,                     },
+       { .name = "dso",        .entry =&sort_dso,                      },
+       { .name = "dso_from",   .entry =&sort_dso_from,.taken = true    },
+       { .name = "dso_to",     .entry =&sort_dso_to,   .taken = true   },
+       { .name = "symbol",     .entry =&sort_sym,                      },
+       { .name = "symbol_from",.entry =&sort_sym_from,.taken = true    },
+       { .name = "symbol_to",  .entry =&sort_sym_to,   .taken = true   },
+       { .name = "parent",     .entry =&sort_parent,                   },
+       { .name = "cpu",        .entry =&sort_cpu,                      },

Quoted 2 lines

+       { .name = "mispredict", .entry =&sort_mispredict, }, };

The new sort dimensions don't seem to show up in perf report -h. Could you please update the help text?

Also:

# perf script -h

   -f, --fields <str>    comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace,raw. Fields:
comm,tid,pid,time,cpu,event,trace,ip,sym,dso,addr

You probably want to add a field here, so I could:

perf record -b any_call,u -e cycles:u
perf script -f event,branch_stack

and examine raw (symbolized) samples like I can with

perf record -g
perf script -f event,ip,sym

 -Arun

This patchset adds an important and useful new feature to
perf_events: branch stack sampling. In other words, the ability to capture taken branches into each sample.

Statistical sampling of taken branch should not be confused for branch tracing. Not all branches are necessarily captured

Sampling taken branches is important for basic block profiling, statistical call graph, function call counts. Many of those measurements can help drive a compiler optimizer.

The branch stack is a software abstraction which sits on top of the PMU hardware. As such, it is not available on all processors. For now, the patch provides the generic interface and the Intel X86 implementation where it leverages the Last Branch Record (LBR) feature (from Core2 to SandyBridge).

Branch stack sampling is supported for both per-thread and system-wide modes.

It is possible to filter the type and privilege level of branches to sample. The target of the branch is used to determine the privilege level.

For each branch, the source and destination are captured. On some hardware platforms, it may be possible to also extract the target prediction and, in that case, it is also exposed to end users.

The branch stack can record a variable number of taken branches per sample. Those branches are always consecutive in time. The number of branches captured depends on the filtering and the underlying hardware. On Intel Nehalem and later, up to 16 consecutive branches can be captured per sample.

Branch sampling is always coupled with an event. It can be any PMU event but it can't be a SW or tracepoint event.

Branch sampling is requested by setting a new sample_type flag called: PERF_SAMPLE_BRANCH_STACK.

To support branch filtering, we introduce a new field to the perf_event_attr struct: branch_sample_type. We chose NOT to overload the config1, config2 field because those are related to the event encoding. Branch stack is a separate feature which is combined with the event.

The branch_sample_type is a bitmask of possible filters.
The following filters are defined (more can be added):
- PERF_SAMPLE_BRANCH_ANY     : any control flow change
- PERF_SAMPLE_BRANCH_USER    : capture branches when target is at user level
- PERF_SAMPLE_BRANCH_KERNEL  : capture branches when target is at user level
- PERF_SAMPLE_BRANCH_ANY_CALL: capture call branches (incl. syscalls)
- PERF_SAMPLE_BRANCH_ANY_RET : capture return branches (incl. syscall returns)
- PERF_SAMPLE_BRANCH_IND_CALL: capture indirect calls

It is possible to combine filters, e.g., IND_CALL|USER|KERNEL.

When the privilege level is not specified, the branch stack inherits that of the associated event.

Some processors may not offer hardware branch filtering, e.g., Intel Atom. Some may have HW filtering bugs (e.g., Nehalem). The Intel X86 implementation in this patchset also provides a SW branch filter which works on a best effort basis. It can compensate for the lack of LBR filtering. But first and foremost, it helps work around LBR filtering errata. The goal is to only capture the type of branches requested by the user.

It is possible to combine branch stack sampling with PEBS on Intel X86 processors. Depending on the precise_sampling mode, there are certain filterting restrictions. When precise_sampling=1, then there are no filtering restrictions. When precise_sampling > 1, then only ANY|USER|KERNEL filter can be used. This comes from the fact that the kernel uses LBR to compensate for the PEBS off-by-1 skid on the instruction pointer.

To demonstrate how the perf_event branch stack sampling interface works, the patchset also modifies perf record to capture taken branches. Similarly perf report is enhanced to display a histogram of taken branches.

I would like to thank Roberto Vitillo @ LBL for his work on the perf tool for this.

Enough talking, let's take a simple example. Our trivial test program goes like this:

void f2(void)
{}
void f3(void)
{}
void f1(unsigned long n)
{
 if (n & 1UL)
   f2();
 else
   f3();
}
int main(void)
{
 unsigned long i;

 for (i=0; i < N; i++)
  f1(i);
 return 0;
}

$ perf record -b any branchy
$ perf report -b
# Events: 23K cycles
#
# Overhead  Source Symbol     Target Symbol
# ........  ................  ................

   18.13%  [.] f1            [.] main
   18.10%  [.] main          [.] main
   18.01%  [.] main          [.] f1
   15.69%  [.] f1            [.] f1
    9.11%  [.] f3            [.] f1
    6.78%  [.] f1            [.] f3
    6.74%  [.] f1            [.] f2
    6.71%  [.] f2            [.] f1

Of the total number of branches captured, 18.13% were from f1() -> main().

Let's make this clearer by filtering the user call branches only:

$ perf record -b any_call -e cycles:u branchy
$ perf report -b
# Events: 19K cycles
#
# Overhead  Source Symbol              Target Symbol
# ........  .........................  .........................
#
   52.50%  [.] main                   [.] f1
   23.99%  [.] f1                     [.] f3
   23.48%  [.] f1                     [.] f2
    0.03%  [.] _IO_default_xsputn     [.] _IO_new_file_overflow
    0.01%  [k] _start                 [k] __libc_start_main

Now it is more obvious. %52 of all the captured branches where calls from main() -> f1(). The rest is split 50/50 between f1() -> f2() and f1() -> f3() which is expected given that f1() dispatches based on odd vs. even values of n which is constantly increasing.

Here is a kernel example, where we want to sample indirect calls:

$ perf record -a -C 1 -b ind_call -e r1c4:k sleep 10
$ perf report -b
#
# Overhead  Source Symbol               Target Symbol
# ........  ..........................  ..........................
#
   36.36%  [k] __delay                 [k] delay_tsc
    9.09%  [k] ktime_get               [k] read_tsc
    9.09%  [k] getnstimeofday          [k] read_tsc
    9.09%  [k] notifier_call_chain     [k] tick_notify
    4.55%  [k] cpuidle_idle_call       [k] intel_idle
    4.55%  [k] cpuidle_idle_call       [k] menu_reflect
    2.27%  [k] handle_irq              [k] handle_edge_irq
    2.27%  [k] ack_apic_edge           [k] native_apic_mem_write
    2.27%  [k] hpet_interrupt_handler  [k] hrtimer_interrupt
    2.27%  [k] __run_hrtimer           [k] watchdog_timer_fn
    2.27%  [k] enqueue_task            [k] enqueue_task_rt
    2.27%  [k] try_to_wake_up          [k] select_task_rq_rt
    2.27%  [k] do_timer                [k] read_tsc

Due to HW limitations, branch filtering may be approximate on Core, Atom processors. It is more accurate on Nehalem, Westmere and best on Sandy Bridge.

In version 2, we've updated the patch to tip/master (commit 5734857) and we've incoporated the feedback from v1 concerning anynous bitfield struct for branch_stack_entry and the hanlding of i386 ABI binaries on 64-bit host in the instr decoder for the LBR SW filter.

In version 3, we've updated to 3.2.0-tip. The Atom revision check has been put into its own patch. We fixed a browser issue with report report. We fixed all the style issues as well.

Signed-off-by Stephane Eranian <[email protected]>

---

Roberto Agostino Vitillo (3):
 perf: add code to support PERF_SAMPLE_BRANCH_STACK
 perf: add support for sampling taken branch to perf record
 perf: add support for taken branch sampling to perf report

Stephane Eranian (10):
 perf_events: add generic taken branch sampling support (v3)
 perf_events: add Intel LBR MSR definitions
 perf_events: add Intel X86 LBR sharing logic
 perf_events: sync branch stack sampling with X86 precise_sampling
 perf_events: add LBR mappings for PERF_SAMPLE_BRANCH filters
 perf_events: disable LBR support for older Intel Atom processors
 perf_events: implement PERF_SAMPLE_BRANCH for Intel X86
 perf_events: add LBR software filter support for Intel X86
 perf_events: disable PERF_SAMPLE_BRANCH_* when not supported
 perf_events: add hook to flush branch_stack on context switch

 arch/alpha/kernel/perf_event.c             |    4 + arch/arm/kernel/perf_event.c               |    4 + arch/mips/kernel/perf_event_mipsxx.c       |    4 + arch/powerpc/kernel/perf_event.c           |    4 + arch/sh/kernel/perf_event.c                |    4 + arch/sparc/kernel/perf_event.c             |    4 +

 arch/x86/include/asm/msr-index.h           |    7 +
 arch/x86/kernel/cpu/perf_event.c           |   47 +++-
 arch/x86/kernel/cpu/perf_event.h           |   19 +

 arch/x86/kernel/cpu/perf_event_amd.c       |    3 + arch/x86/kernel/cpu/perf_event_intel.c     |  120 +++++-- arch/x86/kernel/cpu/perf_event_intel_ds.c  |   22 +- arch/x86/kernel/cpu/perf_event_intel_lbr.c |  525 ++++++++++++++++++++++++++-- include/linux/perf_event.h                 |   78 ++++- kernel/events/core.c                       |  167 +++++++++ kernel/events/hw_breakpoint.c              |    6 + tools/perf/Documentation/perf-record.txt   |   18 + tools/perf/Documentation/perf-report.txt   |    7 + tools/perf/builtin-record.c                |   69 ++++ tools/perf/builtin-report.c                |   95 +++++- tools/perf/perf.h                          |   18 + tools/perf/util/annotate.c                 |    2 +-

 tools/perf/util/event.h                    |    1 +
 tools/perf/util/evsel.c                    |   14 +
 tools/perf/util/hist.c                     |   93 ++++-
 tools/perf/util/hist.h                     |    7 +

 tools/perf/util/session.c                  |   72 ++++ tools/perf/util/session.h                  |    4 + tools/perf/util/sort.c                     |  361 ++++++++++++++----- tools/perf/util/sort.h                     |    5 + tools/perf/util/symbol.h                   |   13 +

 31 files changed, 1601 insertions(+), 196 deletions(-)

Any comments on this patch set?

Quoted 2 lines · Stephane Eranian

Any comments on this patch set?

Queued it, thanks!

Now, we still need to make progress on:
  - fixing throttling
  - PEBS-LL

  - IBS
  - event scheduling
  - LWP

Quoted 2 lines · Stephane Eranian

Any comments on this patch set?

Queued it, thanks!

Hi,

The branch stack sampling patch exposes a flaw in the sampling buffer format as currently exported by the kernel.

In the current format, sample records (RECORD_SAMPLE) are NOT self-describing. That means that by looking at the fixed size header, it is not possible to determine which event caused the sample to be recorded and what's in the body of the variable length sample.

Such introspection is only possible once we know the event unique id (PERF_SAMPLE_ID). But to get the event ID, we need to parse the sample. But, given that a sample has a variable length, there is no predefined position for that ID in the sample. You have a chicken and egg problem here. There is no room left in the fixed size header to fit this in.

Quoted 10 lines · Stephane Eranian

Hi,

The branch stack sampling patch exposes a flaw in the sampling buffer format as currently exported by the kernel.

In the current format, sample records (RECORD_SAMPLE) are NOT self-describing. That means that by looking at the fixed size header, it is not possible to determine which event caused the sample to be recorded and what's in the body of the variable length sample.

Quoted 6 lines

Such introspection is only possible once we know the event unique id (PERF_SAMPLE_ID). But to get the event ID, we need to parse the sample. But, given that a sample has a variable length, there is no predefined position for that ID in the sample. You have a chicken and egg problem here. There is no room left in the fixed size header to fit this in.

I brought this up last Fall as well. As I recall the response is to move each sample_type based stream into its own data file and then merge the data files while processing.

Quoted 10 lines · Stephane Eranian

Hi,

The branch stack sampling patch exposes a flaw in the sampling buffer format as currently exported by the kernel.

In the current format, sample records (RECORD_SAMPLE) are NOT self-describing. That means that by looking at the fixed size header, it is not possible to determine which event caused the sample to be recorded and what's in the body of the variable length sample.

Quoted 6 lines

Such introspection is only possible once we know the event unique id (PERF_SAMPLE_ID). But to get the event ID, we need to parse the sample. But, given that a sample has a variable length, there is no predefined position for that ID in the sample. You have a chicken and egg problem here. There is no room left in the fixed size header to fit this in.

I brought this up last Fall as well. As I recall the response is to move each sample_type based stream into its own data file and then merge the data files while processing.

David

I would like to thank Roberto Vitillo @ LBL for his work on the perf tool for this.

Enough talking, let's take a simple example. Our trivial test program goes like this:

void f2(void)
{}
void f3(void)
{}
void f1(unsigned long n)
{
if (n & 1UL)

    f2();
  else
    f3();
}

int main(void)
{
unsigned long i;

for (i=0; i < N; i++)
f1(i);
return 0;
}

$ perf record -b any branchy
$ perf report -b
# Events: 23K cycles
#
# Overhead  Source Symbol     Target Symbol
# ........  ................  ................

    18.13%  [.] f1            [.] main                          
    18.10%  [.] main          [.] main                          
    18.01%  [.] main          [.] f1                            
    15.69%  [.] f1            [.] f1                            
     9.11%  [.] f3            [.] f1                            
     6.78%  [.] f1            [.] f3                            
     6.74%  [.] f1            [.] f2                            
     6.71%  [.] f2            [.] f1                            

Of the total number of branches captured, 18.13% were from f1() -> main().

Let's make this clearer by filtering the user call branches only:

$ perf record -b any_call -e cycles:u branchy
$ perf report -b
# Events: 19K cycles
#
# Overhead  Source Symbol              Target Symbol
# ........  .........................  .........................
#
    52.50%  [.] main                   [.] f1                   
    23.99%  [.] f1                     [.] f3                   
    23.48%  [.] f1                     [.] f2                   
     0.03%  [.] _IO_default_xsputn     [.] _IO_new_file_overflow
     0.01%  [k] _start                 [k] __libc_start_main    

Now it is more obvious. %52 of all the captured branches where calls from main() -> f1(). The rest is split 50/50 between f1() -> f2() and f1() -> f3() which is expected given that f1() dispatches based on odd vs. even values of n which is constantly increasing.

Here is a kernel example, where we want to sample indirect calls:

$ perf record -a -C 1 -b ind_call -e r1c4:k sleep 10 
$ perf report -b
#
# Overhead  Source Symbol               Target Symbol
# ........  ..........................  ..........................
#
    36.36%  [k] __delay                 [k] delay_tsc             
     9.09%  [k] ktime_get               [k] read_tsc              
     9.09%  [k] getnstimeofday          [k] read_tsc              
     9.09%  [k] notifier_call_chain     [k] tick_notify           
     4.55%  [k] cpuidle_idle_call       [k] intel_idle            
     4.55%  [k] cpuidle_idle_call       [k] menu_reflect          
     2.27%  [k] handle_irq              [k] handle_edge_irq       
     2.27%  [k] ack_apic_edge           [k] native_apic_mem_write 
     2.27%  [k] hpet_interrupt_handler  [k] hrtimer_interrupt     
     2.27%  [k] __run_hrtimer           [k] watchdog_timer_fn     
     2.27%  [k] enqueue_task            [k] enqueue_task_rt       
     2.27%  [k] try_to_wake_up          [k] select_task_rq_rt     
     2.27%  [k] do_timer                [k] read_tsc              

Due to HW limitations, branch filtering may be approximate on Core, Atom processors. It is more accurate on Nehalem, Westmere and best on Sandy Bridge.

Quoted 21 lines

Here is a kernel example, where we want to sample indirect calls:

$ perf record -a -C 1 -b ind_call -e r1c4:k sleep 10
$ perf report -b
#
# Overhead  Source Symbol               Target Symbol
# ........  ..........................  ..........................
#
     36.36%  [k] __delay                 [k] delay_tsc
      9.09%  [k] ktime_get               [k] read_tsc
      9.09%  [k] getnstimeofday          [k] read_tsc
      9.09%  [k] notifier_call_chain     [k] tick_notify
      4.55%  [k] cpuidle_idle_call       [k] intel_idle
      4.55%  [k] cpuidle_idle_call       [k] menu_reflect
      2.27%  [k] handle_irq              [k] handle_edge_irq
      2.27%  [k] ack_apic_edge           [k] native_apic_mem_write
      2.27%  [k] hpet_interrupt_handler  [k] hrtimer_interrupt
      2.27%  [k] __run_hrtimer           [k] watchdog_timer_fn
      2.27%  [k] enqueue_task            [k] enqueue_task_rt
      2.27%  [k] try_to_wake_up          [k] select_task_rq_rt
      2.27%  [k] do_timer                [k] read_tsc

Quoted 3 lines

Due to HW limitations, branch filtering may be approximate on Core, Atom processors. It is more accurate on Nehalem, Westmere and best on Sandy Bridge.

Can I have you ACK on this userspace stuff (patches 11-13)?