perf: Add PERF_SAMPLE_PHYS_ADDR

From: Kan Liang <[email protected]>

For understanding how the workload maps to memory channels and hardware behavior, it's very important to collect address maps with physical addresses. For example, 3D XPoint access can only be found by filtering the physical address.
However, perf doesn't collect physical address information in sampling.

The virtual address which stored in data->addr can be used to calculate the physical address.
For kernel direct mapping addresses, virt_to_phys is used to convert the virtual addresses to physical address.
For user virtual addresses, __get_user_pages_fast is used to walk the pages tables for user physical address.
This does not work for vmalloc addresses. Right now these are not resolved, but code to do that could be added.
For security, the physical address can only be exposed to root or privileged user.
A new sample type PERF_SAMPLE_PHYS_ADDR is introduced to expose the physical addresses.

Signed-off-by Kan Liang <[email protected]>

---

This patch is kernel patch.
The user space patch can be found here.
https://www.spinics.net/lists/kernel/msg2587093.html

Changes since V5
- Move virt_to_phys to generic code (PeterZ)

 arch/x86/events/perf_event.h    |  2 +-
 include/linux/perf_event.h      |  2 ++
 include/uapi/linux/perf_event.h |  4 +++-
 kernel/events/core.c            | 47 +++++++++++++++++++++++++++++++++++++++++
 4 files changed, 53 insertions(+), 2 deletions(-)

arch/x86/events/perf_event.h +1 -1

index 476aec3..65bb91e 100644
@@ -91,7 +91,7 @@ struct amd_nb {
 	(PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
 	PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
 	PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
-	PERF_SAMPLE_TRANSACTION)
+	PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR)
 
 /*
  * A debug store configuration.

include/linux/perf_event.h +2 -0

index b14095b..74fb87e 100644
@@ -944,6 +944,8 @@ struct perf_sample_data {
 
 	struct perf_regs		regs_intr;
 	u64				stack_user_size;
+
+	u64				phys_addr;
 } ____cacheline_aligned;
 
 /* default value for data source */

include/uapi/linux/perf_event.h +3 -1

index 642db5f..cbea02f 100644
@@ -139,8 +139,9 @@ enum perf_event_sample_format {
 	PERF_SAMPLE_IDENTIFIER			= 1U << 16,
 	PERF_SAMPLE_TRANSACTION			= 1U << 17,
 	PERF_SAMPLE_REGS_INTR			= 1U << 18,
+	PERF_SAMPLE_PHYS_ADDR			= 1U << 19,
 
-	PERF_SAMPLE_MAX = 1U << 19,		/* non-ABI */
+	PERF_SAMPLE_MAX = 1U << 20,		/* non-ABI */
 };
 
 /*

@@ -814,6 +815,7 @@ enum perf_event_type {
 	 *	{ u64			transaction; } && PERF_SAMPLE_TRANSACTION
 	 *	{ u64			abi; # enum perf_sample_regs_abi
 	 *	  u64			regs[weight(mask)]; } && PERF_SAMPLE_REGS_INTR
+	 *	{ u64			phys_addr;} && PERF_SAMPLE_PHYS_ADDR
 	 * };
 	 */
 	PERF_RECORD_SAMPLE			= 9,

kernel/events/core.c +27 -0

index d704e23..b991af3 100644
@@ -1570,6 +1570,9 @@ static void __perf_event_header_size(struct perf_event *event, u64 sample_type)
 	if (sample_type & PERF_SAMPLE_TRANSACTION)
 		size += sizeof(data->txn);
 
+	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+		size += sizeof(data->phys_addr);
+
 	event->header_size = size;
 }
 

@@ -6012,6 +6015,9 @@ void perf_output_sample(struct perf_output_handle *handle,
 		}
 	}
 
+	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+		perf_output_put(handle, data->phys_addr);
+
 	if (!event->attr.watermark) {
 		int wakeup_events = event->attr.wakeup_events;
 

@@ -6027,6 +6033,38 @@ void perf_output_sample(struct perf_output_handle *handle,
 	}
 }
 
+static u64 perf_virt_to_phys(u64 virt)
+{
+	u64 phys_addr = 0;
+	struct page *p = NULL;
+
+	if (!virt)
+		return 0;
+
+	if (virt >= TASK_SIZE) {
+		/* If it's vmalloc()d memory, leave phys_addr as 0 */
+		if (virt_addr_valid(virt) &&
+		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
+			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
+	} else {
+		/*
+		 * Walking the pages tables for user address.
+		 * Interrupts are disabled, so it prevents any tear down
+		 * of the page tables.
+		 * Try IRQ-safe __get_user_pages_fast first.
+		 * If failed, leave phys_addr as 0.
+		 */

kernel/events/core.c +6 -0

 }
 
 static void __always_inline

@@ -9892,6 +9933,12 @@ SYSCALL_DEFINE5(perf_event_open,
 			return -EINVAL;
 	}
 
+	/* Only privileged users can get kernel addresses */
+	if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR) &&
+	    perf_paranoid_kernel() &&
+	    !capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
 	if (!attr.sample_max_stack)
 		attr.sample_max_stack = sysctl_perf_event_max_stack;
 

Signature · 1 line

2.4.3

kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 6188

Quoted 7 lines

kernel/events/core.c +3 -0

@@ -6145,6 +6183,9 @@ void perf_prepare_sample(struct perf_event_header *header,
 
 		header->size += size;
 	}
+
+	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+		data->phys_addr = perf_virt_to_phys(data->addr);

Only problem with this now is that it requires SAMPLE_ADDR to also be set in order to obtain data->addr.

Either fix all to set data->attr for (SAMPLE_ADDR || SAMPLE_PHYS_ADDR) or mandate SAMPLE_ADDR when SAMPLE_PHYS_ADDR.

Quoted 11 lines · Peter Zijlstra

kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 6188

Quoted 7 lines

kernel/events/core.c +3 -0

@@ -6145,6 +6183,9 @@ void perf_prepare_sample(struct perf_event_header *header,
 
            header->size += size;
    }
+
+   if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+           data->phys_addr = perf_virt_to_phys(data->addr);

Only problem with this now is that it requires SAMPLE_ADDR to also be set in order to obtain data->addr.

Quoted 2 lines

Either fix all to set data->attr for (SAMPLE_ADDR || SAMPLE_PHYS_ADDR) or mandate SAMPLE_ADDR when SAMPLE_PHYS_ADDR.

I think the former suggestion is better, as it allows for smaller samples.

Quoted 15 lines · Peter Zijlstra

Quoted 13 lines

kernel/events/core.c void perf_prepare_sample(struct perf_event_header *header, · line 6188

Quoted 7 lines

kernel/events/core.c +3 -0

@@ -6145,6 +6183,9 @@ void perf_prepare_sample(struct perf_event_header *header,
 
            header->size += size;
    }
+
+   if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+           data->phys_addr = perf_virt_to_phys(data->addr);

Only problem with this now is that it requires SAMPLE_ADDR to also be set in order to obtain data->addr.

Either fix all to set data->attr for (SAMPLE_ADDR || SAMPLE_PHYS_ADDR) or mandate SAMPLE_ADDR when SAMPLE_PHYS_ADDR.

Quoted 2 lines

I think the former suggestion is better, as it allows for smaller samples.

Agreed. I may only care about physical addresses, so SAMPLE_PHYS_ADDR should suffice from the user.
The kernel must save the virtual address in data->addr, and then convert it.

Quoted 12 lines

Review void perf_prepare_sample(struct · line 6189

Quoted 8 lines

(quoted diff) +3 -0

@@ -6145,6 +6183,9 @@ void perf_prepare_sample(struct
 perf_event_header *header,
 
 		header->size += size;
 	}
+
+	if (sample_type & PERF_SAMPLE_PHYS_ADDR)
+		data->phys_addr = perf_virt_to_phys(data->addr);

Only problem with this now is that it requires SAMPLE_ADDR to also be set in order to obtain data->addr.

Quoted 2 lines

Either fix all to set data->attr for (SAMPLE_ADDR || SAMPLE_PHYS_ADDR) or mandate SAMPLE_ADDR when SAMPLE_PHYS_ADDR.

I think the former suggestion is better, as it allows for smaller samples.

For x86, I think we can do something as below. But I'm not sure other architectures.

Quoted 33 lines

kernel/events/core.c +25 -0

index d704e23..b991af3 100644
@@ -6027,6 +6033,38 @@ void perf_output_sample(struct perf_output_handle *handle,
 	}
 }
 
+static u64 perf_virt_to_phys(u64 virt)
+{
+	u64 phys_addr = 0;
+	struct page *p = NULL;
+
+	if (!virt)
+		return 0;
+
+	if (virt >= TASK_SIZE) {
+		/* If it's vmalloc()d memory, leave phys_addr as 0 */
+		if (virt_addr_valid(virt) &&
+		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
+			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
+	} else {
+		/*
+		 * Walking the pages tables for user address.
+		 * Interrupts are disabled, so it prevents any tear down
+		 * of the page tables.
+		 * Try IRQ-safe __get_user_pages_fast first.
+		 * If failed, leave phys_addr as 0.
+		 */
+		if ((current->mm != NULL) &&
+		    (__get_user_pages_fast(virt, 1, 0, &p) == 1))
+			phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
+

kernel/events/core.c void perf_output_sample(struct perf_output_handle *handle, · line 6066

Quoted 6 lines

kernel/events/core.c +6 -0

+		if (p)
+			put_page(p);
+	}
+
+	return phys_addr;
+}

Michael, does this work for PPC as is?

Quoted 33 lines · Peter Zijlstra

Quoted 33 lines

kernel/events/core.c +25 -0

index d704e23..b991af3 100644
@@ -6027,6 +6033,38 @@ void perf_output_sample(struct perf_output_handle *handle,
 	}
 }
 
+static u64 perf_virt_to_phys(u64 virt)
+{
+	u64 phys_addr = 0;
+	struct page *p = NULL;
+
+	if (!virt)
+		return 0;
+
+	if (virt >= TASK_SIZE) {
+		/* If it's vmalloc()d memory, leave phys_addr as 0 */
+		if (virt_addr_valid(virt) &&
+		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
+			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
+	} else {
+		/*
+		 * Walking the pages tables for user address.
+		 * Interrupts are disabled, so it prevents any tear down
+		 * of the page tables.
+		 * Try IRQ-safe __get_user_pages_fast first.
+		 * If failed, leave phys_addr as 0.
+		 */
+		if ((current->mm != NULL) &&
+		    (__get_user_pages_fast(virt, 1, 0, &p) == 1))
+			phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
+

Quoted 8 lines

kernel/events/core.c void perf_output_sample(struct perf_output_handle *handle, · line 6066

Quoted 6 lines

kernel/events/core.c +6 -0

+		if (p)
+			put_page(p);
+	}
+
+	return phys_addr;
+}

Michael, does this work for PPC as is?

I think so.

We have about 8 MMUs and 32-bit and 64-bit so it's a bit hard to say for sure.

I'm pretty sure virt_addr_valid() will exclude everything except the linear mapping for us, so the vmalloc check is redundant but that's fine. So that looks safe AFAICS.

I'm not an expert on GUP fast, but AFAIK there's nothing special required for us.

Quoted 43 lines · Michael Ellerman

Quoted 41 lines · Peter Zijlstra

Quoted 29 lines

kernel/events/core.c +21 -0

index d704e23..b991af3 100644
@@ -6027,6 +6033,38 @@ void perf_output_sample(struct perf_output_handle *handle,
 	}
 }
 
+static u64 perf_virt_to_phys(u64 virt)
+{
+	u64 phys_addr = 0;
+	struct page *p = NULL;
+
+	if (!virt)
+		return 0;
+
+	if (virt >= TASK_SIZE) {
+		/* If it's vmalloc()d memory, leave phys_addr as 0 */
+		if (virt_addr_valid(virt) &&
+		    !(virt >= VMALLOC_START && virt < VMALLOC_END))
+			phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt);
+	} else {
+		/*
+		 * Walking the pages tables for user address.
+		 * Interrupts are disabled, so it prevents any tear down
+		 * of the page tables.
+		 * Try IRQ-safe __get_user_pages_fast first.
+		 * If failed, leave phys_addr as 0.
+		 */

kernel/events/core.c void perf_output_sample(struct perf_output_handle *handle, · line 6066

Quoted 10 lines

kernel/events/core.c +10 -0

+		if ((current->mm != NULL) &&
+		    (__get_user_pages_fast(virt, 1, 0, &p) == 1))
+			phys_addr = page_to_phys(p) + virt % PAGE_SIZE;
+
+		if (p)
+			put_page(p);
+	}
+
+	return phys_addr;
+}

Michael, does this work for PPC as is?

Quoted 4 lines

I think so.

We have about 8 MMUs and 32-bit and 64-bit so it's a bit hard to say for sure.

You can limit it to those that include a PMU that supports data->addr, which is book3s, whatever that gets you.