perf script: add script to profile and resolve physical mem type
PATCH v2 1/1 perf script: add script to profile and resolve physical mem type
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload maps to those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.
Profile with MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS event if any of them is available.
Look up the /proc/iomem and resolve the physical address. Provide memory type summary.
Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------
System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%
---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW event
tools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report create mode 100644 tools/perf/scripts/python/mem-phys-addr.py
tools/perf/scripts/python/bin/mem-phys-addr-record new file +19 -0
@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops event +# MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS +# + +load=`perf list | grep mem_inst_retired.all_loads` +if [ -z "$load" ]; then + load=`perf list | grep mem_uops_retired.all_loads` +fi +if [ -z "$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@
tools/perf/scripts/python/bin/mem-phys-addr-report new file +3 -0
@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/mem-phys-addr.py
tools/perf/scripts/python/mem-phys-addr.py new file +97 -0
97 lines not on this page · raw message
tools/perf/util/scripting-engines/trace-event-python.c +2 -0
@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); +pydict_set_item_string_decref (dict_sample,"phys_addr" , +PyLong_FromUnsignedLongLong (sample->phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);pydict_set_item_string_decref (dict,"sample" , dict_sample);
Signature · 1 line
2.7.4
Hi Stephane,
Any comments for the script?
Thanks,
Kan
Quoted 209 lines
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload maps to
those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.Profile with MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS
event if any of them is available.
Look up the/proc/iomemand resolve the physical address. Provide memory type summary.Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%Signed-off-by Kan Liang <[email protected]>---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW eventtools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report create mode 100644 tools/perf/scripts/python/mem-phys-addr.pytools/perf/scripts/python/bin/mem-phys-addr-record new file +19
-0@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops event +# MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS +# + +load=`perf list | grep mem_inst_retired.all_loads` +if [ -z "$load" ]; then + load=`perf list | grep mem_uops_retired.all_loads` +fi +if [ -z "$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@tools/perf/scripts/python/bin/mem-phys-addr-report new file +3
-0@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/mem-phys-addr.pytools/perf/scripts/python/mem-phys-addr.py new file +9
-0@@ -0,0 +1,97 @@ +# mem-phys-addr.py: Resolve physical address samples +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License fortools/perf/scripts/python/mem-phys-addr.py new file +34
-0+# more details. + +from __future__ import division +import os +import sys +import struct +import re +import bisect +import collections + +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +system_ram = [] +pmem = [] +f = None +load_mem_type_cnt = collections.Counter() +event_name = None + +def parse_iomem(): + global f + f = open('/proc/iomem', 'r') + for i, j in enumerate(f): + m = re.split('-|:',j,2) + if m[2].strip() == 'System RAM': + system_ram.append(long(m[0], 16)) + system_ram.append(long(m[1], 16)) + if m[2].strip() == 'Persistent Memory': + pmem.append(long(m[0], 16)) + pmem.append(long(m[1], 16)) + +def print_memory_type(): + print "Event: %s" % (event_name) + print "%-40s %10s %10s\n" % ("Memory type", "count","percentage"),
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+ print "%-40s %10s %10s\n" % ("----------------------------------------", \ + "-----------", "-----------"), + total = sum(load_mem_type_cnt.values()) + for mem_type, count in sorted(load_mem_type_cnt.most_common(),\
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ key = lambda(k, v): (v, k), reverse =True):
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ print "%-40s %10d %10.1f%%\n" % (mem_type, count, 100 *count / total),
tools/perf/scripts/python/mem-phys-addr.py new file +47
-0+ +def trace_begin(): + parse_iomem() + +def trace_end(): + print_memory_type() + f.close() + +def is_system_ram(phys_addr): + #/proc/iomem is sorted + position = bisect.bisect(system_ram, phys_addr) + if position % 2 == 0: + return False + return True + +def is_persistent_mem(phys_addr): + position = bisect.bisect(pmem, phys_addr) + if position % 2 == 0: + return False + return True + +def find_memory_type(phys_addr): + if phys_addr == 0: + return "N/A" + if is_system_ram(phys_addr): + return "System RAM" + + if is_persistent_mem(phys_addr): + return "Persistent Memory" + + #slow path, search all + f.seek(0, 0) + for j in f: + m = re.split('-|:',j,2) + if long(m[0], 16) <= phys_addr <= long(m[1], 16): + return m[2] + return "N/A" + +def process_event(param_dict): + name = param_dict["ev_name"] + sample = param_dict["sample"] + phys_addr = sample["phys_addr"] + + global event_name + if event_name == None: + event_name = name + load_mem_type_cnt[find_memory_type(phys_addr)] += 1tools/perf/util/scripting-engines/trace-event-python.c +2
-0@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); +pydict_set_item_string_decref (dict_sample,"phys_addr" , +PyLong_FromUnsignedLongLong (sample- >phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);
pydict_set_item_string_decref(dict, "sample", dict_sample);--
2.7.4
Hi all,
Any comments for the patch?
Thanks,
Kan
Quoted 209 lines
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload maps to
those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.Profile with MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS
event if any of them is available.
Look up the/proc/iomemand resolve the physical address. Provide memory type summary.Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%Signed-off-by Kan Liang <[email protected]>---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW eventtools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report create mode 100644 tools/perf/scripts/python/mem-phys-addr.pytools/perf/scripts/python/bin/mem-phys-addr-record new file +19
-0@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops event +# MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS +# + +load=`perf list | grep mem_inst_retired.all_loads` +if [ -z "$load" ]; then + load=`perf list | grep mem_uops_retired.all_loads` +fi +if [ -z "$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@tools/perf/scripts/python/bin/mem-phys-addr-report new file +3
-0@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/mem-phys-addr.pytools/perf/scripts/python/mem-phys-addr.py new file +9
-0@@ -0,0 +1,97 @@ +# mem-phys-addr.py: Resolve physical address samples +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License fortools/perf/scripts/python/mem-phys-addr.py new file +34
-0+# more details. + +from __future__ import division +import os +import sys +import struct +import re +import bisect +import collections + +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +system_ram = [] +pmem = [] +f = None +load_mem_type_cnt = collections.Counter() +event_name = None + +def parse_iomem(): + global f + f = open('/proc/iomem', 'r') + for i, j in enumerate(f): + m = re.split('-|:',j,2) + if m[2].strip() == 'System RAM': + system_ram.append(long(m[0], 16)) + system_ram.append(long(m[1], 16)) + if m[2].strip() == 'Persistent Memory': + pmem.append(long(m[0], 16)) + pmem.append(long(m[1], 16)) + +def print_memory_type(): + print "Event: %s" % (event_name) + print "%-40s %10s %10s\n" % ("Memory type", "count","percentage"),
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+ print "%-40s %10s %10s\n" % ("----------------------------------------", \ + "-----------", "-----------"), + total = sum(load_mem_type_cnt.values()) + for mem_type, count in sorted(load_mem_type_cnt.most_common(),\
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ key = lambda(k, v): (v, k), reverse =True):
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ print "%-40s %10d %10.1f%%\n" % (mem_type, count, 100 *count / total),
tools/perf/scripts/python/mem-phys-addr.py new file +47
-0+ +def trace_begin(): + parse_iomem() + +def trace_end(): + print_memory_type() + f.close() + +def is_system_ram(phys_addr): + #/proc/iomem is sorted + position = bisect.bisect(system_ram, phys_addr) + if position % 2 == 0: + return False + return True + +def is_persistent_mem(phys_addr): + position = bisect.bisect(pmem, phys_addr) + if position % 2 == 0: + return False + return True + +def find_memory_type(phys_addr): + if phys_addr == 0: + return "N/A" + if is_system_ram(phys_addr): + return "System RAM" + + if is_persistent_mem(phys_addr): + return "Persistent Memory" + + #slow path, search all + f.seek(0, 0) + for j in f: + m = re.split('-|:',j,2) + if long(m[0], 16) <= phys_addr <= long(m[1], 16): + return m[2] + return "N/A" + +def process_event(param_dict): + name = param_dict["ev_name"] + sample = param_dict["sample"] + phys_addr = sample["phys_addr"] + + global event_name + if event_name == None: + event_name = name + load_mem_type_cnt[find_memory_type(phys_addr)] += 1tools/perf/util/scripting-engines/trace-event-python.c +2
-0@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); +pydict_set_item_string_decref (dict_sample,"phys_addr" , +PyLong_FromUnsignedLongLong (sample- >phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);
pydict_set_item_string_decref(dict, "sample", dict_sample);--
2.7.4
Hi Arnaldo,
Ping.
Any comments for the script?
Thanks,
Kan
Quoted 209 lines
Quoted 5 lines
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload
maps
Quoted 7 lines
to those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.
Profile with MEM_INST_RETIRED.ALL_LOADS or
MEM_UOPS_RETIRED.ALL_LOADS
Quoted 21 lines
event if any of them is available.
Look up the/proc/iomemand resolve the physical address. Provide memory type summary.Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%Signed-off-by Kan Liang <[email protected]>---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW eventQuoted 7 lines
tools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record
Quoted 10 lines
create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report create mode 100644 tools/perf/scripts/python/mem-phys-addr.pytools/perf/scripts/python/bin/mem-phys-addr-record new file +0
-0tools/perf/scripts/python/bin/mem-phys-addr-record line 5
Quoted 5 lines
tools/perf/scripts/python/bin/mem-phys-addr-record new file +5
-0@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops +event # MEM_INST_RETIRED.ALL_LOADS orMEM_UOPS_RETIRED.ALL_LOADS #
Quoted 150 lines
tools/perf/scripts/python/bin/mem-phys-addr-record new file +12
-0+ +load=`perf list | grep mem_inst_retired.all_loads` if [ -z "$load" ]; +then + load=`perf list | grep mem_uops_retired.all_loads` fi if [ -z +"$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@tools/perf/scripts/python/bin/mem-phys-addr-report new file +3
-0@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples perf script $@ -s +"$PERF_EXEC_PATH"/scripts/python/mem-phys-addr.pytools/perf/scripts/python/mem-phys-addr.py new file +9
-0@@ -0,0 +1,97 @@ +# mem-phys-addr.py: Resolve physical address samples # Copyright (c) +2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or +modify it # under the terms and conditions of the GNU General Public +License, # version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but +WITHOUT # ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or
tools/perf/scripts/python/mem-phys-addr.py new file +2
-0+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public +Licensefor
tools/perf/scripts/python/mem-phys-addr.py new file +33
-0+# more details. + +from __future__ import division +import os +import sys +import struct +import re +import bisect +import collections + +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +system_ram = [] +pmem = [] +f = None +load_mem_type_cnt = collections.Counter() event_name = None + +def parse_iomem(): + global f + f = open('/proc/iomem', 'r') + for i, j in enumerate(f): + m = re.split('-|:',j,2) + if m[2].strip() == 'System RAM': + system_ram.append(long(m[0], 16)) + system_ram.append(long(m[1], 16)) + if m[2].strip() == 'Persistent Memory': + pmem.append(long(m[0], 16)) + pmem.append(long(m[1], 16)) + +def print_memory_type(): + print "Event: %s" % (event_name) + print "%-40s %10s %10s\n" % ("Memory type", "count","percentage"),
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+ print "%-40s %10s %10s\n" % ("----------------------------------------", \ + "-----------", "-----------"), + total = sum(load_mem_type_cnt.values()) + for mem_type, count in sorted(load_mem_type_cnt.most_common(),\
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ key = lambda(k, v): (v, k), reverse =True):
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ print "%-40s %10d %10.1f%%\n" % (mem_type, count, 100 *count / total),
tools/perf/scripts/python/mem-phys-addr.py new file +47
-0+ +def trace_begin(): + parse_iomem() + +def trace_end(): + print_memory_type() + f.close() + +def is_system_ram(phys_addr): + #/proc/iomem is sorted + position = bisect.bisect(system_ram, phys_addr) + if position % 2 == 0: + return False + return True + +def is_persistent_mem(phys_addr): + position = bisect.bisect(pmem, phys_addr) + if position % 2 == 0: + return False + return True + +def find_memory_type(phys_addr): + if phys_addr == 0: + return "N/A" + if is_system_ram(phys_addr): + return "System RAM" + + if is_persistent_mem(phys_addr): + return "Persistent Memory" + + #slow path, search all + f.seek(0, 0) + for j in f: + m = re.split('-|:',j,2) + if long(m[0], 16) <= phys_addr <= long(m[1], 16): + return m[2] + return "N/A" + +def process_event(param_dict): + name = param_dict["ev_name"] + sample = param_dict["sample"] + phys_addr = sample["phys_addr"] + + global event_name + if event_name == None: + event_name = name + load_mem_type_cnt[find_memory_type(phys_addr)] += 1tools/perf/util/scripting-engines/trace-event-python.c +2
-0@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); +pydict_set_item_string_decref (dict_sample,"phys_addr" , +PyLong_FromUnsignedLongLong (sample- >phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);
pydict_set_item_string_decref(dict, "sample", dict_sample);--
2.7.4
Hi Stephane and Andi,
Could you please review the script?
If it's OK for you, could you please Ack/Review this?
Thanks,
Kan
Quoted 209 lines
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload maps to
those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.Profile with MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS
event if any of them is available.
Look up the/proc/iomemand resolve the physical address. Provide memory type summary.Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%Signed-off-by Kan Liang <[email protected]>---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW eventtools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report create mode 100644 tools/perf/scripts/python/mem-phys-addr.pytools/perf/scripts/python/bin/mem-phys-addr-record new file +19
-0@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops event +# MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS +# + +load=`perf list | grep mem_inst_retired.all_loads` +if [ -z "$load" ]; then + load=`perf list | grep mem_uops_retired.all_loads` +fi +if [ -z "$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@tools/perf/scripts/python/bin/mem-phys-addr-report new file +3
-0@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/mem-phys-addr.pytools/perf/scripts/python/mem-phys-addr.py new file +9
-0@@ -0,0 +1,97 @@ +# mem-phys-addr.py: Resolve physical address samples +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License fortools/perf/scripts/python/mem-phys-addr.py new file +34
-0+# more details. + +from __future__ import division +import os +import sys +import struct +import re +import bisect +import collections + +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +system_ram = [] +pmem = [] +f = None +load_mem_type_cnt = collections.Counter() +event_name = None + +def parse_iomem(): + global f + f = open('/proc/iomem', 'r') + for i, j in enumerate(f): + m = re.split('-|:',j,2) + if m[2].strip() == 'System RAM': + system_ram.append(long(m[0], 16)) + system_ram.append(long(m[1], 16)) + if m[2].strip() == 'Persistent Memory': + pmem.append(long(m[0], 16)) + pmem.append(long(m[1], 16)) + +def print_memory_type(): + print "Event: %s" % (event_name) + print "%-40s %10s %10s\n" % ("Memory type", "count","percentage"),
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+ print "%-40s %10s %10s\n" % ("----------------------------------------", \ + "-----------", "-----------"), + total = sum(load_mem_type_cnt.values()) + for mem_type, count in sorted(load_mem_type_cnt.most_common(),\
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ key = lambda(k, v): (v, k), reverse =True):
tools/perf/scripts/python/mem-phys-addr.py new file +1
-0+ print "%-40s %10d %10.1f%%\n" % (mem_type, count, 100 *count / total),
tools/perf/scripts/python/mem-phys-addr.py new file +47
-0+ +def trace_begin(): + parse_iomem() + +def trace_end(): + print_memory_type() + f.close() + +def is_system_ram(phys_addr): + #/proc/iomem is sorted + position = bisect.bisect(system_ram, phys_addr) + if position % 2 == 0: + return False + return True + +def is_persistent_mem(phys_addr): + position = bisect.bisect(pmem, phys_addr) + if position % 2 == 0: + return False + return True + +def find_memory_type(phys_addr): + if phys_addr == 0: + return "N/A" + if is_system_ram(phys_addr): + return "System RAM" + + if is_persistent_mem(phys_addr): + return "Persistent Memory" + + #slow path, search all + f.seek(0, 0) + for j in f: + m = re.split('-|:',j,2) + if long(m[0], 16) <= phys_addr <= long(m[1], 16): + return m[2] + return "N/A" + +def process_event(param_dict): + name = param_dict["ev_name"] + sample = param_dict["sample"] + phys_addr = sample["phys_addr"] + + global event_name + if event_name == None: + event_name = name + load_mem_type_cnt[find_memory_type(phys_addr)] += 1tools/perf/util/scripting-engines/trace-event-python.c +2
-0@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); +pydict_set_item_string_decref (dict_sample,"phys_addr" , +PyLong_FromUnsignedLongLong (sample- >phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);
pydict_set_item_string_decref(dict, "sample", dict_sample);--
2.7.4
Dear Kan,
Quoted 102 lines · Kan
Hi Stephane and Andi,
Could you please review the script?
If it's OK for you, could you please Ack/Review this?
Thanks,
KanQuoted 93 lines
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload maps to
those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.Profile with MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS
event if any of them is available.
Look up the/proc/iomemand resolve the physical address. Provide memory type summary.Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%Signed-off-by Kan Liang <[email protected]>---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW eventtools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report create mode 100644 tools/perf/scripts/python/mem-phys-addr.pytools/perf/scripts/python/bin/mem-phys-addr-record new file +19
-0@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops event +# MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS +# + +load=`perf list | grep mem_inst_retired.all_loads` +if [ -z "$load" ]; then + load=`perf list | grep mem_uops_retired.all_loads` +fi +if [ -z "$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@tools/perf/scripts/python/bin/mem-phys-addr-report new file +3
-0@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/mem-phys-addr.pytools/perf/scripts/python/mem-phys-addr.py new file +5
-0@@ -0,0 +1,97 @@ +# mem-phys-addr.py: Resolve physical address samples +# Copyright (c) 2017, Intel Corporation. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License,
Quoted 8 lines
Quoted 8 lines
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
May be you could consider using an SPDX tag instead of this fine but long legalese?
See Thomas doc patches [1] for details
[1] https://lkml.org/lkml/2017/12/28/323
Quoted 9 lines
Quoted 9 lines
tools/perf/scripts/python/mem-phys-addr.py new file +9
-0+ +from __future__ import division +import os +import sys +import struct +import re +import bisect +import collections +
Quoted 8 lines
tools/perf/scripts/python/mem-phys-addr.py line 28
Quoted 8 lines
tools/perf/scripts/python/mem-phys-addr.py new file +8
-0+sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +system_ram = [] +pmem = [] +f = None +load_mem_type_cnt = collections.Counter() +event_name = None
nit. may be it does not matter too much.... but using globals throughout as opposed to return values makes the script harder to read IMHO.
Quoted 92 lines
Quoted 91 lines
tools/perf/scripts/python/mem-phys-addr.py new file +16
-0+ +def parse_iomem(): + global f + f = open('/proc/iomem', 'r') + for i, j in enumerate(f): + m = re.split('-|:',j,2) + if m[2].strip() == 'System RAM': + system_ram.append(long(m[0], 16)) + system_ram.append(long(m[1], 16)) + if m[2].strip() == 'Persistent Memory': + pmem.append(long(m[0], 16)) + pmem.append(long(m[1], 16)) + +def print_memory_type(): + print "Event: %s" % (event_name) + print "%-40s %10s %10s\n" % ("Memory type", "count","percentage"),
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+ print "%-40s %10s %10s\n" % ("----------------------------------------", \ + "-----------", "-----------"), + total = sum(load_mem_type_cnt.values()) + for mem_type, count in sorted(load_mem_type_cnt.most_common(),\
+ key =lambda(k, v): (v, k), reverse =
True):
+ print "%-40s %10d %10.1f%%\n" % (mem_type, count, 100 *
count / total),tools/perf/scripts/python/mem-phys-addr.py new file +47
-0+ +def trace_begin(): + parse_iomem() + +def trace_end(): + print_memory_type() + f.close() + +def is_system_ram(phys_addr): + #/proc/iomem is sorted + position = bisect.bisect(system_ram, phys_addr) + if position % 2 == 0: + return False + return True + +def is_persistent_mem(phys_addr): + position = bisect.bisect(pmem, phys_addr) + if position % 2 == 0: + return False + return True + +def find_memory_type(phys_addr): + if phys_addr == 0: + return "N/A" + if is_system_ram(phys_addr): + return "System RAM" + + if is_persistent_mem(phys_addr): + return "Persistent Memory" + + #slow path, search all + f.seek(0, 0) + for j in f: + m = re.split('-|:',j,2) + if long(m[0], 16) <= phys_addr <= long(m[1], 16): + return m[2] + return "N/A" + +def process_event(param_dict): + name = param_dict["ev_name"] + sample = param_dict["sample"] + phys_addr = sample["phys_addr"] + + global event_name + if event_name == None: + event_name = name + load_mem_type_cnt[find_memory_type(phys_addr)] += 1tools/perf/util/scripting-engines/trace-event-python.c +2
-0@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); +pydict_set_item_string_decref (dict_sample,"phys_addr" , +PyLong_FromUnsignedLongLong (sample- >phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);pydict_set_item_string_decref(dict, "sample", dict_sample);--
2.7.4
Signature · 2 lines
Cordially Philippe Ombredanne
Quoted 117 lines
Dear Kan,
Quoted 14 lines · Kan
Hi Stephane and Andi,
Could you please review the script?
If it's OK for you, could you please Ack/Review this?
Thanks,
KanQuoted 5 lines
From: Kan Liang <[email protected]>
There could be different types of memory in the system. E.g normal System Memory, Persistent Memory. To understand how the workload
maps
Quoted 30 lines
Quoted 30 lines
to
those memories, it's important to know the I/O statistics of them. Perf can collect physical addresses, but those are raw data. It still needs extra work to resolve the physical addresses. Provide a script to facilitate the physical addresses resolving and I/O statistics.Profile with MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS
event if any of them is available.
Look up the/proc/iomemand resolve the physical address. Provide memory type summary.Here is an example output.
#perf script report mem-phys-addr
Event: mem_inst_retired.all_loads:P
Memory type count percentage
---------------------------------------- ----------- -----------System RAM 74 53.2% Persistent Memory 55 39.6% N/A 10 7.2%Signed-off-by Kan Liang <[email protected]>---
Changes since V1:
- Do not mix DLA and Load Latency. Do not compare the loads and stores. Only profile the loads.
- Use event name to replace the RAW eventQuoted 7 lines
Quoted 7 lines
tools/perf/scripts/python/bin/mem-phys-addr-record | 19 +++++ tools/perf/scripts/python/bin/mem-phys-addr-report | 3 + tools/perf/scripts/python/mem-phys-addr.py | 97 ++++++++++++++++++++++ .../util/scripting-engines/trace-event-python.c | 2 + 4 files changed, 121 insertions(+) create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-record
Quoted 1 line
Quoted 1 line
create mode 100644 tools/perf/scripts/python/bin/mem-phys-addr-report
Quoted 30 lines
Quoted 30 lines
create mode 100644 tools/perf/scripts/python/mem-phys-addr.pytools/perf/scripts/python/bin/mem-phys-addr-record new file +19
-0@@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Profiling physical memory by all retired load instructions/uops event +# MEM_INST_RETIRED.ALL_LOADS or MEM_UOPS_RETIRED.ALL_LOADS +# + +load=`perf list | grep mem_inst_retired.all_loads` +if [ -z "$load" ]; then + load=`perf list | grep mem_uops_retired.all_loads` +fi +if [ -z "$load" ]; then + echo "There is no event to count all retired load instructions/uops." + exit 1 +fi + +arg=$(echo $load | tr -d ' ') +arg="$arg:P" +perf record --phys-data -e $arg $@tools/perf/scripts/python/bin/mem-phys-addr-report +0
-0Quoted 8 lines
tools/perf/scripts/python/bin/mem-phys-addr-report line 3
Quoted 8 lines
new file mode 100644
index 0000000..3f2b847tools/perf/scripts/python/bin/mem-phys-addr-report new file +3
-0@@ -0,0 +1,3 @@ +#!/bin/bash +# description: resolve physical address samples +perf script $@ -s "$PERF_EXEC_PATH"/scripts/python/mem-phys-
addr.pyQuoted 10 lines
Quoted 10 lines
tools/perf/scripts/python/mem-phys-addr.py new file +3
-0@@ -0,0 +1,97 @@ +# mem-phys-addr.py: Resolve physical address samples +# Copyright (c) 2017, Intel Corporation. +#Quoted 8 lines
Quoted 8 lines
tools/perf/scripts/python/mem-phys-addr.py new file +6
-0+# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General PublicLicense
Quoted 9 lines
Quoted 2 lines
Quoted 2 lines
for
+# more details.May be you could consider using an SPDX tag instead of this fine but long legalese?
See Thomas doc patches [1] for details
Thanks Philippe.
I will update the license in V3.
Quoted 13 lines
Quoted 13 lines
Quoted 13 lines
tools/perf/scripts/python/mem-phys-addr.py new file +13
-0+ +from __future__ import division +import os +import sys +import struct +import re +import bisect +import collections + +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +system_ram = []
Quoted 9 lines
Quoted 4 lines
tools/perf/scripts/python/mem-phys-addr.py line 28
Quoted 4 lines
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+pmem = [] +f = None +load_mem_type_cnt = collections.Counter() +event_name = Nonenit. may be it does not matter too much.... but using globals throughout as opposed to return values makes the script harder to read IMHO.
The python script is embedded into C code. It implements the standard interfaces. I think the globals should be a good way to share between functions and among each calls. Otherwise, it has to change the interfaces, which will impact other existing scripts.
Thanks,
Kan
Quoted 97 lines
Quoted 85 lines
Quoted 85 lines
tools/perf/scripts/python/mem-phys-addr.py new file +16
-0+ +def parse_iomem(): + global f + f = open('/proc/iomem', 'r') + for i, j in enumerate(f): + m = re.split('-|:',j,2) + if m[2].strip() == 'System RAM': + system_ram.append(long(m[0], 16)) + system_ram.append(long(m[1], 16)) + if m[2].strip() == 'Persistent Memory': + pmem.append(long(m[0], 16)) + pmem.append(long(m[1], 16)) + +def print_memory_type(): + print "Event: %s" % (event_name) + print "%-40s %10s %10s\n" % ("Memory type", "count","percentage"),
tools/perf/scripts/python/mem-phys-addr.py new file +4
-0+ print "%-40s %10s %10s\n" % ("----------------------------------------", \ + "-----------", "-----------"), + total = sum(load_mem_type_cnt.values()) + for mem_type, count in sorted(load_mem_type_cnt.most_common(),\
+ key =lambda(k, v): (v, k), reverse =
True):
+ print "%-40s %10d %10.1f%%\n" % (mem_type, count, 100 *
count / total),tools/perf/scripts/python/mem-phys-addr.py new file +47
-0+ +def trace_begin(): + parse_iomem() + +def trace_end(): + print_memory_type() + f.close() + +def is_system_ram(phys_addr): + #/proc/iomem is sorted + position = bisect.bisect(system_ram, phys_addr) + if position % 2 == 0: + return False + return True + +def is_persistent_mem(phys_addr): + position = bisect.bisect(pmem, phys_addr) + if position % 2 == 0: + return False + return True + +def find_memory_type(phys_addr): + if phys_addr == 0: + return "N/A" + if is_system_ram(phys_addr): + return "System RAM" + + if is_persistent_mem(phys_addr): + return "Persistent Memory" + + #slow path, search all + f.seek(0, 0) + for j in f: + m = re.split('-|:',j,2) + if long(m[0], 16) <= phys_addr <= long(m[1], 16): + return m[2] + return "N/A" + +def process_event(param_dict): + name = param_dict["ev_name"] + sample = param_dict["sample"] + phys_addr = sample["phys_addr"] + + global event_name + if event_name == None: + event_name = name + load_mem_type_cnt[find_memory_type(phys_addr)] += 1tools/perf/util/scripting-engines/trace-event-python.c +2
-0@@ -500,6 +500,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, PyLong_FromUnsignedLongLong(sample->time));pydict_set_item_string_decref (dict_sample,"period" ,PyLong_FromUnsignedLongLong (sample->period)); + pydict_set_item_string_decref(dict_sample, "phys_addr", + PyLong_FromUnsignedLongLong(sample-Quoted 7 lines
Quoted 6 lines
tools/perf/util/scripting-engines/trace-event-python.c +0
-0>phys_addr));set_sample_read_in_dict (dict_sample, sample, evsel);pydict_set_item_string_decref(dict, "sample", dict_sample);--
2.7.4--
Cordially
Philippe Ombredanne