mm/migrate_device: add class-based migration trace events
Subsystems (3)
PATCH v2 1/1 mm/migrate_device: add class-based migration trace events
Device-private migrations currently require ad-hoc instrumentation to understand selection, unmap, copy, and finalization behavior.
Add reusable ftrace event classes for migrate_vma ranges, device batches, and individual folios. Emit start, done traces with ranges, flags, page counts, success/failure summaries, PFNs, and migrate flags.
Enabling all traces can make the migration traces verbose, when used they can also help debug issues with device migration.
--- include/trace/events/migrate_device.h | 324 ++++++++++++++++++++++++++ mm/migrate_device.c | 79 ++++++- 2 files changed, 391 insertions(+), 12 deletions(-) create mode 100644 include/trace/events/migrate_device.h
include/trace/events/migrate_device.h new file +324 -0
@@ -0,0 +1,324 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +*/ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM migrate_device + +#if !defined(_TRACE_MIGRATE_DEVICE_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_MIGRATE_DEVICE_H + +#include <linux/migrate.h> +#include <linux/tracepoint.h> + +/* +* Define enums for tracing information. +*/ +#ifndef __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY +#define __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY + +/* +* Why a page table walk callback classified the range the way it did. +* Several call sites collect the same kind of range, so the reason is +* what distinguishes them in a trace. +*/ +enum migrate_vma_walk_reason { +MIGRATE_VMA_WALK_HOLE , +MIGRATE_VMA_WALK_HOLE_COMPOUND , +MIGRATE_VMA_WALK_HUGE_PMD , +MIGRATE_VMA_WALK_SKIP_NOT_ANON , +MIGRATE_VMA_WALK_SKIP_NOT_SELECTED , +MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH , +MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED , +MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL , +MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED , +}; + +#endif /* __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY */ + +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_SYSTEM ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_DEVICE_PRIVATE ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_DEVICE_COHERENT ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_COMPOUND ); + +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_HOLE ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_HOLE_COMPOUND ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_HUGE_PMD ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_NOT_ANON ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_NOT_SELECTED ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED ); + +#define show_walk_reason (reason) \ + __print_symbolic(reason, \ + { MIGRATE_VMA_WALK_HOLE, "HOLE" }, \ + { MIGRATE_VMA_WALK_HOLE_COMPOUND, "HOLE_COMPOUND" }, \ + { MIGRATE_VMA_WALK_HUGE_PMD, "HUGE_PMD" }, \ + { MIGRATE_VMA_WALK_SKIP_NOT_ANON, "NOT_ANON" }, \ + { MIGRATE_VMA_WALK_SKIP_NOT_SELECTED, "NOT_SELECTED" }, \ + { MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH, "OWNER_MISMATCH" }, \ + { MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED, "LOCK_CONTENDED" }, \ + { MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL, "COMPOUND_TAIL" }, \ + { MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED, "SPLIT_FAILED" }) + +#define show_migrate_vma_flags (flags) \ + __print_flags(flags, "|", \ + { MIGRATE_VMA_SELECT_SYSTEM, "SYSTEM" }, \ + { MIGRATE_VMA_SELECT_DEVICE_PRIVATE, "DEVICE_PRIVATE" }, \ + { MIGRATE_VMA_SELECT_DEVICE_COHERENT, "DEVICE_COHERENT" }, \ + { MIGRATE_VMA_SELECT_COMPOUND, "COMPOUND" }) + +DECLARE_EVENT_CLASS (migrate_vma_range_class, + + TP_PROTO(const struct migrate_vma *migrate), + +TP_ARGS (migrate), + +TP_STRUCT__entry ( + __field(void *, vma) + __field(void *, mm) + __field(void *, src) + __field(void *, dst) + __field(void *, pgmap_owner) + __field(void *, fault_page) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned long, npages) + __field(unsigned long, cpages) + __field(unsigned long, flags) + ), + +TP_fast_assign ( + __entry->vma = migrate->vma; + __entry->mm = migrate->vma ? migrate->vma->vm_mm : NULL; + __entry->src = migrate->src; + __entry->dst = migrate->dst; + __entry->pgmap_owner = migrate->pgmap_owner; + __entry->fault_page = migrate->fault_page; + __entry->start = migrate->start; + __entry->end = migrate->end; + __entry->npages = migrate->npages; + __entry->cpages = migrate->cpages; + __entry->flags = migrate->flags; + ), + +TP_printk ("mm=%p vma=%p range=%#lx-%#lx bytes=%lu npages=%lu cpages=%lu flags=%s src=%p dst=%p owner=%p fault_page=%p" , + __entry->mm, __entry->vma, __entry->start, __entry->end, + __entry->end - __entry->start, __entry->npages, + __entry->cpages, +show_migrate_vma_flags (__entry->flags), + __entry->src, __entry->dst, __entry->pgmap_owner, + __entry->fault_page) +); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_setup_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_setup_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_collect_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_collect_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_pages_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_pages_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_finalize_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_finalize_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +/* +* Page table walk callbacks act on a sub-range of the migration and can be +* reached from several call sites, so they report the range they were handed +* and why, in addition to the migrate-wide state. +*/ +DECLARE_EVENT_CLASS (migrate_vma_walk_class, + + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), + +TP_ARGS (migrate, start, end, reason), + +TP_STRUCT__entry ( + __field(void *, vma) + __field(void *, mm) + __field(void *, src) + __field(void *, dst) + __field(void *, pgmap_owner) + __field(void *, fault_page) + __field(unsigned long, walk_start) + __field(unsigned long, walk_end) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned long, npages) + __field(unsigned long, cpages) + __field(unsigned long, flags) + __field(unsigned int, reason) + ), + +TP_fast_assign ( + __entry->vma = migrate->vma; + __entry->mm = migrate->vma ? migrate->vma->vm_mm : NULL; + __entry->src = migrate->src; + __entry->dst = migrate->dst; + __entry->pgmap_owner = migrate->pgmap_owner; + __entry->fault_page = migrate->fault_page; + __entry->walk_start = start; + __entry->walk_end = end; + __entry->start = migrate->start; + __entry->end = migrate->end; + __entry->npages = migrate->npages; + __entry->cpages = migrate->cpages; + __entry->flags = migrate->flags; + __entry->reason = reason; + ), + +TP_printk ("mm=%p vma=%p walk=%#lx-%#lx bytes=%lu reason=%s range=%#lx-%#lx npages=%lu cpages=%lu flags=%s src=%p dst=%p owner=%p fault_page=%p" , + __entry->mm, __entry->vma, + __entry->walk_start, __entry->walk_end, + __entry->walk_end - __entry->walk_start, +show_walk_reason (__entry->reason), + __entry->start, __entry->end, __entry->npages, + __entry->cpages, +show_migrate_vma_flags (__entry->flags), + __entry->src, __entry->dst, __entry->pgmap_owner, + __entry->fault_page) +); + +DEFINE_EVENT (migrate_vma_walk_class, migrate_vma_collect_skip, + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), +TP_ARGS (migrate, start, end, reason)); + +DEFINE_EVENT (migrate_vma_walk_class, migrate_vma_collect_hole, + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), +TP_ARGS (migrate, start, end, reason)); + +DEFINE_EVENT (migrate_vma_walk_class, migrate_vma_collect_huge_pmd, + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), +TP_ARGS (migrate, start, end, reason)); + +DECLARE_EVENT_CLASS (migrate_device_batch_class, + + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), + +TP_ARGS (src, dst, npages), + +TP_STRUCT__entry ( + __field(void *, src) + __field(void *, dst) + __field(unsigned long, src_head) + __field(unsigned long, dst_head) + __field(unsigned long, npages) + ), + +TP_fast_assign ( + __entry->src = (void *)src; + __entry->dst = (void *)dst; + __entry->src_head = npages && src ? src[0] : 0; + __entry->dst_head = npages && dst ? dst[0] : 0; + __entry->npages = npages; + ), + +TP_printk ("src=%p dst=%p npages=%lu src_head=%#lx dst_head=%#lx" , + __entry->src, __entry->dst, __entry->npages, + __entry->src_head, __entry->dst_head) +); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_unmap_done, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_pages_start, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_pages_done, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_finalize_start, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_finalize_done, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DECLARE_EVENT_CLASS (migrate_device_folio_class, + + TP_PROTO(unsigned long index, unsigned long nr_pages, + unsigned long src, unsigned long dst), + +TP_ARGS (index, nr_pages, src, dst), + +TP_STRUCT__entry ( + __field(unsigned long, index) + __field(unsigned long, nr_pages) + __field(unsigned long, src) + __field(unsigned long, dst) + __field(unsigned long, src_pfn) + __field(unsigned long, dst_pfn) + ), + +TP_fast_assign ( + __entry->index = index; + __entry->nr_pages = nr_pages; + __entry->src = src; + __entry->dst = dst; + __entry->src_pfn = src >> MIGRATE_PFN_SHIFT; + __entry->dst_pfn = dst >> MIGRATE_PFN_SHIFT; + ), + +TP_printk ("index=%lu nr_pages=%lu src=%#lx src_pfn=%#lx src_flags=%s dst=%#lx dst_pfn=%#lx dst_flags=%s" , + __entry->index, __entry->nr_pages, __entry->src, + __entry->src_pfn, +__print_flags (__entry->src & ((1UL <<MIGRATE_PFN_SHIFT ) -1 ), +"|" , + { MIGRATE_PFN_VALID, "VALID" }, + { MIGRATE_PFN_MIGRATE, "MIGRATE" }, + { MIGRATE_PFN_WRITE, "WRITE" }, + { MIGRATE_PFN_COMPOUND, "COMPOUND" }), + __entry->dst, __entry->dst_pfn, +__print_flags (__entry->dst & ((1UL <<MIGRATE_PFN_SHIFT ) -1 ), +"|" , + { MIGRATE_PFN_VALID, "VALID" }, + { MIGRATE_PFN_MIGRATE, "MIGRATE" }, + { MIGRATE_PFN_WRITE, "WRITE" }, + { MIGRATE_PFN_COMPOUND, "COMPOUND" })) +); + +DEFINE_EVENT (migrate_device_folio_class, migrate_device_folio_finalize, + TP_PROTO(unsigned long index, unsigned long nr_pages, + unsigned long src, unsigned long dst), +TP_ARGS (index, nr_pages, src, dst)); + +#endif /* _TRACE_MIGRATE_DEVICE_H */ + +#include <trace/define_trace.h>
mm/migrate_device.c +67 -12
238 lines not on this page · raw message
Signature · 1 line
2.55.0
Matthew Brost
reviewerQuoted 8 lines · Balbir Singh
Device-private migrations currently require ad-hoc instrumentation to understand selection, unmap, copy, and finalization behavior.
Add reusable ftrace event classes for
migrate_vmaranges, device batches, and individual folios. Emit start, done traces with ranges, flags, page counts, success/failure summaries, PFNs, and migrate flags.
Quoted 5 lines
Enabling all traces can make the migration traces verbose, when used they can also help debug issues with device migration.
Signed-off-by Balbir Singh <[email protected]>Suggested-by Steven Rostedt <[email protected]>
This seems like a pretty good adding tracepoints - anytime I've had to debug issues in migrate_device.c it has basically start adding printks to figure out what is going on. Hopefully witth the right tracepoints, this would now be turn on ftrace.
So +1 on the idea.
In the middle of a few other things, but will circle back with an in depth look at individual tracepoints added here when a bit more time.
Matt
Quoted 619 lines
Assisted-by Codex:gpt-5.6-solCc Steven Rostedt <[email protected]>CC Masami Hiramatsu <[email protected]>CC Mathieu Desnoyers <[email protected]>CC Andrew Morton <[email protected]>CC David Hildenbrand <[email protected]>CC Zi Yan <[email protected]>CC Matthew Brost <[email protected]>CC Joshua Hahn <[email protected]>CC Rakie Kim <[email protected]>CC Byungchul Park <[email protected]>CC Gregory Price <[email protected]>CC Ying Huang <[email protected]>CC Alistair Popple <[email protected]>--- include/trace/events/migrate_device.h | 324 ++++++++++++++++++++++++++ mm/migrate_device.c | 79 ++++++- 2 files changed, 391 insertions(+), 12 deletions(-) create mode 100644 include/trace/events/migrate_device.hinclude/trace/events/migrate_device.h new file +324
-0@@ -0,0 +1,324 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +* Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +*/ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM migrate_device + +#if !defined(_TRACE_MIGRATE_DEVICE_H) || defined(TRACE_HEADER_MULTI_READ ) +#define _TRACE_MIGRATE_DEVICE_H + +#include <linux/migrate.h> +#include <linux/tracepoint.h> + +/* +* Define enums for tracing information. +*/ +#ifndef __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY +#define __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY + +/* +* Why a page table walk callback classified the range the way it did. +* Several call sites collect the same kind of range, so the reason is +* what distinguishes them in a trace. +*/ +enum migrate_vma_walk_reason { +MIGRATE_VMA_WALK_HOLE , +MIGRATE_VMA_WALK_HOLE_COMPOUND , +MIGRATE_VMA_WALK_HUGE_PMD , +MIGRATE_VMA_WALK_SKIP_NOT_ANON , +MIGRATE_VMA_WALK_SKIP_NOT_SELECTED , +MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH , +MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED , +MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL , +MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED , +}; + +#endif /* __MIGRATE_DEVICE_DECLARE_TRACE_ENUMS_ONCE_ONLY */ + +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_SYSTEM ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_DEVICE_PRIVATE ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_DEVICE_COHERENT ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_SELECT_COMPOUND ); + +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_HOLE ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_HOLE_COMPOUND ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_HUGE_PMD ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_NOT_ANON ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_NOT_SELECTED ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL ); +TRACE_DEFINE_ENUM (MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED ); + +#define show_walk_reason (reason) \ + __print_symbolic(reason, \ + { MIGRATE_VMA_WALK_HOLE, "HOLE" }, \ + { MIGRATE_VMA_WALK_HOLE_COMPOUND, "HOLE_COMPOUND" }, \ + { MIGRATE_VMA_WALK_HUGE_PMD, "HUGE_PMD" }, \ + { MIGRATE_VMA_WALK_SKIP_NOT_ANON, "NOT_ANON" }, \ + { MIGRATE_VMA_WALK_SKIP_NOT_SELECTED, "NOT_SELECTED" }, \ + { MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH, "OWNER_MISMATCH" }, \ + { MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED, "LOCK_CONTENDED" }, \ + { MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL, "COMPOUND_TAIL" }, \ + { MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED, "SPLIT_FAILED" }) + +#define show_migrate_vma_flags (flags) \ + __print_flags(flags, "|", \ + { MIGRATE_VMA_SELECT_SYSTEM, "SYSTEM" }, \ + { MIGRATE_VMA_SELECT_DEVICE_PRIVATE, "DEVICE_PRIVATE" }, \ + { MIGRATE_VMA_SELECT_DEVICE_COHERENT, "DEVICE_COHERENT" }, \ + { MIGRATE_VMA_SELECT_COMPOUND, "COMPOUND" }) + +DECLARE_EVENT_CLASS (migrate_vma_range_class, + + TP_PROTO(const struct migrate_vma *migrate), + +TP_ARGS (migrate), + +TP_STRUCT__entry ( + __field(void *, vma) + __field(void *, mm) + __field(void *, src) + __field(void *, dst) + __field(void *, pgmap_owner) + __field(void *, fault_page) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned long, npages) + __field(unsigned long, cpages) + __field(unsigned long, flags) + ), + +TP_fast_assign ( + __entry->vma = migrate->vma; + __entry->mm = migrate->vma ? migrate->vma->vm_mm : NULL; + __entry->src = migrate->src; + __entry->dst = migrate->dst; + __entry->pgmap_owner = migrate->pgmap_owner; + __entry->fault_page = migrate->fault_page; + __entry->start = migrate->start; + __entry->end = migrate->end; + __entry->npages = migrate->npages; + __entry->cpages = migrate->cpages; + __entry->flags = migrate->flags; + ), + +TP_printk ("mm=%p vma=%p range=%#lx-%#lx bytes=%lu npages=%lu cpages=%lu flags=%s src=%p dst=%p owner=%p fault_page=%p" , + __entry->mm, __entry->vma, __entry->start, __entry->end, + __entry->end - __entry->start, __entry->npages, + __entry->cpages, +show_migrate_vma_flags (__entry->flags), + __entry->src, __entry->dst, __entry->pgmap_owner, + __entry->fault_page) +); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_setup_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_setup_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_collect_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_collect_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_pages_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_pages_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_finalize_start, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +DEFINE_EVENT (migrate_vma_range_class, migrate_vma_finalize_done, + TP_PROTO(const struct migrate_vma *migrate), +TP_ARGS (migrate)); + +/* +* Page table walk callbacks act on a sub-range of the migration and can be +* reached from several call sites, so they report the range they were handed +* and why, in addition to the migrate-wide state. +*/ +DECLARE_EVENT_CLASS (migrate_vma_walk_class, + + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), + +TP_ARGS (migrate, start, end, reason), + +TP_STRUCT__entry ( + __field(void *, vma) + __field(void *, mm) + __field(void *, src) + __field(void *, dst) + __field(void *, pgmap_owner) + __field(void *, fault_page) + __field(unsigned long, walk_start) + __field(unsigned long, walk_end) + __field(unsigned long, start) + __field(unsigned long, end) + __field(unsigned long, npages) + __field(unsigned long, cpages) + __field(unsigned long, flags) + __field(unsigned int, reason) + ), + +TP_fast_assign ( + __entry->vma = migrate->vma; + __entry->mm = migrate->vma ? migrate->vma->vm_mm : NULL; + __entry->src = migrate->src; + __entry->dst = migrate->dst; + __entry->pgmap_owner = migrate->pgmap_owner; + __entry->fault_page = migrate->fault_page; + __entry->walk_start = start; + __entry->walk_end = end; + __entry->start = migrate->start; + __entry->end = migrate->end; + __entry->npages = migrate->npages; + __entry->cpages = migrate->cpages; + __entry->flags = migrate->flags; + __entry->reason = reason; + ), + +TP_printk ("mm=%p vma=%p walk=%#lx-%#lx bytes=%lu reason=%s range=%#lx-%#lx npages=%lu cpages=%lu flags=%s src=%p dst=%p owner=%p fault_page=%p" , + __entry->mm, __entry->vma, + __entry->walk_start, __entry->walk_end, + __entry->walk_end - __entry->walk_start, +show_walk_reason (__entry->reason), + __entry->start, __entry->end, __entry->npages, + __entry->cpages, +show_migrate_vma_flags (__entry->flags), + __entry->src, __entry->dst, __entry->pgmap_owner, + __entry->fault_page) +); + +DEFINE_EVENT (migrate_vma_walk_class, migrate_vma_collect_skip, + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), +TP_ARGS (migrate, start, end, reason)); + +DEFINE_EVENT (migrate_vma_walk_class, migrate_vma_collect_hole, + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), +TP_ARGS (migrate, start, end, reason)); + +DEFINE_EVENT (migrate_vma_walk_class, migrate_vma_collect_huge_pmd, + TP_PROTO(const struct migrate_vma *migrate, unsigned long start, + unsigned long end, enum migrate_vma_walk_reason reason), +TP_ARGS (migrate, start, end, reason)); + +DECLARE_EVENT_CLASS (migrate_device_batch_class, + + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), + +TP_ARGS (src, dst, npages), + +TP_STRUCT__entry ( + __field(void *, src) + __field(void *, dst) + __field(unsigned long, src_head) + __field(unsigned long, dst_head) + __field(unsigned long, npages) + ), + +TP_fast_assign ( + __entry->src = (void *)src; + __entry->dst = (void *)dst; + __entry->src_head = npages && src ? src[0] : 0; + __entry->dst_head = npages && dst ? dst[0] : 0; + __entry->npages = npages; + ), + +TP_printk ("src=%p dst=%p npages=%lu src_head=%#lx dst_head=%#lx" , + __entry->src, __entry->dst, __entry->npages, + __entry->src_head, __entry->dst_head) +); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_unmap_done, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_pages_start, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_pages_done, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_finalize_start, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DEFINE_EVENT (migrate_device_batch_class, migrate_device_finalize_done, + TP_PROTO(const unsigned long *src, const unsigned long *dst, + unsigned long npages), +TP_ARGS (src, dst, npages)); + +DECLARE_EVENT_CLASS (migrate_device_folio_class, + + TP_PROTO(unsigned long index, unsigned long nr_pages, + unsigned long src, unsigned long dst), + +TP_ARGS (index, nr_pages, src, dst), + +TP_STRUCT__entry ( + __field(unsigned long, index) + __field(unsigned long, nr_pages) + __field(unsigned long, src) + __field(unsigned long, dst) + __field(unsigned long, src_pfn) + __field(unsigned long, dst_pfn) + ), + +TP_fast_assign ( + __entry->index = index; + __entry->nr_pages = nr_pages; + __entry->src = src; + __entry->dst = dst; + __entry->src_pfn = src >> MIGRATE_PFN_SHIFT; + __entry->dst_pfn = dst >> MIGRATE_PFN_SHIFT; + ), + +TP_printk ("index=%lu nr_pages=%lu src=%#lx src_pfn=%#lx src_flags=%s dst=%#lx dst_pfn=%#lx dst_flags=%s" , + __entry->index, __entry->nr_pages, __entry->src, + __entry->src_pfn, +__print_flags (__entry->src & ((1UL <<MIGRATE_PFN_SHIFT ) -1 ), +"|" , + { MIGRATE_PFN_VALID, "VALID" }, + { MIGRATE_PFN_MIGRATE, "MIGRATE" }, + { MIGRATE_PFN_WRITE, "WRITE" }, + { MIGRATE_PFN_COMPOUND, "COMPOUND" }), + __entry->dst, __entry->dst_pfn, +__print_flags (__entry->dst & ((1UL <<MIGRATE_PFN_SHIFT ) -1 ), +"|" , + { MIGRATE_PFN_VALID, "VALID" }, + { MIGRATE_PFN_MIGRATE, "MIGRATE" }, + { MIGRATE_PFN_WRITE, "WRITE" }, + { MIGRATE_PFN_COMPOUND, "COMPOUND" })) +); + +DEFINE_EVENT (migrate_device_folio_class, migrate_device_folio_finalize, + TP_PROTO(unsigned long index, unsigned long nr_pages, + unsigned long src, unsigned long dst), +TP_ARGS (index, nr_pages, src, dst)); + +#endif /* _TRACE_MIGRATE_DEVICE_H */ + +#include <trace/define_trace.h> mm/migrate_device.c +67
-12@@ -16,11 +16,16 @@#include <linux/leafops.h> #include <linux/pgalloc.h> #include <asm/tlbflush.h> + +#define CREATE_TRACE_POINTS +#include <trace/events/migrate_device.h> +#include "internal.h" static int migrate_vma_collect_skip (unsigned long start,unsigned long end, -struct mm_walk *walk) +struct mm_walk *walk, +enum migrate_vma_walk_reason reason) {struct migrate_vma *migrate = walk->private;unsigned long addr;@@ -30,6 +35,7 @@ static int migrate_vma_collect_skip(unsigned long start, migrate->src[migrate->npages++] =0 ; } +trace_migrate_vma_collect_skip (migrate, start, end, reason);return 0 ; }@@ -43,7 +49,8 @@ static int migrate_vma_collect_hole(unsigned long start,/* Only allow populating anonymous memory. */ if (!vma_is_anonymous (walk->vma)) -return migrate_vma_collect_skip (start, end, walk); +return migrate_vma_collect_skip (start, end, walk, +MIGRATE_VMA_WALK_SKIP_NOT_ANON ); if (thp_migration_supported() && (migrate->flags & MIGRATE_VMA_SELECT_COMPOUND) &&@@ -55,11 +62,15 @@ static int migrate_vma_collect_hole(unsigned long start, migrate->npages++; migrate->cpages++; +trace_migrate_vma_collect_hole (migrate, start, end, +MIGRATE_VMA_WALK_HOLE_COMPOUND ); +/* * Collect the remaining entries as holes, in case we * need to split later */ -return migrate_vma_collect_skip (start +PAGE_SIZE , end, walk); +return migrate_vma_collect_skip (start +PAGE_SIZE , end, walk, +MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL ); }for (addr = start; addr < end; addr +=PAGE_SIZE ) {@@ -69,6 +80,8 @@ static int migrate_vma_collect_hole(unsigned long start, migrate->cpages++; } +trace_migrate_vma_collect_hole (migrate, start, end, +MIGRATE_VMA_WALK_HOLE );return 0 ; }@@ -154,7 +167,8 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,if (pmd_trans_huge (*pmdp)) {if (!(migrate->flags &MIGRATE_VMA_SELECT_SYSTEM )) {spin_unlock (ptl); -return migrate_vma_collect_skip (start, end, walk); +return migrate_vma_collect_skip (start, end, walk, +MIGRATE_VMA_WALK_SKIP_NOT_SELECTED ); } folio =pmd_folio (*pmdp);@@ -170,13 +184,15 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,if (!softleaf_is_device_private (entry) || !(migrate->flags &MIGRATE_VMA_SELECT_DEVICE_PRIVATE )) {spin_unlock (ptl); -return migrate_vma_collect_skip (start, end, walk); +return migrate_vma_collect_skip (start, end, walk, +MIGRATE_VMA_WALK_SKIP_NOT_SELECTED ); } folio =softleaf_to_folio (entry);if (folio->pgmap->owner != migrate->pgmap_owner) {spin_unlock (ptl); -return migrate_vma_collect_skip (start, end, walk); +return migrate_vma_collect_skip (start, end, walk, +MIGRATE_VMA_WALK_SKIP_OWNER_MISMATCH ); }if (softleaf_is_device_private_write (entry))@@ -190,7 +206,8 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,if (folio != fault_folio &&unlikely (!folio_trylock (folio))) {spin_unlock (ptl);folio_put (folio); -return migrate_vma_collect_skip (start, end, walk); +return migrate_vma_collect_skip (start, end, walk, +MIGRATE_VMA_WALK_SKIP_LOCK_CONTENDED ); } if (thp_migration_supported() &&@@ -220,8 +237,12 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, migrate->dst[migrate->npages] =0 ; goto fallback; } -migrate_vma_collect_skip (start +PAGE_SIZE , end, walk); +migrate_vma_collect_skip (start +PAGE_SIZE , end, walk, +MIGRATE_VMA_WALK_SKIP_COMPOUND_TAIL );spin_unlock (ptl); + +trace_migrate_vma_collect_huge_pmd (migrate, start, end, +MIGRATE_VMA_WALK_HUGE_PMD );return 0 ; }@@ -234,7 +255,8 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start,folio_unlock (folio);folio_put (folio);if (ret) -return migrate_vma_collect_skip (start, end, walk); +return migrate_vma_collect_skip (start, end, walk, +MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED );if (pmd_none (pmdp_get_lockless (pmdp)))return migrate_vma_collect_hole (start, end,-1 , walk);@@ -323,7 +345,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,if (unmapped)flush_tlb_range (walk->vma, start, end); -return migrate_vma_collect_skip (addr, end, walk); +return migrate_vma_collect_skip (addr, + end, walk, +MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED ); } goto again;@@ -369,7 +393,9 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,if (unmapped)flush_tlb_range (walk->vma, start, end); -return migrate_vma_collect_skip (addr, end, walk); +return migrate_vma_collect_skip (addr, + end, walk, +MIGRATE_VMA_WALK_SKIP_SPLIT_FAILED ); } goto again;@@ -514,6 +540,8 @@ static void migrate_vma_collect(struct migrate_vma *migrate) {struct mmu_notifier_range range; +trace_migrate_vma_collect_start (migrate); +/* * Note that the pgmap_owner is passed to the mmu notifier callback so * that the registered device driver can skip invalidating device @@ -529,6 +557,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate)mmu_notifier_invalidate_range_end (&range); migrate->end = migrate->start + (migrate->npages <<PAGE_SHIFT ); +trace_migrate_vma_collect_done (migrate); }/* @@ -653,6 +682,7 @@ static unsigned long migrate_device_unmap(unsigned long *src_pfns, restore--; } +trace_migrate_device_unmap_done (src_pfns,NULL , npages);return unmapped; }@@ -763,11 +793,13 @@ int migrate_vma_setup(struct migrate_vma *args)memset (args->src,0 ,sizeof (*args->src) * nr_pages); args->cpages =0 ; args->npages =0 ; +trace_migrate_vma_setup_start (args);migrate_vma_collect (args);if (args->cpages)migrate_vma_unmap (args); +trace_migrate_vma_setup_done (args);/* * At this point pages are locked and unmapped, and thus they have @@ -1115,7 +1147,9 @@ static void __migrate_device_pages(unsigned long *src_pfns,bool notified = false;unsigned long addr; -for (i =0 ; i < npages; ) { +trace_migrate_device_pages_start (src_pfns, dst_pfns, npages); + +for (i =0 ; i < npages;) {struct page *newpage =migrate_pfn_to_page (dst_pfns[i]);struct page *page =migrate_pfn_to_page (src_pfns[i]);struct address_space *mapping;@@ -1257,6 +1291,8 @@ static void __migrate_device_pages(unsigned long *src_pfns,if (notified)mmu_notifier_invalidate_range_end (&range); + +trace_migrate_device_pages_done (src_pfns, dst_pfns, npages); }/** @@ -1285,7 +1321,9 @@ EXPORT_SYMBOL(migrate_device_pages);*/ void migrate_vma_pages (struct migrate_vma *migrate) { +trace_migrate_vma_pages_start (migrate);__migrate_device_pages (migrate->src, migrate->dst, migrate->npages, migrate); +trace_migrate_vma_pages_done (migrate); }EXPORT_SYMBOL (migrate_vma_pages);@@ -1298,11 +1336,24 @@ static void __migrate_device_finalize(unsigned long *src_pfns, page_folio(fault_page) : NULL;unsigned long i; +trace_migrate_device_finalize_start (src_pfns, dst_pfns, npages); +for (i =0 ; i < npages; i++) {struct folio *dst =NULL , *src =NULL ;struct page *newpage =migrate_pfn_to_page (dst_pfns[i]);struct page *page =migrate_pfn_to_page (src_pfns[i]); +if (trace_migrate_device_folio_finalize_enabled () && + (page || newpage)) { +unsigned long nr_pages = page ? +folio_nr_pages (page_folio (page)) : +folio_nr_pages (page_folio (newpage)); + +trace_migrate_device_folio_finalize (i, nr_pages, + src_pfns[i], + dst_pfns[i]); + } +if (newpage) dst =page_folio (newpage);@@ -1339,6 +1390,8 @@ static void __migrate_device_finalize(unsigned long *src_pfns,folio_put (dst); } } + +trace_migrate_device_finalize_done (src_pfns, dst_pfns, npages); }/* @@ -1371,8 +1424,10 @@ EXPORT_SYMBOL(migrate_device_finalize);*/ void migrate_vma_finalize (struct migrate_vma *migrate) { +trace_migrate_vma_finalize_start (migrate);__migrate_device_finalize (migrate->src, migrate->dst, migrate->npages, migrate->fault_page); +trace_migrate_vma_finalize_done (migrate); }EXPORT_SYMBOL (migrate_vma_finalize);Signature · 1 line
2.55.0
Quoted 20 lines · Matthew Brost
Quoted 8 lines · Balbir Singh
Device-private migrations currently require ad-hoc instrumentation to understand selection, unmap, copy, and finalization behavior.
Add reusable ftrace event classes for
migrate_vmaranges, device batches, and individual folios. Emit start, done traces with ranges, flags, page counts, success/failure summaries, PFNs, and migrate flags.Quoted 5 lines
Enabling all traces can make the migration traces verbose, when used they can also help debug issues with device migration.
Signed-off-by Balbir Singh <[email protected]>Suggested-by Steven Rostedt <[email protected]>This seems like a pretty good adding tracepoints - anytime I've had to debug issues in
migrate_device.cit has basically start adding printks to figure out what is going on. Hopefully witth the right tracepoints, this would now be turn on ftrace.
Quoted 4 lines
So +1 on the idea.
In the middle of a few other things, but will circle back with an in depth look at individual tracepoints added here when a bit more time.
Thanks, I am hoping to add more tests and use these for debugging corner cases and hopefully get easier debug for issues reported.
Balbir