memory: Add tracepoints for log_sync
cover memory: Add tracepoints for log_sync
[this is a resend to hopefully drop '\r' in cover letter caused by git-publish
that I should have just fixed; nothing changed inside]
It can help to identify which step is slow for migration dirty sync process. We have migration_bitmap_sync_* trace events but it's still a bit coarse.
This should help us to cut migration_bitmap_sync() into finer grained small steps when measurement is needed.
Please review, thanks.
Peter Xu (2):
memory: Name all the memory listeners
memory: Add tracepoint for dirty sync
accel/hvf/hvf-accel-ops.c | 1 + accel/kvm/kvm-all.c | 7 +++++-- hw/i386/xen/xen-hvm.c | 2 ++ hw/intc/openpic_kvm.c | 1 + hw/remote/proxy-memory-listener.c | 1 + hw/vfio/common.c | 1 + hw/vfio/spapr.c | 1 + hw/virtio/vhost-vdpa.c | 1 + hw/virtio/vhost.c | 2 ++ hw/virtio/virtio.c | 1 + hw/xen/xen_pt.c | 2 ++ include/exec/memory.h | 8 ++++++++ include/sysemu/kvm_int.h | 2 +- softmmu/memory.c | 2 ++ softmmu/physmem.c | 1 + softmmu/trace-events | 1 + target/arm/kvm.c | 1 + target/i386/hax/hax-mem.c | 1 + target/i386/kvm/kvm.c | 2 +- target/i386/nvmm/nvmm-all.c | 1 + target/i386/whpx/whpx-all.c | 1 + 21 files changed, 36 insertions(+), 4 deletions(-)
Signature · 1 line
2.31.1
PATCH RESEND 1/2 memory: Name all the memory listeners
Provide a name field for all the memory listeners. It can be used to identify which memory listener is which.
--- accel/hvf/hvf-accel-ops.c | 1 + accel/kvm/kvm-all.c | 7 +++++-- hw/i386/xen/xen-hvm.c | 2 ++ hw/intc/openpic_kvm.c | 1 + hw/remote/proxy-memory-listener.c | 1 + hw/vfio/common.c | 1 + hw/vfio/spapr.c | 1 + hw/virtio/vhost-vdpa.c | 1 + hw/virtio/vhost.c | 2 ++ hw/virtio/virtio.c | 1 + hw/xen/xen_pt.c | 2 ++ include/exec/memory.h | 8 ++++++++ include/sysemu/kvm_int.h | 2 +- softmmu/physmem.c | 1 + target/arm/kvm.c | 1 + target/i386/hax/hax-mem.c | 1 + target/i386/kvm/kvm.c | 2 +- target/i386/nvmm/nvmm-all.c | 1 + target/i386/whpx/whpx-all.c | 1 + 19 files changed, 33 insertions(+), 4 deletions(-)
accel/hvf/hvf-accel-ops.c +1 -0
@@ -291,6 +291,7 @@ static void hvf_region_del(MemoryListener *listener, }static MemoryListener hvf_memory_listener = { + .name ="hvf" , .priority =10 , .region_add = hvf_region_add, .region_del = hvf_region_del,
accel/kvm/kvm-all.c +5 -2
@@ -1129,6 +1129,7 @@ static void kvm_coalesce_pio_del(MemoryListener *listener, }static MemoryListener kvm_coalesced_pio_listener = { + .name ="kvm-coalesced-pio" , .coalesced_io_add = kvm_coalesce_pio_add, .coalesced_io_del = kvm_coalesce_pio_del, };
@@ -1633,7 +1634,7 @@ static void kvm_io_ioeventfd_del(MemoryListener *listener, }void kvm_memory_listener_register (KVMState *s,KVMMemoryListener *kml, -AddressSpace *as,int as_id) +AddressSpace *as,int as_id,const char *name) {int i;
@@ -1649,6 +1650,7 @@ void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, kml->listener.log_start = kvm_log_start; kml->listener.log_stop = kvm_log_stop; kml->listener.priority =10 ; + kml->listener.name = name;if (s->kvm_dirty_ring_size) { kml->listener.log_sync_global = kvm_log_sync_global;
@@ -1669,6 +1671,7 @@ void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml, }static MemoryListener kvm_io_listener = { + .name ="kvm-io" , .eventfd_add = kvm_io_ioeventfd_add, .eventfd_del = kvm_io_ioeventfd_del, .priority =10 ,
@@ -2579,7 +2582,7 @@ static int kvm_init(MachineState *ms) s->memory_listener.listener.coalesced_io_del = kvm_uncoalesce_mmio_region;kvm_memory_listener_register (s, &s->memory_listener, - &address_space_memory,0 ); + &address_space_memory,0 ,"kvm-memory" );if (kvm_eventfds_allowed) {memory_listener_register (&kvm_io_listener, &address_space_io);
hw/i386/xen/xen-hvm.c +2 -0
@@ -721,6 +721,7 @@ static void xen_log_global_stop(MemoryListener *listener) }static MemoryListener xen_memory_listener = { + .name ="xen-memory" , .region_add = xen_region_add, .region_del = xen_region_del, .log_start = xen_log_start,
@@ -732,6 +733,7 @@ static MemoryListener xen_memory_listener = { };static MemoryListener xen_io_listener = { + .name ="xen-io" , .region_add = xen_io_add, .region_del = xen_io_del, .priority =10 ,
hw/intc/openpic_kvm.c +1 -0
@@ -234,6 +234,7 @@ static void kvm_openpic_realize(DeviceState *dev, Error **errp) opp->mem_listener.region_add = kvm_openpic_region_add; opp->mem_listener.region_del = kvm_openpic_region_del; + opp->mem_listener.name ="openpic-kvm" ;memory_listener_register (&opp->mem_listener, &address_space_memory);/* indicate pic capabilities */
hw/remote/proxy-memory-listener.c +1 -0
@@ -219,6 +219,7 @@ void proxy_memory_listener_configure(ProxyMemoryListener *proxy_listener, proxy_listener->listener.region_add = proxy_memory_listener_region_addnop; proxy_listener->listener.region_nop = proxy_memory_listener_region_addnop; proxy_listener->listener.priority =10 ; + proxy_listener->listener.name ="proxy" ;memory_listener_register (&proxy_listener->listener, &address_space_memory);
hw/vfio/common.c +1 -0
@@ -1434,6 +1434,7 @@ static void vfio_listener_log_sync(MemoryListener *listener, }static const MemoryListener vfio_memory_listener = { + .name ="vfio" , .region_add = vfio_listener_region_add, .region_del = vfio_listener_region_del, .log_global_start = vfio_listener_log_global_start,
hw/vfio/spapr.c +1 -0
@@ -136,6 +136,7 @@ static void vfio_prereg_listener_region_del(MemoryListener *listener, }const MemoryListener vfio_prereg_listener = { + .name ="vfio-pre-reg" , .region_add = vfio_prereg_listener_region_add, .region_del = vfio_prereg_listener_region_del, };
hw/virtio/vhost-vdpa.c +1 -0
@@ -234,6 +234,7 @@ static void vhost_vdpa_listener_region_del(MemoryListener *listener,* depends on the addnop(). */ static const MemoryListener vhost_vdpa_memory_listener = { + .name ="vhost-vdpa" , .begin = vhost_vdpa_listener_begin, .commit = vhost_vdpa_listener_commit, .region_add = vhost_vdpa_listener_region_add,
hw/virtio/vhost.c +2 -0
@@ -1341,6 +1341,7 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, hdev->features = features; hdev->memory_listener = (MemoryListener ) { + .name ="vhost" , .begin = vhost_begin, .commit = vhost_commit, .region_add = vhost_region_addnop,
@@ -1356,6 +1357,7 @@ int vhost_dev_init(struct vhost_dev *hdev, void *opaque, }; hdev->iommu_listener = (MemoryListener ) { + .name ="vhost-iommu" , .region_add = vhost_iommu_region_add, .region_del = vhost_iommu_region_del, };
hw/virtio/virtio.c +1 -0
@@ -3671,6 +3671,7 @@ static void virtio_device_realize(DeviceState *dev, Error **errp) } vdev->listener.commit = virtio_memory_listener_commit; + vdev->listener.name ="virtio" ;memory_listener_register (&vdev->listener, vdev->dma_as); }
hw/xen/xen_pt.c +2 -0
@@ -689,12 +689,14 @@ static void xen_pt_io_region_del(MemoryListener *l, MemoryRegionSection *sec) }static const MemoryListener xen_pt_memory_listener = { + .name ="xen-pt-mem" , .region_add = xen_pt_region_add, .region_del = xen_pt_region_del, .priority =10 , };static const MemoryListener xen_pt_io_listener = { + .name ="xen-pt-io" , .region_add = xen_pt_io_region_add, .region_del = xen_pt_io_region_del, .priority =10 ,
include/exec/memory.h +8 -0
@@ -979,6 +979,14 @@ struct MemoryListener {*/ unsigned priority; +/** +* @name: +* +* Name of the listener. It can be used in contexts where we'd like to +* identify one memory listener with the rest. +*/ +const char *name; +/* private: */ AddressSpace *address_space;QTAILQ_ENTRY (MemoryListener) link;
include/sysemu/kvm_int.h +1 -1
@@ -37,7 +37,7 @@ typedef struct KVMMemoryListener { } KVMMemoryListener;void kvm_memory_listener_register (KVMState *s,KVMMemoryListener *kml, -AddressSpace *as,int as_id); +AddressSpace *as,int as_id,const char *name);void kvm_set_max_memslot_size (hwaddr max_slot_size);
softmmu/physmem.c +1 -0
@@ -756,6 +756,7 @@ void cpu_address_space_init(CPUState *cpu, int asidx,if (tcg_enabled ()) { newas->tcg_as_listener.log_global_after_sync = tcg_log_global_after_sync; newas->tcg_as_listener.commit = tcg_commit; + newas->tcg_as_listener.name ="tcg" ;memory_listener_register (&newas->tcg_as_listener, as); } }
target/arm/kvm.c +1 -0
@@ -330,6 +330,7 @@ static void kvm_arm_devlistener_del(MemoryListener *listener, }static MemoryListener devlistener = { + .name ="kvm-arm" , .region_add = kvm_arm_devlistener_add, .region_del = kvm_arm_devlistener_del, };
target/i386/hax/hax-mem.c +1 -0
@@ -285,6 +285,7 @@ static void hax_log_sync(MemoryListener *listener, }static MemoryListener hax_memory_listener = { + .name ="hax" , .begin = hax_transaction_begin, .commit = hax_transaction_commit, .region_add = hax_region_add,
target/i386/kvm/kvm.c +1 -1
@@ -2227,7 +2227,7 @@ static void register_smram_listener(Notifier *n, void *unused)address_space_init (&smram_address_space, &smram_as_root,"KVM-SMRAM" );kvm_memory_listener_register (kvm_state, &smram_listener, - &smram_address_space,1 ); + &smram_address_space,1 ,"kvm-smram" ); }int kvm_arch_init (MachineState *ms,KVMState *s)
target/i386/nvmm/nvmm-all.c +1 -0
@@ -1125,6 +1125,7 @@ nvmm_log_sync(MemoryListener *listener, MemoryRegionSection *section) }static MemoryListener nvmm_memory_listener = { + .name ="nvmm" , .begin = nvmm_transaction_begin, .commit = nvmm_transaction_commit, .region_add = nvmm_region_add,
target/i386/whpx/whpx-all.c +1 -0
@@ -1600,6 +1600,7 @@ static void whpx_log_sync(MemoryListener *listener, }static MemoryListener whpx_memory_listener = { + .name ="whpx" , .begin = whpx_transaction_begin, .commit = whpx_transaction_commit, .region_add = whpx_region_add,
Signature · 1 line
2.31.1
Re: [PATCH RESEND 1/2] memory: Name all the memory listeners
Quoted 26 lines · Peter Xu
Provide a name field for all the memory listeners. It can be used to identify which memory listener is which.
Signed-off-by Peter Xu <[email protected]>--- accel/hvf/hvf-accel-ops.c | 1 + accel/kvm/kvm-all.c | 7 +++++-- hw/i386/xen/xen-hvm.c | 2 ++ hw/intc/openpic_kvm.c | 1 + hw/remote/proxy-memory-listener.c | 1 + hw/vfio/common.c | 1 + hw/vfio/spapr.c | 1 + hw/virtio/vhost-vdpa.c | 1 + hw/virtio/vhost.c | 2 ++ hw/virtio/virtio.c | 1 + hw/xen/xen_pt.c | 2 ++ include/exec/memory.h | 8 ++++++++ include/sysemu/kvm_int.h | 2 +- softmmu/physmem.c | 1 + target/arm/kvm.c | 1 + target/i386/hax/hax-mem.c | 1 + target/i386/kvm/kvm.c | 2 +- target/i386/nvmm/nvmm-all.c | 1 + target/i386/whpx/whpx-all.c | 1 + 19 files changed, 33 insertions(+), 4 deletions(-)
[...]
Quoted 10 lines
static const MemoryListener xen_pt_io_listener = {hw/xen/xen_pt.c +1
-0+ .name = "xen-pt-io", .region_add = xen_pt_io_region_add, .region_del = xen_pt_io_region_del, .priority = 10,include/exec/memory.h +0
-0
include/exec/memory.h struct MemoryListener { · line 986
Quoted 8 lines
include/exec/memory.h +5
-0@@ -979,6 +979,14 @@ struct MemoryListener {*/ unsigned priority; +/** +* @name: +* +* Name of the listener. It can be used in contexts where we'd like to +* identify one memory listener with the rest. > + */
Not a native speaker, maybe simply "Name of the listener, primarily useful for debugging. Names can change in the future and are not fixed."
Apart from that LGTM
Signature · 3 lines
Thanks, David / dhildenb
PATCH RESEND 2/2 memory: Add tracepoint for dirty sync
In reply to Peter XuTrace at memory_region_sync_dirty_bitmap() for log_sync() or global_log_sync() on memory regions. One trace line should suffice when it finishes, so as to estimate the time used for each log sync process.
--- softmmu/memory.c | 2 ++ softmmu/trace-events | 1 + 2 files changed, 3 insertions(+)
softmmu/memory.c +2 -0
@@ -2149,6 +2149,7 @@ static void memory_region_sync_dirty_bitmap(MemoryRegion *mr) } }flatview_unref (view); +trace_memory_region_sync_dirty (mr ? mr->name :"(all)" , listener->name,0 ); }else if (listener->log_sync_global) {/* * No matter whether MR is specified, what we can do here
@@ -2156,6 +2157,7 @@ static void memory_region_sync_dirty_bitmap(MemoryRegion *mr)* sync in a finer granularity. */ listener->log_sync_global (listener); +trace_memory_region_sync_dirty (mr ? mr->name :"(all)" , listener->name,1 ); } } }
softmmu/trace-events +1 -0
@@ -15,6 +15,7 @@ memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t va memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset 0x%"PRIx64" value 0x%"PRIx64" size %u" memory_region_ram_device_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u" memory_region_ram_device_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr 0x%"PRIx64" value 0x%"PRIx64" size %u" +memory_region_sync_dirty(const char *mr, const char *listener, int global) "mr '%s' listener '%s' synced (global=%d)" flatview_new(void *view, void *root) "%p (root %p)" flatview_destroy(void *view, void *root) "%p (root %p)" flatview_destroy_rcu(void *view, void *root) "%p (root %p)"
Signature · 1 line
2.31.1
Re: [PATCH RESEND 2/2] memory: Add tracepoint for dirty sync
Quoted 3 lines · Peter Xu
Trace at
memory_region_sync_dirty_bitmap()forlog_sync()orglobal_log_sync()on memory regions. One trace line should suffice when it finishes, so as to estimate the time used for each log sync process.
I wonder if a start/finish would be even nicer. At least it wouldn't really result in significantly more code changes :)
Signature · 3 lines
Thanks, David / dhildenb
Re: [PATCH RESEND 2/2] memory: Add tracepoint for dirty sync
Quoted 7 lines · David Hildenbrand
Quoted 3 lines · Peter Xu
Trace at
memory_region_sync_dirty_bitmap()forlog_sync()orglobal_log_sync()on memory regions. One trace line should suffice when it finishes, so as to estimate the time used for each log sync process.I wonder if a start/finish would be even nicer. At least it wouldn't really result in significantly more code changes :)
Note that the "name"s I added is not only for not using start/end, it's about knowing which memory listener is slow. Start/end won't achieve that if we don't have a name for them. So far I just wanted to identify majorly kvm, vhost and kvm-smram, however it'll always be good when some log_sync is missed when tracing.
I'm also wondering whether kvm-smram needs a whole bitmap as I don't know what RAM would be touched within system manager mode (as I thought it should only touch a very limited range and should be defined somewhere), but that's off-topic.
If we want to make it start/end pair, I can do that too. But the 1st patch will still be wanted.
Thanks,
Signature · 1 line
Peter Xu
Re: [PATCH RESEND 2/2] memory: Add tracepoint for dirty sync
Quoted 15 lines · Peter Xu
Quoted 7 lines · David Hildenbrand
Quoted 3 lines · Peter Xu
Trace at
memory_region_sync_dirty_bitmap()forlog_sync()orglobal_log_sync()on memory regions. One trace line should suffice when it finishes, so as to estimate the time used for each log sync process.I wonder if a start/finish would be even nicer. At least it wouldn't really result in significantly more code changes :)
Note that the "name"s I added is not only for not using start/end, it's about knowing which memory listener is slow. Start/end won't achieve that if we don't have a name for them. So far I just wanted to identify majorly kvm, vhost and kvm-smram, however it'll always be good when some
log_syncis missed when tracing.
Quoted 7 lines
I'm also wondering whether kvm-smram needs a whole bitmap as I don't know what RAM would be touched within system manager mode (as I thought it should only touch a very limited range and should be defined somewhere), but that's off-topic.
If we want to make it start/end pair, I can do that too. But the 1st patch will still be wanted.
Yeah, absolutely, not complaining about the name, it will be valuable to have!
Signature · 3 lines
Thanks, David / dhildenb
Re: [PATCH RESEND 2/2] memory: Add tracepoint for dirty sync
Quoted 6 lines · David Hildenbrand
Quoted 4 lines
If we want to make it start/end pair, I can do that too. But the 1st patch will still be wanted.
Yeah, absolutely, not complaining about the name, it will be valuable to have!
You could have start/end tracepoints for memory_region_sync_dirty_bitmap on top of this one that you are adding, so I queued the patches.
Paolo
Re: [PATCH RESEND 2/2] memory: Add tracepoint for dirty sync
In reply to Peter XuQuoted 5 lines · Peter Xu
I'm also wondering whether kvm-smram needs a whole bitmap as I don't know what RAM would be touched within system manager mode (as I thought it should only touch a very limited range and should be defined somewhere), but that's off-topic.
The kvm-smram dirty bitmap will include all memory touched while the SMM address space is in effect, so not just SMRAM. The two KVM dirty bitmaps end up in just one QEMU dirty bitmap (the one with id DIRTY_MEMORY_MIGRATION) but they are separate at the kernel level.
Paolo