migration: fix migrate_cancel problems of multifd
cover migration: fix migrate_cancel problems of multifd
The patches fix the problems encountered in multifd migration when try to cancel the migration by migrate_cancel.
Ivan Ren (3):
migration: fix migrate_cancel leads live_migration thread endless loop
migration: fix migrate_cancel leads live_migration thread hung forever
migration: fix migrate_cancel multifd migration leads destination hung
forever
migration/ram.c | 62 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 11 deletions(-)
Signature · 1 line
2.17.2 (Apple Git-113)
PATCH 1/3 migration: fix migrate_cancel leads live_migration thread endless loop
When we 'migrate_cancel' a multifd migration, live_migration thread may go into endless loop in multifd_send_pages functions.
Reproduce steps:
(qemu) migrate_set_capability multifd on
(qemu) migrate -d url
(qemu) [wait a while]
(qemu) migrate_cancel
Then may get live_migration 100% cpu usage in following stack:
pthread_mutex_lockqemu_mutex_lock_implmultifd_send_pagesmultifd_queue_pageram_save_multifd_pageram_save_target_pageram_save_host_pageram_find_and_save_blockram_find_and_save_blockram_save_iterateqemu_savevm_state_iteratemigration_iteration_runmigration_threadqemu_thread_startstart_thread
clone
--- migration/ram.c | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-)
migration/ram.c +29 -7
@@ -920,7 +920,7 @@ struct {* false. */ -static void multifd_send_pages (void ) +static int multifd_send_pages (void ) {int i;static int next_channel;
@@ -933,6 +933,11 @@ static void multifd_send_pages(void) p = &multifd_send_state->params[i];qemu_mutex_lock (&p->mutex); +if (p->quit) { +error_report ("%s: channel %d has already quit!" , __func__, i); +qemu_mutex_unlock (&p->mutex); +return -1 ; + }if (!p->pending_job) { p->pending_job++; next_channel = (i +1 ) %migrate_multifd_channels ();
@@ -951,9 +956,11 @@ static void multifd_send_pages(void) ram_counters.transferred += transferred;;qemu_mutex_unlock (&p->mutex);qemu_sem_post (&p->sem); + +return 1 ; } -static void multifd_queue_page (RAMBlock *block,ram_addr_t offset) +static int multifd_queue_page (RAMBlock *block,ram_addr_t offset) {MultiFDPages_t *pages = multifd_send_state->pages;
@@ -968,15 +975,19 @@ static void multifd_queue_page(RAMBlock *block, ram_addr_t offset) pages->used++;if (pages->used < pages->allocated) { -return ; +return 1 ; } } -multifd_send_pages (); +if (multifd_send_pages () <0 ) { +return -1 ; + }if (pages->block != block) { -multifd_queue_page (block, offset); +return multifd_queue_page (block, offset); } + +return 1 ; }static void multifd_send_terminate_threads (Error *err)
@@ -1049,7 +1060,10 @@ static void multifd_send_sync_main(void)return ; }if (multifd_send_state->pages->used) { -multifd_send_pages (); +if (multifd_send_pages () <0 ) { +error_report ("%s: multifd_send_pages fail" , __func__); +return ; + } }for (i =0 ; i <migrate_multifd_channels (); i++) {MultiFDSendParams *p = &multifd_send_state->params[i];
@@ -1058,6 +1072,12 @@ static void multifd_send_sync_main(void)qemu_mutex_lock (&p->mutex); +if (p->quit) { +error_report ("%s: channel %d has already quit" , __func__, i); +qemu_mutex_unlock (&p->mutex); +return ; + } + p->packet_num = multifd_send_state->packet_num++; p->flags |=MULTIFD_FLAG_SYNC ; p->pending_job++;
@@ -2000,7 +2020,9 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)static int ram_save_multifd_page (RAMState *rs,RAMBlock *block,ram_addr_t offset) { -multifd_queue_page (block, offset); +if (multifd_queue_page (block, offset) <0 ) { +return -1 ; + } ram_counters.normal++;return 1 ;
Signature · 1 line
2.17.2 (Apple Git-113)
Re: [Qemu-devel] [PATCH 1/3] migration: fix migrate_cancel leads live_migration thread endless loop
Quoted 29 lines · Ivan Ren
When we '
migrate_cancel' a multifd migration,live_migrationthread may go into endless loop inmultifd_send_pagesfunctions.Reproduce steps:
(qemu)
migrate_set_capabilitymultifd on
(qemu) migrate -d url
(qemu) [wait a while]
(qemu)migrate_cancelThen may get
live_migration100% cpu usage in following stack:
pthread_mutex_lockqemu_mutex_lock_implmultifd_send_pagesmultifd_queue_pageram_save_multifd_pageram_save_target_pageram_save_host_pageram_find_and_save_blockram_find_and_save_blockram_save_iterateqemu_savevm_state_iteratemigration_iteration_runmigration_threadqemu_thread_startstart_thread
clone
Quoted 1 line
Signed-off-by Ivan Ren <[email protected]>
Will sent for rc3.
PATCH 2/3 migration: fix migrate_cancel leads live_migration thread hung forever
In reply to Ivan RenWhen we 'migrate_cancel' a multifd migration, live_migration thread may hung forever at some points, because of multifd_send_thread has already exit for socket error:
1. multifd_send_pages may hung at qemu_sem_wait(&multifd_send_state-> channels_ready)
2. multifd_send_sync_main my hung at qemu_sem_wait(&multifd_send_state-> sem_sync)
--- migration/ram.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-)
migration/ram.c +19 -4
@@ -1097,7 +1097,11 @@ static void *multifd_send_thread(void *opaque) {MultiFDSendParams *p = opaque;Error *local_err =NULL ; -int ret; +int ret =0 ; + +uint32_t used =0 ; +uint64_t packet_num =0 ; +uint32_t flags =0 ;trace_multifd_send_thread_start (p->id);rcu_register_thread ();
@@ -1113,9 +1117,9 @@ static void *multifd_send_thread(void *opaque)qemu_mutex_lock (&p->mutex);if (p->pending_job) { -uint32_t used = p->pages->used; -uint64_t packet_num = p->packet_num; -uint32_t flags = p->flags; + used = p->pages->used; + packet_num = p->packet_num; + flags = p->flags; p->next_packet_size = used *qemu_target_page_size ();multifd_send_fill_packet (p);
@@ -1164,6 +1168,17 @@ out:multifd_send_terminate_threads (local_err); } +/* +* Error happen, I will exit, but I can't just leave, tell +* who pay attention to me. +*/ +if (ret !=0 ) { +if (flags &MULTIFD_FLAG_SYNC ) { +qemu_sem_post (&multifd_send_state->sem_sync); + } +qemu_sem_post (&multifd_send_state->channels_ready); + } +qemu_mutex_lock (&p->mutex); p->running = false;qemu_mutex_unlock (&p->mutex);
Signature · 1 line
2.17.2 (Apple Git-113)
Re: [Qemu-devel] [PATCH 2/3] migration: fix migrate_cancel leads live_migration thread hung forever
Quoted 18 lines · Ivan Ren
When we '
migrate_cancel' a multifd migration,live_migrationthread may hung forever at some points, because ofmultifd_send_threadhas already exit for socket error:
1.multifd_send_pagesmay hung atqemu_sem_wait(&multifd_send_state-> channels_ready)
2.multifd_send_sync_mainmy hung atqemu_sem_wait(&multifd_send_state-> sem_sync)Signed-off-by Ivan Ren <[email protected]>--- migration/ram.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-)migration/ram.c +0
-0
migration/ram.c static void *multifd_send_thread(void *opaque) · line 1100
Quoted 5 lines
migration/ram.c +1
-1@@ -1097,7 +1097,11 @@ static void *multifd_send_thread(void *opaque) {MultiFDSendParams *p = opaque;Error *local_err =NULL ; -int ret; +int ret =0 ;
I agree with the ret change.
migration/ram.c static void *multifd_send_thread(void *opaque) · line 1104
Quoted 4 lines
migration/ram.c +4
-0+ +uint32_t used =0 ; +uint64_t packet_num =0 ; +uint32_t flags =0 ;
This movement is unneeded.
Quoted 19 lines
migration/ram.c +3
-3trace_multifd_send_thread_start (p->id);rcu_register_thread ();@@ -1113,9 +1117,9 @@ static void *multifd_send_thread(void *opaque)qemu_mutex_lock (&p->mutex);if (p->pending_job) { -uint32_t used = p->pages->used; -uint64_t packet_num = p->packet_num; -uint32_t flags = p->flags; + used = p->pages->used; + packet_num = p->packet_num; + flags = p->flags; p->next_packet_size = used *qemu_target_page_size ();multifd_send_fill_packet (p);@@ -1164,6 +1168,17 @@ out:multifd_send_terminate_threads (local_err); }
migration/ram.c out: · line 1180
Quoted 10 lines
migration/ram.c +10
-0+/* +* Error happen, I will exit, but I can't just leave, tell +* who pay attention to me. +*/ +if (ret !=0 ) { +if (flags &MULTIFD_FLAG_SYNC ) { +qemu_sem_post (&multifd_send_state->sem_sync); + } +qemu_sem_post (&multifd_send_state->channels_ready); + }
The real change is just this one. Good catch, thanks.
Later, Juan.
Re: [Qemu-devel] [PATCH 2/3] migration: fix migrate_cancel leads live_migration thread hung forever
In reply to Ivan Renmigration/ram.c out: · line 1177
Quoted 13 lines · Ivan Ren
When we '
migrate_cancel' a multifd migration,live_migrationthread may hung forever at some points, because ofmultifd_send_threadhas already exit for socket error:
1.multifd_send_pagesmay hung atqemu_sem_wait(&multifd_send_state-> channels_ready)
2.multifd_send_sync_mainmy hung atqemu_sem_wait(&multifd_send_state-> sem_sync)Signed-off-by Ivan Ren <[email protected]>---
migration/ram.c +3
-0+if (ret !=0 ) { +if (flags &MULTIFD_FLAG_SYNC ) { +qemu_sem_post (&multifd_send_state->sem_sync);
flags is also needed.
Sorry, Juan.
Quoted 7 lines
migration/ram.c +4
-0+ } +qemu_sem_post (&multifd_send_state->channels_ready); + } +qemu_mutex_lock (&p->mutex); p->running = false;qemu_mutex_unlock (&p->mutex);
PATCH 3/3 migration: fix migrate_cancel multifd migration leads destination hung forever
In reply to Ivan RenWhen migrate_cancel a multifd migration, if run sequence like this:
[source] [destination]
multifd_send_sync_main[finish]
multifd_recv_thread wait &p->sem_sync
shutdown to_dst_file
detect error from_src_file
send RAM_SAVE_FLAG_EOS[fail] [no chance to run multifd_recv_sync_main]
multifd_load_cleanup join multifd receive thread forever
will lead destination qemu hung at following stack:
pthread_joinqemu_thread_joinmultifd_load_cleanupprocess_incoming_migration_cocoroutine_trampoline
--- migration/ram.c | 5 +++++ 1 file changed, 5 insertions(+)
migration/ram.c +5 -0
@@ -1291,6 +1291,11 @@ int multifd_load_cleanup(Error **errp)MultiFDRecvParams *p = &multifd_recv_state->params[i];if (p->running) { +/* +* multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code, +* however try to wakeup it without harm in cleanup phase. +*/ +qemu_sem_post (&p->sem_sync);qemu_thread_join (&p->thread); }object_unref (OBJECT (p->c));
Signature · 1 line
2.17.2 (Apple Git-113)
Re: [Qemu-devel] [PATCH 3/3] migration: fix migrate_cancel multifd migration leads destination hung forever
ping for review
problem still exist in qemu-4.1.0-rc2
Threads: 24 total, 0 running, 24 sleeping, 0 stopped, 0 zombie
%Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si,
0.0 st
KiB Mem : 39434172+total, 36798950+free, 2948836 used, 23403388 buff/cache
KiB Swap: 0 total, 0 free, 0 used. 38926476+avail Mem
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
286108 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:01.19
qemu-system-x86
286109 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00
qemu-system-x86
286113 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 IOmon_iothread
286114 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
0/KVM
286115 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
1/KVM
286116 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
2/KVM
286117 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
3/KVM
286118 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
4/KVM
286119 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
5/KVM
286120 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
6/KVM
286121 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
7/KVM
286122 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
8/KVM
286123 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
9/KVM
286124 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
10/KVM
286125 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
11/KVM
286126 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
12/KVM
286127 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
13/KVM
286128 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
14/KVM
286129 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00 CPU
15/KVM
286131 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.00vnc_worker
286132 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.01multifdrecv_0
286133 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.01multifdrecv_1
286134 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.01multifdrecv_2
286136 root 20 0 0.127t 110596 12684 S 0.0 0.0 0:00.01multifdrecv_3
Thread 2 (Thread 0x7f9d075fe700 (LWP 286136)):
#0 0x00007fbd67123a0b in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007fbd67123a9f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007fbd67123b3b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00005582236e2514 in qemu_sem_wait (sem=sem@entry=0x558226364dc8) at util/qemu-thread-posix.c:319
#4 0x00005582232efb67 in multifd_recv_thread (opaque=opaque@entry=0x558226364d30)
at /qemu-4.1.0-rc2/migration/ram.c:1356
#5 0x00005582236e1b26 in qemu_thread_start (args=<optimized out>) at util/qemu-thread-posix.c:502
#6 0x00007fbd6711de25 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbd66e4a35d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7fbd6c7a4cc0 (LWP 286108)):
#0 0x00007fbd6711ef57 in pthread_join () from /lib64/libpthread.so.0 #1 0x00005582236e28ff in qemu_thread_join (thread=thread@entry=0x558226364b00) at util/qemu-thread-posix.c:570
#2 0x00005582232f36b4 in multifd_load_cleanup (errp=errp@entry=0x7fbd341fff58) at /qemu-4.1.0-rc2/migration/ram.c:1259
#3 0x00005582235765a9 in process_incoming_migration_co (opaque=<optimized out>) at migration/migration.c:510
#4 0x00005582236f4c0a in coroutine_trampoline (i0=<optimized out>, i1=<optimized out>) at util/coroutine-ucontext.c:115
#5 0x00007fbd66d98d40 in ?? () from /lib64/libc.so.6 #6 0x00007ffec0bf24d0 in ?? () #7 0x0000000000000000 in ?? ()
Quoted 47 lines · Ivan Ren
When
migrate_cancela multifd migration, if run sequence like this:[source] [destination]
multifd_send_sync_main[finish]
multifd_recv_thread wait &p->sem_syncshutdown
to_dst_file
detect errorfrom_src_file
send RAM_SAVE_FLAG_EOS[fail] [no chance to runmultifd_recv_sync_main]
multifd_load_cleanupjoin multifd receive thread foreverwill lead destination qemu hung at following stack:
pthread_joinqemu_thread_joinmultifd_load_cleanupprocess_incoming_migration_cocoroutine_trampolineSigned-off-by Ivan Ren <[email protected]>--- migration/ram.c | 5 +++++ 1 file changed, 5 insertions(+)migration/ram.c +5
-0@@ -1291,6 +1291,11 @@ int multifd_load_cleanup(Error **errp)MultiFDRecvParams *p = &multifd_recv_state->params[i];if (p->running) { +/* +* multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code, +* however try to wakeup it without harm in cleanup phase. +*/ +qemu_sem_post (&p->sem_sync);qemu_thread_join (&p->thread); }object_unref(OBJECT(p->c));--
2.17.2 (Apple Git-113)
Re: [Qemu-devel] [PATCH 3/3] migration: fix migrate_cancel multifd migration leads destination hung forever
In reply to Ivan RenQuoted 14 lines · Ivan Ren
When
migrate_cancela multifd migration, if run sequence like this:[source] [destination]
multifd_send_sync_main[finish]
multifd_recv_thread wait &p->sem_syncshutdown
to_dst_file
detect errorfrom_src_file
send RAM_SAVE_FLAG_EOS[fail] [no chance to runmultifd_recv_sync_main]
multifd_load_cleanupjoin multifd receive thread foreverwill lead destination qemu hung at following stack:
Quoted 7 lines
pthread_joinqemu_thread_joinmultifd_load_cleanupprocess_incoming_migration_cocoroutine_trampolineSigned-off-by Ivan Ren <[email protected]>
I think this one is not enough. We need to set some error code, or disable the running bit at that point.
Quoted 10 lines
--- migration/ram.c | 5 +++++ 1 file changed, 5 insertions(+)migration/ram.c +0
-0@@ -1291,6 +1291,11 @@ int multifd_load_cleanup(Error **errp)MultiFDRecvParams *p = &multifd_recv_state->params[i];
migration/ram.c int multifd_load_cleanup(Error **errp) · line 1301
Quoted 10 lines
migration/ram.c +5
-0if (p->running) { +/* +* multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code, +* however try to wakeup it without harm in cleanup phase. +*/ +qemu_sem_post (&p->sem_sync);qemu_thread_join (&p->thread); }object_unref (OBJECT (p->c));
Let's see where we wait for p->sem_sync:
static void *multifd_recv_thread(void *opaque)
{
....
while (true) {
uint32_t used;
uint32_t flags;
ret = qio_channel_read_all_eof(p->c, (void *)p->packet,
p->packet_len, &local_err);
.....
if (flags & MULTIFD_FLAG_SYNC) {
qemu_sem_post(&multifd_recv_state->sem_sync);
qemu_sem_wait(&p->sem_sync);
}
}
if (local_err) {
multifd_recv_terminate_threads(local_err);
}
qemu_mutex_lock(&p->mutex);
p->running = false;
qemu_mutex_unlock(&p->mutex);
rcu_unregister_thread();
trace_multifd_recv_thread_end(p->id, p->num_packets, p->num_pages);
return NULL;
}If we just post it there, we get out of the wait (that bit is ok), but then we go back to the beggining of the bucle, we (probably) got one error on the qui_channel_read_all_eof(), and we go back to multifd_recv_terminate_threads(), or wait there.
I think that it is better to *also* set an p->quit variable there, and not even try to receive anything for that channel?
I will send a patch later.
Good catch.
Later, Juan.
Re: [Qemu-devel] [PATCH 3/3] migration: fix migrate_cancel multifd migration leads destination hung forever
Quoted 9 lines
If we just post it there, we get out of the wait (that bit is ok), but then we go back to the beggining of the bucle, we (probably) got one error on the
qui_channel_read_all_eof(), and we go back tomultifd_recv_terminate_threads(), or wait there.I think that it is better to *also* set an
p->quitvariable there, and not even try to receive anything for that channel?I will send a patch later.
Yes, agree.
Thanks for review.
Quoted 95 lines · Juan Quintela
Quoted 9 lines · Ivan Ren
When
migrate_cancela multifd migration, if run sequence like this:[source] [destination]
multifd_send_sync_main[finish]
multifd_recv_thread wait &p->sem_syncshutdown
to_dst_file
detect errorfrom_src_file
send RAM_SAVE_FLAG_EOS[fail] [no chance to run
multifd_recv_sync_main]Quoted 5 lines
multifd_load_cleanupjoin multifd receive thread foreverwill lead destination qemu hung at following stack:
Quoted 7 lines
pthread_joinqemu_thread_joinmultifd_load_cleanupprocess_incoming_migration_cocoroutine_trampolineSigned-off-by Ivan Ren <[email protected]>I think this one is not enough. We need to set some error code, or disable the running bit at that point.
Quoted 9 lines
--- migration/ram.c | 5 +++++ 1 file changed, 5 insertions(+)migration/ram.c +0
-0migration/ram.c int multifd_load_cleanup(Error **errp) · line 1295
Quoted 5 lines
migration/ram.c +2
-0@@ -1291,6 +1291,11 @@ int multifd_load_cleanup(Error **errp)MultiFDRecvParams *p = &multifd_recv_state->params[i];if (p->running) { +/* +* multifd_recv_thread may hung at MULTIFD_FLAG_SYNC handle code,
migration/ram.c int multifd_load_cleanup(Error **errp) · line 1301
Quoted 6 lines
migration/ram.c +3
-0+* however try to wakeup it without harm in cleanup phase. +*/ +qemu_sem_post (&p->sem_sync);qemu_thread_join (&p->thread); }object_unref (OBJECT (p->c));Let's see where we wait for
p->sem_sync:static void *multifd_recv_thread(void *opaque)
{ .... while (true) { uint32_t used; uint32_t flags; ret = qio_channel_read_all_eof(p->c, (void *)p->packet, p->packet_len, &local_err); ..... if (flags & MULTIFD_FLAG_SYNC) { qemu_sem_post(&multifd_recv_state->sem_sync); qemu_sem_wait(&p->sem_sync); } } if (local_err) { multifd_recv_terminate_threads(local_err); } qemu_mutex_lock(&p->mutex); p->running = false; qemu_mutex_unlock(&p->mutex); rcu_unregister_thread(); trace_multifd_recv_thread_end(p->id, p->num_packets, p->num_pages); return NULL; }If we just post it there, we get out of the wait (that bit is ok), but then we go back to the beggining of the bucle, we (probably) got one error on the
qui_channel_read_all_eof(), and we go back tomultifd_recv_terminate_threads(), or wait there.I think that it is better to *also* set an
p->quitvariable there, and not even try to receive anything for that channel?I will send a patch later.
Good catch.
Later, Juan.
The problem still exists in mainline, Ping for review
Quoted 16 lines · Ivan Ren
The patches fix the problems encountered in multifd migration when try to cancel the migration by
migrate_cancel.Ivan Ren (3):
migration: fixmigrate_cancelleadslive_migrationthread endless loop
migration: fixmigrate_cancelleadslive_migrationthread hung forever
migration: fixmigrate_cancelmultifd migration leads destination hung
forevermigration/ram.c | 62 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 11 deletions(-)--
2.17.2 (Apple Git-113)