vhost: make SET_VRING_ADDR, SET_FEATURES send replies
PATCH v2 1/1 vhost: make SET_VRING_ADDR, SET_FEATURES send replies
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if VHOST_USER_PROTOCOL_F_LOG_SHMFD is negotiated. Qemu sends VHOST_USER_SET_FEATURES to enable buffers logging and VHOST_USER_SET_VRING_ADDR per each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.
The race can appear as follows: on migration setup, qemu enables dirty page logging by sending VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on with VHOST_USER_SET_VRING_ADDR command.
To resolve this issue, this patch makes qemu wait for the commands result explicilty if VHOST_USER_PROTOCOL_F_REPLY_ACK is negotiated and logging is enabled.
---
v1 -> v2:
* send reply only when logging is enabled [mst]
v0 -> v1:
* send reply for SET_VRING_ADDR, SET_FEATURES only [mst]
hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)
hw/virtio/vhost-user.c +34 -3
@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {
@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }
@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,
@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)
Signature · 1 line
2.25.1
PATCH v2 1/1 vhost: make SET_VRING_ADDR, SET_FEATURES send replies
ping!
Quoted 125 lines · Denis Plotnikov
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if
VHOST_USER_PROTOCOL_F_LOG_SHMFDis negotiated. Qemu sendsVHOST_USER_SET_FEATURESto enable buffers logging andVHOST_USER_SET_VRING_ADDRper each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.The race can appear as follows: on migration setup, qemu enables dirty page logging by sending
VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on withVHOST_USER_SET_VRING_ADDRcommand.To resolve this issue, this patch makes qemu wait for the commands result explicilty if
VHOST_USER_PROTOCOL_F_REPLY_ACKis negotiated and logging is enabled.Signed-off-by Denis Plotnikov <[email protected]>---
v1 -> v2:
* send reply only when logging is enabled [mst]v0 -> v1:
* send reply forSET_VRING_ADDR,SET_FEATURESonly [mst]hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)hw/virtio/vhost-user.c +34
-3@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)
PATCH v2 1/1 vhost: make SET_VRING_ADDR, SET_FEATURES send replies
Quoted 129 lines · Denis Plotnikov
ping!
Quoted 125 lines · Denis Plotnikov
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if
VHOST_USER_PROTOCOL_F_LOG_SHMFDis negotiated. Qemu sendsVHOST_USER_SET_FEATURESto enable buffers logging andVHOST_USER_SET_VRING_ADDRper each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.The race can appear as follows: on migration setup, qemu enables dirty page logging by sending
VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on withVHOST_USER_SET_VRING_ADDRcommand.To resolve this issue, this patch makes qemu wait for the commands result explicilty if
VHOST_USER_PROTOCOL_F_REPLY_ACKis negotiated and logging is enabled.Signed-off-by Denis Plotnikov<[email protected]>---
v1 -> v2:
* send reply only when logging is enabled [mst]v0 -> v1:
* send reply forSET_VRING_ADDR,SET_FEATURESonly [mst]hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)hw/virtio/vhost-user.c +34
-3@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)
Cc more ppl.
Quoted 131 lines · Denis Plotnikov
Quoted 129 lines · Denis Plotnikov
ping!
Quoted 125 lines · Denis Plotnikov
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if
VHOST_USER_PROTOCOL_F_LOG_SHMFDis negotiated. Qemu sendsVHOST_USER_SET_FEATURESto enable buffers logging andVHOST_USER_SET_VRING_ADDRper each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.The race can appear as follows: on migration setup, qemu enables dirty page logging by sending
VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on withVHOST_USER_SET_VRING_ADDRcommand.To resolve this issue, this patch makes qemu wait for the commands result explicilty if
VHOST_USER_PROTOCOL_F_REPLY_ACKis negotiated and logging is enabled.Signed-off-by Denis Plotnikov <[email protected]>---
v1 -> v2:
* send reply only when logging is enabled [mst]v0 -> v1:
* send reply forSET_VRING_ADDR,SET_FEATURESonly [mst]hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)hw/virtio/vhost-user.c +34
-3@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)
Quoted 1 line · Philippe Mathieu-Daudé
Cc more ppl.
This needs to go through Michael Tsirkin's tree.
Stefan
Quoted 134 lines · Denis Plotnikov
Quoted 131 lines · Denis Plotnikov
Quoted 129 lines
ping!
Quoted 125 lines · Denis Plotnikov
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if
VHOST_USER_PROTOCOL_F_LOG_SHMFDis negotiated. Qemu sendsVHOST_USER_SET_FEATURESto enable buffers logging andVHOST_USER_SET_VRING_ADDRper each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.The race can appear as follows: on migration setup, qemu enables dirty page logging by sending
VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on withVHOST_USER_SET_VRING_ADDRcommand.To resolve this issue, this patch makes qemu wait for the commands result explicilty if
VHOST_USER_PROTOCOL_F_REPLY_ACKis negotiated and logging is enabled.Signed-off-by Denis Plotnikov <[email protected]>---
v1 -> v2:
* send reply only when logging is enabled [mst]v0 -> v1:
* send reply forSET_VRING_ADDR,SET_FEATURESonly [mst]hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)hw/virtio/vhost-user.c +34
-3@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)
Quoted 65 lines · Denis Plotnikov
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if
VHOST_USER_PROTOCOL_F_LOG_SHMFDis negotiated. Qemu sendsVHOST_USER_SET_FEATURESto enable buffers logging andVHOST_USER_SET_VRING_ADDRper each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.The race can appear as follows: on migration setup, qemu enables dirty page logging by sending
VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on withVHOST_USER_SET_VRING_ADDRcommand.To resolve this issue, this patch makes qemu wait for the commands result explicilty if
VHOST_USER_PROTOCOL_F_REPLY_ACKis negotiated and logging is enabled.Signed-off-by Denis Plotnikov <[email protected]>---
v1 -> v2:
* send reply only when logging is enabled [mst]v0 -> v1:
* send reply forSET_VRING_ADDR,SET_FEATURESonly [mst]hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)hw/virtio/vhost-user.c +12
-0@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {
hw/virtio/vhost-user.c static int vhost_user_set_vring_addr(struct vhost_dev *dev, · line 1130
Quoted 10 lines
hw/virtio/vhost-user.c +4
-0return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }
OK this is good, but the problem is that we then still have a race if VHOST_USER_PROTOCOL_F_REPLY_ACK is not set. Bummer.
Let's send VHOST_USER_GET_FEATURES in this case to flush out outstanding messages?
Quoted 52 lines
hw/virtio/vhost-user.c +18
-3@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)Signature · 1 line
2.25.1
Quoted 79 lines · Michael S. Tsirkin
Quoted 65 lines · Denis Plotnikov
On vhost-user-blk migration, qemu normally sends a number of commands to enable logging if
VHOST_USER_PROTOCOL_F_LOG_SHMFDis negotiated. Qemu sendsVHOST_USER_SET_FEATURESto enable buffers logging andVHOST_USER_SET_VRING_ADDRper each started ring to enable "used ring" data logging.
The issue is that qemu doesn't wait for reply from the vhost daemon for these commands which may result in races between qemu expectation of logging starting and actual login starting in vhost daemon.The race can appear as follows: on migration setup, qemu enables dirty page logging by sending
VHOST_USER_SET_FEATURES. The command doesn't arrive to a vhost-user-blk daemon immediately and the daemon needs some time to turn the logging on internally. If qemu doesn't wait for reply, after sending the command, qemu may start migrate memory pages to a destination. At this time, the logging may not be actually turned on in the daemon but some guest pages, which the daemon is about to write to, may have already been transferred without logging to the destination. Since the logging wasn't turned on, those pages won't be transferred again as dirty. So we may end up with corrupted data on the destination.
The same scenario is applicable for "used ring" data logging, which is turned on withVHOST_USER_SET_VRING_ADDRcommand.To resolve this issue, this patch makes qemu wait for the commands result explicilty if
VHOST_USER_PROTOCOL_F_REPLY_ACKis negotiated and logging is enabled.Signed-off-by Denis Plotnikov <[email protected]>---
v1 -> v2:
* send reply only when logging is enabled [mst]v0 -> v1:
* send reply forSET_VRING_ADDR,SET_FEATURESonly [mst]hw/virtio/vhost-user.c | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-)hw/virtio/vhost-user.c +12
-0@@ -1095,6 +1095,11 @@ static int vhost_user_set_mem_table(struct vhost_dev *dev,return 0 ; } +static bool log_enabled (uint64_t features) +{ +return !!(features & (0x1ULL <<VHOST_F_LOG_ALL )); +} +static int vhost_user_set_vring_addr (struct vhost_dev *dev,struct vhost_vring_addr *addr) {@@ -1105,10 +1110,21 @@ static int vhost_user_set_vring_addr(struct vhost_dev *dev, .hdr.size = sizeof(msg.payload.addr), }; +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); + +if (reply_supported &&log_enabled (msg.hdr.flags)) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {hw/virtio/vhost-user.c static int vhost_user_set_vring_addr(struct vhost_dev *dev, · line 1130
Quoted 10 lines
hw/virtio/vhost-user.c +4
-0return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }OK this is good, but the problem is that we then still have a race if
VHOST_USER_PROTOCOL_F_REPLY_ACKis not set. Bummer.
Quoted 2 lines
Let's send
VHOST_USER_GET_FEATURESin this case to flush out outstanding messages?
Ok, I've already sent v3 with related changes.
Quoted 53 lines
Quoted 53 lines
hw/virtio/vhost-user.c +18
-3@@ -1288,7 +1304,8 @@ static int vhost_user_set_vring_call(struct vhost_dev *dev,return vhost_set_vring_file (dev,VHOST_USER_SET_VRING_CALL , file); } -static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64) +static int vhost_user_set_u64 (struct vhost_dev *dev,int request,uint64_t u64, +bool need_reply) {VhostUserMsg msg = { .hdr.request = request,@@ -1297,23 +1314,37 @@ static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64) .hdr.size = sizeof(msg.payload.u64), }; +if (need_reply) { +bool reply_supported =virtio_has_feature (dev->protocol_features, +VHOST_USER_PROTOCOL_F_REPLY_ACK ); +if (reply_supported) { + msg.hdr.flags |=VHOST_USER_NEED_REPLY_MASK ; + } + } +if (vhost_user_write (dev, &msg,NULL ,0 ) <0 ) {return -1 ; } +if (msg.hdr.flags &VHOST_USER_NEED_REPLY_MASK ) { +return process_message_reply (dev, &msg); + } +return 0 ; }static int vhost_user_set_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_FEATURES , features, +log_enabled (features)); }static int vhost_user_set_protocol_features (struct vhost_dev *dev,uint64_t features) { -return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features); +return vhost_user_set_u64 (dev,VHOST_USER_SET_PROTOCOL_FEATURES , features, + false); }static int vhost_user_get_u64 (struct vhost_dev *dev,int request,uint64_t *u64)Signature · 1 line
2.25.1