vhost-user block device backend implementation
cover vhost-user block device backend implementation
This patch series is an implementation of vhost-user-backend server, thanks to Stefan's guidance after reviewing the draft version and the instructions on https://wiki.qemu.org/Google_Summer_of_Code_2019#vhost-user-blk_device_backend,
Vhost-user-backend server is a UserCreatable object and can be started using object_add,
(qemu) object_add vhost-user-server,id=ID,unix_socket=/tmp/vhost-user-blk_vhost.socket,name=DRIVE_NAME,writable=off (qemu) object_del ID
or appending the "-object" option when starting QEMU,
$ -object vhost-user-server,id=disk,unix_socket=/tmp/vhost-user-blk_vhost.socket,name=disk,writable=off
Then vhost-user client can connect to the server backend. For example, QEMU could act as a client,
$ -m 256 -object memory-backend-memfd,id=mem,size=256M,share=on -numa node,memdev=mem -chardev socket,id=char1,path=/tmp/vhost-user-blk_vhost.socket -device vhost-user-blk-pci,id=blk0,chardev=char1
And guest OS could access this vhost-user block device after mouting it.
patches are against commit 035eed4c0d257c905a556fa0f4865a0c077b4e7f.
Coiby Xu (5):
vhost-user block device backend
extend libvhost to support IOThread
a standone-alone tool to directly share disk image file via vhost-user
protocol
new qTest case for the vhost-user-blk device backend building configuration files changes
Makefile | 1 + Makefile.objs | 2 +- Makefile.target | 1 + blockdev-vu.c | 1008 +++++++++++++++++++++++++ configure | 2 +- contrib/libvhost-user/libvhost-user.c | 64 +- contrib/libvhost-user/libvhost-user.h | 36 +- include/block/vhost-user.h | 46 ++ qemu-vu.c | 264 +++++++ tests/Makefile.include | 5 +- tests/libqos/vhost-user-blk.c | 125 +++ tests/libqos/vhost-user-blk.h | 44 ++ tests/vhost-user-blk-test.c | 691 +++++++++++++++++ vl.c | 4 + 14 files changed, 2277 insertions(+), 16 deletions(-) create mode 100644 blockdev-vu.c create mode 100644 include/block/vhost-user.h create mode 100644 qemu-vu.c create mode 100644 tests/libqos/vhost-user-blk.c create mode 100644 tests/libqos/vhost-user-blk.h create mode 100644 tests/vhost-user-blk-test.c
--
2.24.1
PATCH 1/5 vhost-user block device backend
By making use of libvhost, multiple block device drives can be exported and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.
--- blockdev-vu.c | 1008 ++++++++++++++++++++++++++++++++++++ include/block/vhost-user.h | 46 ++ vl.c | 4 + 3 files changed, 1058 insertions(+) create mode 100644 blockdev-vu.c create mode 100644 include/block/vhost-user.h
blockdev-vu.c new file +1008 -0
@@ -0,0 +1,1008 @@ +#include "qemu/osdep.h" +#include "block/vhost-user.h" +#include "qapi/error.h" +#include "qapi/qapi-types-sockets.h" +#include "qapi/qapi-commands-block.h" + +#include "sysemu/block-backend.h" +#include "qemu/main-loop.h" + +#include "qemu/units.h" + +#include "block/block.h" + +#include "qom/object_interfaces.h" + +#include <sys/eventfd.h> + +#include "hw/qdev-properties.h" +enum { +VHOST_USER_BLK_MAX_QUEUES =8 , +}; + +struct virtio_blk_inhdr { +unsigned char status; +}; + + +static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs); + + +typedef struct VubReq { +VuVirtqElement *elem; +int64_t sector_num; +size_t size; +struct virtio_blk_inhdr *in; +struct virtio_blk_outhdr out; +VuClient *client; +struct VuVirtq *vq; +}VubReq ; + +static void +remove_watch (VuDev *vu_dev,int fd) +{ +VuClient *client; + +g_assert (vu_dev); +g_assert (fd >=0 ); + + client =container_of (vu_dev, VuClient, parent); +aio_set_fd_handler (client->blk->ctx, fd, false,NULL ,NULL ,NULL ,NULL ); +} + +static void close_client (VuClient *client) +{ +vu_deinit (&client->parent); +/** g_source_destroy(vub_device->parent.src); */ + client->sioc =NULL ; +object_unref (OBJECT (client->ioc)); + client->closed = true; + +} + +static void vub_panic_cb (VuDev *vu_dev,const char *buf) +{ +if (buf) { +g_warning ("vu_panic: %s" , buf); + } + +VuClient *client =container_of (vu_dev, VuClient, parent); +if (client->blk->exit_panic) { + client->blk->close = true; + } +if (!client->closed) { +close_client (client); + } +} + + +static void vub_req_complete (VubReq *req) +{ +VuDev *vu_dev = &req->client->parent; + +/* IO size with 1 extra status byte */ +vu_queue_push (vu_dev, req->vq, req->elem, + req->size +1 ); +vu_queue_notify (vu_dev, req->vq); + +if (req->elem) { +free (req->elem); + } + +g_free (req); +} + + + +static int +vub_discard_write_zeroes (VubReq *req,struct iovec *iov,uint32_t iovcnt, +uint32_t type) +{ +struct virtio_blk_discard_write_zeroes *desc; +ssize_t size; +void *buf; + + size =iov_size (iov, iovcnt); +if (size !=sizeof (*desc)) { +fprintf (stderr,"Invalid size %ld, expect %ld\n" , size,sizeof (*desc)); +return -1 ; + } + buf =g_new0 (char, size); + +iov_to_buf_full (iov, iovcnt,0 , buf, size); + + +#if defined(__linux__) && defined(BLKDISCARD ) && defined(BLKZEROOUT ) +VubDev *vdev_blk; +VuClient *client =container_of (dev, VuClient, parent); + vdev_blk = client->blk; + desc = (struct virtio_blk_discard_write_zeroes *)buf; +uint64_t range[2 ] = {le64toh (desc->sector) <<9 , +le32toh (desc->num_sectors) <<9 }; +if (type ==VIRTIO_BLK_T_DISCARD ) { +if (blk_pdiscard (vdev_blk->blk, range[0 ], range[1 ]) ==0 ) { +g_free (buf); +return 0 ; + } + }else if (type ==VIRTIO_BLK_T_WRITE_ZEROES ) { +if (blk_pwrite_zeroes (vdev_blk->blk, range[0 ], range[1 ]) ==0 ) { +g_free (buf); +return 0 ; + } + } +#endif + +g_free (buf); +return -1 ; +} + + +static void +vub_flush (VubReq *req) +{ +VuClient *client = req->client; +blk_co_flush (client->blk->backend); +} + + +#define NOT_DONE 0x7fffffff/* used while emulated sync operation in progress */ +typedef struct BlkRwCo { +BlockBackend *blk; +int64_t offset; +void *iobuf; +int ret; +BdrvRequestFlags flags; +}BlkRwCo ; + +static void blk_read_entry (void *opaque) +{ +BlkRwCo *rwco = opaque; +QEMUIOVector *qiov = rwco->iobuf; + + rwco->ret =blk_co_preadv (rwco->blk, rwco->offset, qiov->size, + qiov, rwco->flags); +aio_wait_kick (); +} + + +static void blk_write_entry (void *opaque) +{ +BlkRwCo *rwco = opaque; +QEMUIOVector *qiov = rwco->iobuf; + + rwco->ret =blk_co_pwritev (rwco->blk, rwco->offset, qiov->size, + qiov, rwco->flags); +aio_wait_kick (); +} + + +static int blk_prw (BlockBackend *blk,QEMUIOVector *qiov,int64_t offset, +CoroutineEntry co_entry,BdrvRequestFlags flags) +{ + +BlkRwCo rwco = { + .blk = blk, + .offset = offset, + .iobuf = qiov, + .flags = flags, + .ret =NOT_DONE , + }; + +if (qemu_in_coroutine ()) { +/* Fast-path if already in coroutine context */ +co_entry (&rwco); + }else { +Coroutine *co =qemu_coroutine_create (co_entry, &rwco); +bdrv_coroutine_enter (blk_bs (blk), co); +BDRV_POLL_WHILE (blk_bs (blk), rwco.ret ==NOT_DONE ); + } + +return rwco.ret; +} + + +static ssize_t +vub_rwv (VubReq *req,struct iovec *iov, +uint32_t iovcnt, +CoroutineEntry co_entry) +{ +VuClient *client = req->client; +ssize_t rc; + +if (!iovcnt) { +fprintf (stderr,"Invalid Read/Write IOV count\n" ); +return -1 ; + } + +int64_t offset = req->sector_num *512 ; +QEMUIOVector *qiov =g_new0 (QEMUIOVector,1 ); +qemu_iovec_init_external (qiov, iov, iovcnt); + rc =blk_prw (client->blk->backend, qiov, offset, co_entry,0 ); + + req->size =iov_size (iov, iovcnt); +if (rc <0 ) { +fprintf (stderr,"%s, Sector %" PRIu64", Size %lu failed with %s\n" , + client->blk->name, req->sector_num, req->size, +strerror (errno)); +return -1 ; + } + +return rc; +} + +static int vub_virtio_process_req (VuClient *client, +VuVirtq *vq) +{ +VuDev *vu_dev = &client->parent; +VuVirtqElement *elem; +uint32_t type; +VubReq *req; + + elem =vu_queue_pop (vu_dev, vq,sizeof (VuVirtqElement) +sizeof (VubReq)); +if (!elem) { +return -1 ; + } + +struct iovec *in_iov = elem->in_sg; +struct iovec *out_iov = elem->out_sg; +unsigned in_num = elem->in_num; +unsigned out_num = elem->out_num; +/* refer to hw/block/virtio_blk.c */ +if (elem->out_num <1 || elem->in_num <1 ) { +fprintf (stderr,"virtio-blk request missing headers\n" ); +free (elem); +return -1 ; + } + + req =g_new0 (VubReq,1 ); + req->client = client; + req->vq = vq; + req->elem = elem; + +if (unlikely (iov_to_buf (out_iov, out_num,0 , &req->out, +sizeof (req->out)) !=sizeof (req->out))) { +fprintf (stderr,"virtio-blk request outhdr too short" ); + goto err; + } + +iov_discard_front (&out_iov, &out_num,sizeof (req->out)); + +if (in_iov[in_num -1 ].iov_len <sizeof (struct virtio_blk_inhdr )) { +fprintf (stderr,"virtio-blk request inhdr too short" ); + goto err; + } + +/* We always touch the last byte, so just see how big in_iov is. */ + req->in = (void *)in_iov[in_num -1 ].iov_base + + in_iov[in_num -1 ].iov_len + -sizeof (struct virtio_blk_inhdr ); +iov_discard_back (in_iov, &in_num,sizeof (struct virtio_blk_inhdr )); + + + type =le32toh (req->out.type); +switch (type & ~VIRTIO_BLK_T_BARRIER ) { +case VIRTIO_BLK_T_IN : +case VIRTIO_BLK_T_OUT : { +ssize_t ret =0 ; +bool is_write = type &VIRTIO_BLK_T_OUT ; + req->sector_num =le64toh (req->out.sector); +if (is_write) { + ret =vub_rwv (req, out_iov, out_num, blk_write_entry); + }else { + ret =vub_rwv (req, in_iov, in_num, blk_read_entry); + } +if (ret >=0 ) { + req->in->status =VIRTIO_BLK_S_OK ; + }else { + req->in->status =VIRTIO_BLK_S_IOERR ; + } +vub_req_complete (req); +break ; + } +case VIRTIO_BLK_T_FLUSH : +vub_flush (req); + req->in->status =VIRTIO_BLK_S_OK ; +vub_req_complete (req); +break ; +case VIRTIO_BLK_T_GET_ID : { +size_t size =MIN (iov_size (&elem->in_sg[0 ], in_num), +VIRTIO_BLK_ID_BYTES ); +snprintf (elem->in_sg[0 ].iov_base, size,"%s" ,"vhost_user_blk" ); + req->in->status =VIRTIO_BLK_S_OK ; + req->size = elem->in_sg[0 ].iov_len; +vub_req_complete (req); +break ; + } +case VIRTIO_BLK_T_DISCARD : +case VIRTIO_BLK_T_WRITE_ZEROES : { +int rc; + rc =vub_discard_write_zeroes (req, &elem->out_sg[1 ], out_num, type); +if (rc ==0 ) { + req->in->status =VIRTIO_BLK_S_OK ; + }else { + req->in->status =VIRTIO_BLK_S_IOERR ; + } +vub_req_complete (req); +break ; + } +default : + req->in->status =VIRTIO_BLK_S_UNSUPP ; +vub_req_complete (req); +break ; + } + +return 0 ; + +err: +free (elem); +g_free (req); +return -1 ; +} + + +static void vub_process_vq (VuDev *vu_dev,int idx) +{ +VuClient *client; +VuVirtq *vq; +int ret; + + client =container_of (vu_dev, VuClient, parent); +assert (client); + + vq =vu_get_queue (vu_dev, idx); +assert (vq); + +while (1 ) { + ret =vub_virtio_process_req (client, vq); +if (ret) { +break ; + } + } +} + + +static void vub_queue_set_started (VuDev *vu_dev,int idx,bool started) +{ +VuVirtq *vq; + +assert (vu_dev); + + vq =vu_get_queue (vu_dev, idx); +vu_set_queue_handler (vu_dev, vq, started ? vub_process_vq :NULL ); +} + +static uint64_t +vub_get_features (VuDev *dev) +{ +uint64_t features; +VubDev *vdev_blk; + +VuClient *client =container_of (dev, VuClient, parent); + vdev_blk = client->blk; + + features =1ull <<VIRTIO_BLK_F_SIZE_MAX | +1ull <<VIRTIO_BLK_F_SEG_MAX | +1ull <<VIRTIO_BLK_F_TOPOLOGY | +1ull <<VIRTIO_BLK_F_BLK_SIZE | +1ull <<VIRTIO_BLK_F_FLUSH | + #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT) + 1ull << VIRTIO_BLK_F_DISCARD | +1ull <<VIRTIO_BLK_F_WRITE_ZEROES | + #endif +1ull <<VIRTIO_BLK_F_CONFIG_WCE | +1ull <<VIRTIO_F_VERSION_1 | +1ull <<VIRTIO_RING_F_INDIRECT_DESC | +1ull <<VIRTIO_RING_F_EVENT_IDX | +1ull <<VHOST_USER_F_PROTOCOL_FEATURES ; + +if (!vdev_blk->writable) { + features |=1ull <<VIRTIO_BLK_F_RO ; + } + +return features; +} + +static uint64_t +vub_get_protocol_features (VuDev *dev) +{ +return 1ull <<VHOST_USER_PROTOCOL_F_CONFIG | +1ull <<VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD ; +} + +static int +vub_get_config (VuDev *vu_dev,uint8_t *config,uint32_t len) +{ +VubDev *vdev_blk; + +VuClient *client =container_of (vu_dev, VuClient, parent); + vdev_blk = client->blk; +memcpy (config, &vdev_blk->blkcfg, len); + +return 0 ; +} + +static int +vub_set_config (VuDev *vu_dev,const uint8_t *data, +uint32_t offset,uint32_t size,uint32_t flags) +{ +VubDev *vdev_blk; + +VuClient *client =container_of (vu_dev, VuClient, parent); + vdev_blk = client->blk; +uint8_t wce; + +/* don't support live migration */ +if (flags !=VHOST_SET_CONFIG_TYPE_MASTER ) { +return -1 ; + } + + +if (offset != offsetof(struct virtio_blk_config , wce) || + size !=1 ) { +return -1 ; + } + + wce = *data; +if (wce == vdev_blk->blkcfg.wce) { +/* Do nothing as same with old configuration */ +return 0 ; + } + + vdev_blk->blkcfg.wce = wce; +blk_set_enable_write_cache (vdev_blk->backend, true); +return 0 ; +} + + +/* +* When the client disconnects, it send a VHOST_USER_NONE request +* and vu_process_message will simple call exit which cause the VM +* to exit abruptly. +* To avoid this issue, process VHOST_USER_NONE request ahead +* of vu_process_message. +* +*/ +static int vub_process_msg (VuDev *dev,VhostUserMsg *vmsg,int *do_reply) +{ +if (vmsg->request ==VHOST_USER_NONE ) { + dev->panic (dev,"disconnect" ); +return true; + } +return false; +} + +static void +vmsg_close_fds (VhostUserMsg *vmsg) +{ +int i; +for (i =0 ; i < vmsg->fd_num; i++) { +close (vmsg->fds[i]); + } +} + +static bool +vu_message_read_co (VuDev *vu_dev,int conn_fd,VhostUserMsg *vmsg) +{ +char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS *sizeof (int ))] = { }; +struct iovec iov = { + .iov_base = (char *)vmsg, + .iov_len =VHOST_USER_HDR_SIZE , + }; +struct msghdr msg = { + .msg_iov = &iov, + .msg_iovlen =1 , + .msg_control = control, + .msg_controllen =sizeof (control), + }; +size_t fd_size; +struct cmsghdr *cmsg; +int rc; +char buffer[100 ]; +VuClient *client =container_of (vu_dev, VuClient, parent); +QIOChannel *ioc = client->ioc; +do { + rc =recvmsg (conn_fd, &msg,0 ); +if (rc <0 ) { +if (errno ==EAGAIN ) { +if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_IN ); + }else { +qio_channel_wait (ioc,G_IO_IN ); + } +continue ; + }else if (errno ==EINTR ) { +continue ; + } + } +break ; + }while (true); + +if (rc <0 ) { +sprintf (buffer,"Error while recvmsg: %s" ,strerror (errno)); +vub_panic_cb (vu_dev, buffer); +return false; + } + +assert (rc ==VHOST_USER_HDR_SIZE || rc ==0 ); + + vmsg->fd_num =0 ; +for (cmsg =CMSG_FIRSTHDR (&msg); + cmsg !=NULL ; + cmsg =CMSG_NXTHDR (&msg, cmsg)) + { +if (cmsg->cmsg_level ==SOL_SOCKET && cmsg->cmsg_type ==SCM_RIGHTS ) { + fd_size = cmsg->cmsg_len -CMSG_LEN (0 ); + vmsg->fd_num = fd_size /sizeof (int ); +memcpy (vmsg->fds,CMSG_DATA (cmsg), fd_size); +break ; + } + } + +if (vmsg->size >sizeof (vmsg->payload)) { +sprintf (buffer, +"Error: too big message request: %d, size: vmsg->size: %u, " +"while sizeof(vmsg->payload) = %zu\n" , + vmsg->request, vmsg->size,sizeof (vmsg->payload)); +vub_panic_cb (vu_dev, buffer); + goto fail; + } + +if (vmsg->size) { +do { + rc =read (conn_fd, &vmsg->payload, vmsg->size); +if (rc <0 ) { +if (errno ==EAGAIN ) { +if (qemu_in_coroutine ()) { +qio_channel_yield (ioc,G_IO_IN ); + }else { +qio_channel_wait (ioc,G_IO_IN ); + } +continue ; + }else if (errno ==EINTR ) { +continue ; + } + } +break ; + }while (true); + +if (rc <=0 ) { +sprintf (buffer,"Error while reading: %s" ,strerror (errno)); +vub_panic_cb (vu_dev, buffer); + goto fail; + } + +assert (rc == vmsg->size); + } + +return true; + +fail: +vmsg_close_fds (vmsg); + +return false; +} + +static void vub_kick_cb (void *opaque) +{ +vu_watch_cb_data *data = (vu_watch_cb_data *) opaque; +int index = data->index; +VuDev *dev = data->vu_dev; +VuVirtq *vq = &dev->vq[index]; +int sock = vq->kick_fd; +eventfd_t kick_data; +ssize_t rc; + + rc =eventfd_read (sock, &kick_data); +if (rc ==-1 ) { +char buffer[100 ]; +sprintf (buffer,"kick eventfd_read(): %s" ,strerror (errno)); +vub_panic_cb (dev, buffer); +g_free (data); + dev->remove_watch (dev, dev->vq[index].kick_fd); + }else { +if (vq->handler) { + vq->handler (dev, index); + } + } +} + +static const VuDevIface vub_iface = { + .get_features = vub_get_features, + .queue_set_started = vub_queue_set_started, + .get_protocol_features = vub_get_protocol_features, + .get_config = vub_get_config, + .set_config = vub_set_config, + .process_msg = vub_process_msg, + .read_msg = vu_message_read_co, + .kick_callback = vub_kick_cb, +}; + + +void vub_free (VubDev *vub_dev,bool called_by_QOM) +{ +if (!vub_dev) { +return ; + } + +blk_unref (vub_dev->backend); +g_free (vub_dev->name); +g_free (vub_dev->unix_socket); + +if (vub_dev->next.tqe_circ.tql_prev) { +/* +* if vub_dev->next.tqe_circ.tql_prev = null, +* vub_dev hasn't been inserted into the queue and +* vub_free is called by obj->instance_finalize. +*/ +QTAILQ_REMOVE (&vub_devs, vub_dev, next); + } +/* +* Needn't to free vub_dev if called by QOM +* because QOM will do the clean-up work. +*/ +if (!called_by_QOM) { +g_free (vub_dev); + } +} + +static coroutine_fn void vu_client_trip(void *opaque) +{ +VuClient *client = opaque; + +while (!client->closed) { +vu_dispatch (&client->parent); + } + +QTAILQ_REMOVE (&client->blk->clients, client, next); + +} + +static void vu_client_start (VuClient *client) +{ +Coroutine *co =qemu_coroutine_create (vu_client_trip, client); +qemu_coroutine_enter (co); +} + + +G_STATIC_ASSERT ((int )G_IO_IN == (int )VU_WATCH_IN ); +G_STATIC_ASSERT ((int )G_IO_OUT == (int )VU_WATCH_OUT ); +G_STATIC_ASSERT ((int )G_IO_PRI == (int )VU_WATCH_PRI ); +G_STATIC_ASSERT ((int )G_IO_ERR == (int )VU_WATCH_ERR ); +G_STATIC_ASSERT ((int )G_IO_HUP == (int )VU_WATCH_HUP ); + +static void +set_watch (VuDev *vu_dev,int fd,int vu_evt, +vu_watch_cb_packed_data cb,void *pvt) +{ +/* +* since aio_dispatch can only pass one user data pointer to the +* callback function, pack VuDev, pvt into a struct +*/ +VuClient *client; + +g_assert (vu_dev); +g_assert (fd >=0 ); +g_assert (cb); + client =container_of (vu_dev, VuClient, parent); +vu_watch_cb_data *cb_data =g_new0 (vu_watch_cb_data,1 ); + cb_data->index = (intptr_t ) pvt; + cb_data->vu_dev = vu_dev; +aio_set_fd_handler (client->blk->ctx, fd, false, (void *) cb, +NULL ,NULL , cb_data); +} + + +void vub_accept (QIONetListener *listener,QIOChannelSocket *sioc, +gpointer opaque) +{ +VuClient *client; +VubDev *vub_device = opaque; + client =g_new0 (VuClient,1 ); + +if (!vu_init_packed_data (&client->parent,VHOST_USER_BLK_MAX_QUEUES , + sioc->fd, vub_panic_cb, set_watch, + remove_watch, &vub_iface)) { +fprintf (stderr,"Failed to initialized libvhost-user\n" ); +g_free (client); +return ; + } + + client->blk = vub_device; + client->refcount =1 ; + client->sioc = sioc; +/* +* increase the object reference, so cioc will not freed by +* qio_net_listener_channel_func which will call object_unref(OBJECT(sioc)) +*/ +object_ref (OBJECT (client->sioc)); +qio_channel_set_name (QIO_CHANNEL (sioc),"vhost-user client" ); + client->ioc =QIO_CHANNEL (sioc); +object_ref (OBJECT (client->ioc)); +object_ref (OBJECT (sioc)); + +qio_channel_set_blocking (QIO_CHANNEL (client->sioc), false,NULL ); + client->closed = false; +QTAILQ_INSERT_TAIL (&client->blk->clients, client, next); +vu_client_start (client); +} + + +void +vub_initialize_config (BlockDriverState *bs,struct virtio_blk_config *config) +{ + config->capacity =bdrv_getlength (bs) >>BDRV_SECTOR_BITS ; + config->blk_size =BDRV_SECTOR_SIZE ; + config->size_max =65536 ; + config->seg_max =128 -2 ; + config->min_io_size =1 ; + config->opt_io_size =1 ; + config->num_queues =1 ; +#if defined(__linux__) && defined(BLKDISCARD ) && defined(BLKZEROOUT ) + config->max_discard_sectors =32768 ; + config->max_discard_seg =1 ; + config->discard_sector_alignment = config->blk_size >>9 ; + config->max_write_zeroes_sectors =32768 ; + config->max_write_zeroes_seg =1 ; +#endif +} + + +static VubDev *vub_new (VubDev *vub_device,const char *name, +const char *unix_socket,bool writable,Error **errp) +{ + +BlockBackend *blk; + +/* +* Don't allow resize while the vhost user server is running, +* otherwise we don't care what happens with the node. +*/ +uint64_t perm =BLK_PERM_CONSISTENT_READ ; +int ret; + +AioContext *ctx; + +BlockDriverState *bs =bdrv_lookup_bs (name, + name, + errp); + +if (!bs) { +error_setg (errp, +"No drive with name '%s'." +" Please find the list of names with " +"'info block'" , name); +return NULL ; + } + +if (bdrv_is_read_only (bs)) { + writable = false; + } + +if (writable) { + perm |=BLK_PERM_WRITE ; + } + + ctx =bdrv_get_aio_context (bs); +aio_context_acquire (ctx); +bdrv_invalidate_cache (bs,NULL ); +aio_context_release (ctx); + + blk =blk_new (bdrv_get_aio_context (bs), perm, +BLK_PERM_CONSISTENT_READ |BLK_PERM_WRITE_UNCHANGED | +BLK_PERM_WRITE |BLK_PERM_GRAPH_MOD ); + ret =blk_insert_bs (blk, bs, errp); + +if (ret <0 ) { + goto fail; + } + + +blk_set_enable_write_cache (blk, false); + +blk_set_allow_aio_context_change (blk, true); + + + vub_device->name =g_strdup (name); + vub_device->unix_socket =g_strdup (unix_socket); + vub_device->writable = writable; + vub_device->blkcfg.wce =0 ; + vub_device->backend = blk; + vub_device->ctx = ctx; +vub_initialize_config (bs, &vub_device->blkcfg); +return vub_device; + +fail: +blk_unref (blk); +return NULL ; +} + +void vhost_user_server_free (VubDev *vub_device,bool called_by_QOM) +{ +if (!vub_device) { +return ; + } + +VuClient *client, *next; + QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) { +if (!client->closed) { +close_client (client); + } + } + +if (vub_device->listener) { +qio_net_listener_disconnect (vub_device->listener); +object_unref (OBJECT (vub_device->listener)); + } +vub_free (vub_device, called_by_QOM); + +} + + +VubDev *vub_dev_find (const char *name) +{ +VubDev *vub_device; + QTAILQ_FOREACH(vub_device, &vub_devs, next) { +if (strcmp (name, vub_device->name) ==0 ) { +return vub_device; + } + } + +return NULL ; +} + + +static VubDev *vub_dev_find_by_unix_socket (const char *unix_socket) +{ +VubDev *vub_device; + QTAILQ_FOREACH(vub_device, &vub_devs, next) { +if (strcmp (unix_socket, vub_device->unix_socket) ==0 ) { +return vub_device; + } + } + +return NULL ; +} + +static void vhost_user_server_start (VubDev *vub_device,const char *unix_socket, +const char *name,bool writable, +Error **errp) +{ + +if (vub_dev_find (name) ||vub_dev_find_by_unix_socket (unix_socket)) { +error_setg (errp,"Vhost user server with name '%s' or " +"with socket_path '%s' has already been started" , + name, unix_socket); +return ; + } + + +if (!vub_new (vub_device, name, unix_socket, writable, errp)) { +return ; + } + + + vub_device->listener =qio_net_listener_new (); + +qio_net_listener_set_name (vub_device->listener, +"vhost-user-backend-listener" ); + +SocketAddress *addr =g_new0 (SocketAddress,1 ); + addr->u.q_unix.path = (char *) unix_socket; + addr->type =SOCKET_ADDRESS_TYPE_UNIX ; +if (qio_net_listener_open_sync (vub_device->listener, addr,1 , errp) <0 ) { + goto error; + } + + +QTAILQ_INSERT_TAIL (&vub_devs, vub_device, next); +QTAILQ_INIT (&vub_device->clients); + +qio_net_listener_set_client_func (vub_device->listener, + vub_accept, + vub_device, +NULL ); + +return ; + + error: +vub_free (vub_device, false); +} + +static void vu_set_block_name (Object *obj,const char *value, +Error **errp) +{ +VubDev *vus =VHOST_USER_SERVER (obj);; + +if (vus->name) { +error_setg (errp,"evdev property already set" ); +return ; + } + + vus->name =g_strdup (value); +} + +static char *vu_get_block_name (Object *obj,Error **errp) +{ +VubDev *vus =VHOST_USER_SERVER (obj); +return g_strdup (vus->name); +} + + +static void vu_set_unix_socket (Object *obj,const char *value, +Error **errp) +{ +VubDev *vus =VHOST_USER_SERVER (obj);; + +if (vus->unix_socket) { +error_setg (errp,"unix_socket property already set" ); +return ; + } + + vus->unix_socket =g_strdup (value); +vhost_user_server_start (vus, value, vus->name, + vus->writable, errp); +} + +static char *vu_get_unix_socket (Object *obj,Error **errp) +{ +VubDev *vus =VHOST_USER_SERVER (obj);; +return g_strdup (vus->unix_socket); +} + +static bool vu_get_block_writable (Object *obj,Error **errp) +{ +VubDev *vus =VHOST_USER_SERVER (obj);; +return vus->writable; +} + +static void vu_set_block_writable (Object *obj,bool value,Error **errp) +{ +VubDev *vus =VHOST_USER_SERVER (obj); + + vus->writable = value; +} + +static void vhost_user_server_instance_init (Object *obj) +{ + +object_property_add_bool (obj,"writable" , + vu_get_block_writable, + vu_set_block_writable,NULL ); + +object_property_add_str (obj,"name" , + vu_get_block_name, + vu_set_block_name,NULL ); + +object_property_add_str (obj,"unix_socket" , + vu_get_unix_socket, + vu_set_unix_socket,NULL ); + +} + +static void vhost_user_server_instance_finalize (Object *obj) +{ +VubDev *vus =VHOST_USER_SERVER (obj); +vhost_user_server_free (vus, true); +/* object_del shouldn't free this object struct */ + obj->free =NULL ; +} + +static const TypeInfo vhost_user_server_info = { + .name =TYPE_VHOST_USER_SERVER , + .parent =TYPE_OBJECT , + .instance_size =sizeof (VuDev), + .instance_init = vhost_user_server_instance_init, + .instance_finalize = vhost_user_server_instance_finalize, + .interfaces = (InterfaceInfo []) { + {TYPE_USER_CREATABLE }, + {} + }, +}; + +static void vhost_user_server_register_types (void ) +{ +type_register_static (&vhost_user_server_info); +} + +type_init (vhost_user_server_register_types) +
include/block/vhost-user.h new file +46 -0
46 lines not on this page · raw message
vl.c +4 -0
@@ -2553,6 +2553,10 @@ static bool object_create_initial(const char *type, QemuOpts *opts) }#endif +/* Reason: vhost-user-server property "name" */ +if (g_str_equal (type,"vhost-user-server" )) { +return false; + }/* * Reason: filter-* property "netdev" etc. */
--
2.24.1
PATCH 2/5 extend libvhost to support IOThread
In reply to Coiby XuPreviously libvhost dispatch events in its own GMainContext. Now vhost-user client's kick event can be dispatched in block device drive's AioContext thus IOThread is supported.
--- contrib/libvhost-user/libvhost-user.c | 64 ++++++++++++++++++++++----- contrib/libvhost-user/libvhost-user.h | 36 ++++++++++++++- 2 files changed, 87 insertions(+), 13 deletions(-)
contrib/libvhost-user/libvhost-user.c +53 -11
@@ -67,7 +67,6 @@/* The version of inflight buffer */ #define INFLIGHT_VERSION 1 -#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)/* The version of the protocol we support */ #define VHOST_USER_VERSION 1
@@ -260,7 +259,7 @@ have_userfault(void) }static bool -vu_message_read (VuDev *dev,int conn_fd,VhostUserMsg *vmsg) +vu_message_read_ (VuDev *dev,int conn_fd,VhostUserMsg *vmsg) {char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS *sizeof (int ))] = { };struct iovec iov = {
@@ -286,6 +285,8 @@ vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)return false; } +assert (rc ==VHOST_USER_HDR_SIZE || rc ==0 ); + vmsg->fd_num =0 ; for (cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL;
@@ -328,6 +329,17 @@ fail:return false; } +static bool vu_message_read (VuDev *dev,int conn_fd,VhostUserMsg *vmsg) +{ +vu_read_msg_cb read_msg; +if (dev->iface->read_msg) { + read_msg = dev->iface->read_msg; + }else { + read_msg = vu_message_read_; + } +return read_msg (dev, conn_fd, vmsg); +} +static bool vu_message_write (VuDev *dev,int conn_fd,VhostUserMsg *vmsg) {
@@ -400,7 +412,6 @@ vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)if ((vmsg->flags &VHOST_USER_NEED_REPLY_MASK ) ==0 ) {return true; } -if (!vu_message_read (dev, dev->slave_fd, &msg_reply)) {return false; }
@@ -644,7 +655,8 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)"%s: Failed to madvise(DONTNEED) region %d: %s\n" , __func__, i, strerror(errno)); } -/* Turn off transparent hugepages so we dont get lose wakeups +/* +* Turn off transparent hugepages so we don't get lose wakeups * in neighbouring pages. * TODO: Turn this backon later. */
@@ -1047,9 +1059,13 @@ vu_set_vring_kick_exec(VuDev *dev, VhostUserMsg *vmsg) }if (dev->vq[index].kick_fd !=-1 && dev->vq[index].handler) { - dev->set_watch (dev, dev->vq[index].kick_fd,VU_WATCH_IN , - vu_kick_cb, (void *)(long )index); - +if (dev->set_watch_packed_data) { + dev->set_watch_packed_data (dev, dev->vq[index].kick_fd,VU_WATCH_IN , + dev->iface->kick_callback, (void *)(long )index); + }else { + dev->set_watch (dev, dev->vq[index].kick_fd,VU_WATCH_IN , + vu_kick_cb, (void *)(long )index); + }DPRINT ("Waiting for kicks on fd: %d for vq: %d\n" , dev->vq[index].kick_fd, index); }
@@ -1069,8 +1085,13 @@ void vu_set_queue_handler(VuDev *dev, VuVirtq *vq, vq->handler = handler;if (vq->kick_fd >=0 ) {if (handler) { - dev->set_watch (dev, vq->kick_fd,VU_WATCH_IN , - vu_kick_cb, (void *)(long )qidx); +if (dev->set_watch_packed_data) { + dev->set_watch_packed_data (dev, vq->kick_fd,VU_WATCH_IN , + dev->iface->kick_callback, (void *)(long )qidx); + }else { + dev->set_watch (dev, vq->kick_fd,VU_WATCH_IN , + vu_kick_cb, (void *)(long )qidx); + } }else { dev->remove_watch (dev, vq->kick_fd); }
@@ -1596,6 +1617,12 @@ vu_deinit(VuDev *dev) }if (vq->kick_fd !=-1 ) { +/* remove watch for kick_fd +* When client process is running in gdb and +* quit command is run in gdb, QEMU will still dispatch the event +* which will cause segment fault in the callback function +*/ + dev->remove_watch (dev, vq->kick_fd);close (vq->kick_fd); vq->kick_fd =-1 ; }
@@ -1647,10 +1674,9 @@ vu_init(VuDev *dev, const VuDevIface *iface) {uint16_t i; -assert (max_queues >0 );assert (socket >=0 ); -assert (set_watch); +/* assert(set_watch); */ assert (remove_watch);assert (iface);assert (panic);
@@ -1682,6 +1708,22 @@ vu_init(VuDev *dev,return true; } +bool +vu_init_packed_data (VuDev *dev, +uint16_t max_queues, +int socket, +vu_panic_cb panic, +vu_set_watch_cb_packed_data set_watch_packed_data, +vu_remove_watch_cb remove_watch, +const VuDevIface *iface) +{ +if (vu_init (dev, max_queues, socket, panic,NULL , remove_watch, iface)) { + dev->set_watch_packed_data = set_watch_packed_data; +return true; + } +return false; +} +VuVirtq *vu_get_queue (VuDev *dev,int qidx) {
contrib/libvhost-user/libvhost-user.h +34 -2
@@ -34,6 +34,9 @@ typedef enum VhostSetConfigType {VHOST_SET_CONFIG_TYPE_MIGRATION =1 , } VhostSetConfigType; + +#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64) +/* * Maximum size of virtio device config space */
@@ -200,6 +203,7 @@ typedef uint64_t (*vu_get_features_cb) (VuDev *dev);typedef void (*vu_set_features_cb ) (VuDev *dev,uint64_t features);typedef int (*vu_process_msg_cb ) (VuDev *dev,VhostUserMsg *vmsg,int *do_reply); +typedef bool (*vu_read_msg_cb ) (VuDev *dev,int sock,VhostUserMsg *vmsg);typedef void (*vu_queue_set_started_cb ) (VuDev *dev,int qidx,bool started);typedef bool (*vu_queue_is_processed_in_order_cb ) (VuDev *dev,int qidx);typedef int (*vu_get_config_cb ) (VuDev *dev,uint8_t *config,uint32_t len);
@@ -207,6 +211,15 @@ typedef int (*vu_set_config_cb) (VuDev *dev, const uint8_t *data, uint32_t offset, uint32_t size, uint32_t flags); +typedef struct vu_watch_cb_data { +long index; +VuDev *vu_dev; +}vu_watch_cb_data ; +typedef void (*vu_watch_cb_packed_data ) (void *packed_data); + +typedef void (*vu_set_watch_cb_packed_data ) (VuDev *dev,int fd,int condition, +vu_watch_cb_packed_data cb,void *data); + typedef struct VuDevIface { /* called by VHOST_USER_GET_FEATURES to get the features bitmask */ vu_get_features_cb get_features;
@@ -220,8 +233,11 @@ typedef struct VuDevIface {/* process_msg is called for each vhost-user message received */ /* skip libvhost-user processing if return value != 0 */ vu_process_msg_cb process_msg; +vu_read_msg_cb read_msg; +vu_watch_cb_packed_data kick_callback;/* tells when queues can be processed */ vu_queue_set_started_cb queue_set_started; +/* * If the queue is processed in order, in which case it will be * resumed to vring.used->idx. This can help to support resuming
@@ -366,7 +382,8 @@ struct VuDev {/* @set_watch: add or update the given fd to the watch set, * call cb when condition is met */ vu_set_watch_cb set_watch; - +/* AIO dispatch will only one data pointer to callback function */ +vu_set_watch_cb_packed_data set_watch_packed_data;/* @remove_watch: remove the given fd from the watch set */ vu_remove_watch_cb remove_watch;
@@ -398,7 +415,7 @@ typedef struct VuVirtqElement {* @remove_watch: a remove_watch callback * @iface: a VuDevIface structure with vhost-user device callbacks * -* Intializes a VuDev vhost-user context. +* Initializes a VuDev vhost-user context. * * Returns: true on success, false on failure. **/
@@ -411,6 +428,21 @@ bool vu_init(VuDev *dev, const VuDevIface *iface); +/** +* vu_init_packed_data: +* Same as vu_init except for set_watch_packed_data which will pack +* two parameters into a struct thus QEMU aio_dispatch can pass the +* required data to callback function. +* +* Returns: true on success, false on failure. +**/ +bool vu_init_packed_data (VuDev *dev, +uint16_t max_queues, +int socket, +vu_panic_cb panic, +vu_set_watch_cb_packed_data set_watch_packed_data, +vu_remove_watch_cb remove_watch, +const VuDevIface *iface);/** * vu_deinit: * @dev: a VuDev context
--
2.24.1
PATCH 3/5 a standone-alone tool to directly share disk image file via vhost-user protocol
In reply to Coiby Xuvhost-user-blk can have played as vhost-user backend but it only supports raw file and don't support VIRTIO_BLK_T_DISCARD and VIRTIO_BLK_T_WRITE_ZEROES operations on raw file (ioctl(fd, BLKDISCARD) is only valid for real block device).
--- qemu-vu.c | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 qemu-vu.c
qemu-vu.c new file +264 -0
@@ -0,0 +1,264 @@ +/* +* Copyright (C) 2020 Coiby Xu <[email protected]> +* +* Vhost-user-blk device backend +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; under version 2 of the License. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, see <http://www.gnu.org/licenses/>. +*/ + +#include "qemu/osdep.h" +#include <getopt.h> +#include <libgen.h> +#include "block/vhost-user.h" +#include "qemu-common.h" +#include "qapi/error.h" +#include "qemu/cutils.h" +#include "sysemu/block-backend.h" +#include "block/block_int.h" +#include "qemu/main-loop.h" +#include "qemu/module.h" +#include "qemu/option.h" +#include "qemu/error-report.h" +#include "qapi/qmp/qdict.h" +#include "qapi/qmp/qstring.h" +#include "qom/object_interfaces.h" +#include "io/net-listener.h" +#include "qemu-version.h" + +#define QEMU_VU_OPT_CACHE 256 + +#define QEMU_VU_OPT_AIO 257 + +static char *srcpath; + +static void usage (const char *name) +{ + (printf) ( +"Usage: %s [OPTIONS] FILE\n" +" or: %s -L [OPTIONS]\n" +"QEMU Vhost-user Server Utility\n" +"\n" +" -h, --help display this help and exit\n" +" -V, --version output version information and exit\n" +"\n" +"Connection properties:\n" +" -k, --socket=PATH path to the unix socket\n" +"\n" +"General purpose options:\n" +" -e, -- exit-panic When the panic callback is called, the program\n" +" will exit. Useful for make check-qtest.\n" +"\n" +"Block device options:\n" +" -f, --format=FORMAT set image format (raw, qcow2, ...)\n" +" -r, --read-only export read-only\n" +" -n, --nocache disable host cache\n" +" --cache=MODE set cache mode (none, writeback, ...)\n" +" --aio=MODE set AIO mode (native or threads)\n" +"\n" +QEMU_HELP_BOTTOM "\n" + , name, name); +} + +static void version (const char *name) +{ +printf ( +"%s " QEMU_FULL_VERSION "\n" +"Written by Coiby Xu, based on qemu-nbd by Anthony Liguori\n" +"\n" +QEMU_COPYRIGHT "\n" +"This is free software; see the source for copying conditions. There is NO\n" +"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n" + , name); +} + +static VubDev *vub_device; + +static void vus_shutdown (void ) +{ +job_cancel_sync_all (); +bdrv_close_all (); +vub_free (vub_device, false); +} + +int main (int argc,char **argv) +{ +BlockBackend *blk; +BlockDriverState *bs; +bool readonly = false; +char *sockpath =NULL ; +int64_t fd_size; +const char *sopt ="hVrnvek:f:" ; +struct option lopt[] = { + {"help" , no_argument,NULL ,'h' }, + {"version" , no_argument,NULL ,'V' }, + {"exit-panic" , no_argument,NULL ,'e' }, + {"socket" , required_argument,NULL ,'k' }, + {"read-only" , no_argument,NULL ,'r' }, + {"nocache" , no_argument,NULL ,'n' }, + {"cache" , required_argument,NULL ,QEMU_VU_OPT_CACHE }, + {"aio" , required_argument,NULL ,QEMU_VU_OPT_AIO }, + {"format" , required_argument,NULL ,'f' }, + {NULL ,0 ,NULL ,0 } + }; +int ch; +int opt_ind =0 ; +int flags =BDRV_O_RDWR ; +bool seen_cache = false; +bool seen_aio = false; +const char *fmt =NULL ; +Error *local_err =NULL ; +QDict *options =NULL ; +bool writethrough = true; +bool exit_panic = false; + +error_init (argv[0 ]); + +module_call_init (MODULE_INIT_QOM ); +qemu_init_exec_dir (argv[0 ]); + +while ((ch =getopt_long (argc, argv, sopt, lopt, &opt_ind)) !=-1 ) { +switch (ch) { +case 'e' : + exit_panic = true; +break ; +case 'n' : + optarg = (char *)"none" ; +/* fallthrough */ +case QEMU_VU_OPT_CACHE : +if (seen_cache) { +error_report ("-n and --cache can only be specified once" ); +exit (EXIT_FAILURE ); + } + seen_cache = true; +if (bdrv_parse_cache_mode (optarg, &flags, &writethrough) ==-1 ) { +error_report ("Invalid cache mode `%s'" , optarg); +exit (EXIT_FAILURE ); + } +break ; +case QEMU_VU_OPT_AIO : +if (seen_aio) { +error_report ("--aio can only be specified once" ); +exit (EXIT_FAILURE ); + } + seen_aio = true; +if (!strcmp (optarg,"native" )) { + flags |=BDRV_O_NATIVE_AIO ; + }else if (!strcmp (optarg,"threads" )) { +/* this is the default */ + }else { +error_report ("invalid aio mode `%s'" , optarg); +exit (EXIT_FAILURE ); + } +break ; +case 'r' : + readonly = true; + flags &= ~BDRV_O_RDWR ; +break ; +case 'k' : + sockpath = optarg; +if (sockpath[0 ] !='/' ) { +error_report ("socket path must be absolute" ); +exit (EXIT_FAILURE ); + } +break ; +case 'f' : + fmt = optarg; +break ; +case 'V' : +version (argv[0 ]); +exit (0 ); +break ; +case 'h' : +usage (argv[0 ]); +exit (0 ); +break ; +case '?' : +error_report ("Try `%s --help' for more information." , argv[0 ]); +exit (EXIT_FAILURE ); + } + } + +if ((argc - optind) !=1 ) { +error_report ("Invalid number of arguments" ); +error_printf ("Try `%s --help' for more information.\n" , argv[0 ]); +exit (EXIT_FAILURE ); + } + +if (qemu_init_main_loop (&local_err)) { +error_report_err (local_err); +exit (EXIT_FAILURE ); + } +bdrv_init (); + + srcpath = argv[optind]; +if (fmt) { + options =qdict_new (); +qdict_put_str (options,"driver" , fmt); + } + blk =blk_new_open (srcpath,NULL , options, flags, &local_err); + +if (!blk) { +error_reportf_err (local_err,"Failed to blk_new_open '%s': " , + argv[optind]); +exit (EXIT_FAILURE ); + } + bs =blk_bs (blk); + +blk_set_enable_write_cache (blk, !writethrough); + + fd_size =blk_getlength (blk); +if (fd_size <0 ) { +error_report ("Failed to determine the image length: %s" , +strerror (-fd_size)); +exit (EXIT_FAILURE ); + } + +AioContext *ctx =bdrv_get_aio_context (bs); +bdrv_invalidate_cache (bs,NULL ); + + vub_device =g_new0 (VubDev,1 ); + vub_device->unix_socket =g_strdup (sockpath); + vub_device->writable = !readonly; + vub_device->blkcfg.wce = !writethrough; + vub_device->backend = blk; + vub_device->ctx = ctx; +vub_initialize_config (bs, &vub_device->blkcfg); + vub_device->listener =qio_net_listener_new (); + vub_device->exit_panic = exit_panic; + +qio_net_listener_set_name (vub_device->listener, +"vhost-user-backend-listener" ); + +SocketAddress *addr =g_new0 (SocketAddress,1 ); + addr->u.q_unix.path = (char *) sockpath; + addr->type =SOCKET_ADDRESS_TYPE_UNIX ; +Error **errp =NULL ; +if (qio_net_listener_open_sync (vub_device->listener, addr,1 , errp) <0 ) { + goto error; + } + +qio_net_listener_set_client_func (vub_device->listener, + vub_accept, + vub_device, +NULL ); + +QTAILQ_INIT (&vub_device->clients); + +do { +main_loop_wait (false); + }while (!vub_device->exit_panic || !vub_device->close); + + error: +vus_shutdown (); +exit (EXIT_SUCCESS ); +}
--
2.24.1
PATCH 4/5 new qTest case for the vhost-user-blk device backend
In reply to Coiby XuThis test case has the same tests as tests/virtio-blk-test.c except for tests have block_resize.
--- tests/libqos/vhost-user-blk.c | 125 ++++++ tests/libqos/vhost-user-blk.h | 44 +++ tests/vhost-user-blk-test.c | 691 ++++++++++++++++++++++++++++++++++ 3 files changed, 860 insertions(+) create mode 100644 tests/libqos/vhost-user-blk.c create mode 100644 tests/libqos/vhost-user-blk.h create mode 100644 tests/vhost-user-blk-test.c
tests/libqos/vhost-user-blk.c new file +125 -0
@@ -0,0 +1,125 @@ +/* +* libqos driver framework +* +* Copyright (c) 2018 Emanuele Giuseppe Esposito <[email protected]> +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Lesser General Public +* License version 2 as published by the Free Software Foundation. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Lesser General Public License for more details. +* +* You should have received a copy of the GNU Lesser General Public +* License along with this library; if not, see <http://www.gnu.org/licenses/> +*/ + +#include "qemu/osdep.h" +#include "libqtest.h" +#include "qemu/module.h" +#include "standard-headers/linux/virtio_blk.h" +#include "libqos/qgraph.h" +#include "libqos/vhost-user-blk.h" + +#define PCI_SLOT 0x04 +#define PCI_FN 0x00 + +/* virtio-blk-device */ +static void *qvhost_user_blk_get_driver (QVhostUserBlk *v_blk, +const char *interface) +{ +if (!g_strcmp0 (interface,"vhost-user-blk" )) { +return v_blk; + } +if (!g_strcmp0 (interface,"virtio" )) { +return v_blk->vdev; + } + +fprintf (stderr,"%s not present in vhost-user-blk-device\n" , interface); +g_assert_not_reached (); +} + +static void *qvhost_user_blk_device_get_driver (void *object, +const char *interface) +{ +QVhostUserBlkDevice *v_blk = object; +return qvhost_user_blk_get_driver (&v_blk->blk, interface); +} + +static void *vhost_user_blk_device_create (void *virtio_dev, +QGuestAllocator *t_alloc, +void *addr) +{ +QVhostUserBlkDevice *vhost_user_blk =g_new0 (QVhostUserBlkDevice,1 ); +QVhostUserBlk *interface = &vhost_user_blk->blk; + + interface->vdev = virtio_dev; + + vhost_user_blk->obj.get_driver = qvhost_user_blk_device_get_driver; + +return &vhost_user_blk->obj; +} + +/* virtio-blk-pci */ +static void *qvhost_user_blk_pci_get_driver (void *object,const char *interface) +{ +QVhostUserBlkPCI *v_blk = object; +if (!g_strcmp0 (interface,"pci-device" )) { +return v_blk->pci_vdev.pdev; + } +return qvhost_user_blk_get_driver (&v_blk->blk, interface); +} + +static void *vhost_user_blk_pci_create (void *pci_bus,QGuestAllocator *t_alloc, +void *addr) +{ +QVhostUserBlkPCI *vhost_user_blk =g_new0 (QVhostUserBlkPCI,1 ); +QVhostUserBlk *interface = &vhost_user_blk->blk; +QOSGraphObject *obj = &vhost_user_blk->pci_vdev.obj; + +virtio_pci_init (&vhost_user_blk->pci_vdev, pci_bus, addr); + interface->vdev = &vhost_user_blk->pci_vdev.vdev; + + g_assert_cmphex(interface->vdev->device_type, ==, VIRTIO_ID_BLOCK); + + obj->get_driver = qvhost_user_blk_pci_get_driver; + +return obj; +} + +static void vhost_user_blk_register_nodes (void ) +{ +/* FIXME: every test using these two nodes needs to setup a +* -drive,id=drive0 otherwise QEMU is not going to start. +* Therefore, we do not include "produces" edge for virtio +* and pci-device yet. +*/ + +char *arg =g_strdup_printf ("id=drv0,chardev=char1,addr=%x.%x" , +PCI_SLOT ,PCI_FN ); + +QPCIAddress addr = { + .devfn =QPCI_DEVFN (PCI_SLOT ,PCI_FN ), + }; + +QOSGraphEdgeOptions opts = { }; + +/* virtio-blk-device */ +/** opts.extra_device_opts = "drive=drive0"; */ +qos_node_create_driver ("vhost-user-blk-device" , vhost_user_blk_device_create); +qos_node_consumes ("vhost-user-blk-device" ,"virtio-bus" , &opts); +qos_node_produces ("vhost-user-blk-device" ,"vhost-user-blk" ); + +/* virtio-blk-pci */ + opts.extra_device_opts = arg; +add_qpci_address (&opts, &addr); +qos_node_create_driver ("vhost-user-blk-pci" , vhost_user_blk_pci_create); +qos_node_consumes ("vhost-user-blk-pci" ,"pci-bus" , &opts); +qos_node_produces ("vhost-user-blk-pci" ,"vhost-user-blk" ); + +g_free (arg); +} + +libqos_init (vhost_user_blk_register_nodes);
tests/libqos/vhost-user-blk.h new file +44 -0
44 lines not on this page · raw message
tests/vhost-user-blk-test.c new file +691 -0
691 lines not on this page · raw message
--
2.24.1
PATCH 5/5 building configuration files changes
In reply to Coiby Xu--- Makefile | 1 + Makefile.objs | 2 +- Makefile.target | 1 + configure | 2 +- tests/Makefile.include | 5 ++++- 5 files changed, 8 insertions(+), 3 deletions(-)
Makefile +1 -0
@@ -558,6 +558,7 @@ qemu-img.o: qemu-img-cmds.h qemu-img$(EXESUF): qemu-img.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS) qemu-nbd$(EXESUF): qemu-nbd.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS) +qemu-vu$(EXESUF): qemu-vu.o blockdev-vu.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS) libvhost-user.a qemu-io$(EXESUF): qemu-io.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS) qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o $(COMMON_LDADDS)
Makefile.objs +1 -1
@@ -42,7 +42,7 @@ io-obj-y = io/ # single QEMU executable should support all CPUs and machines. ifeq ($(CONFIG_SOFTMMU),y) -common-obj-y = blockdev.o blockdev-nbd.o block/ +common-obj-y = blockdev.o blockdev-nbd.o blockdev-vu.o block/ common-obj-y += bootdevice.o iothread.o common-obj-y += dump/ common-obj-y += job-qmp.o
Makefile.target +1 -0
@@ -159,6 +159,7 @@ obj-y += monitor/ obj-y += qapi/ obj-y += memory.o obj-y += memory_mapping.o +obj-y += ../contrib/libvhost-user/libvhost-user.o obj-y += migration/ram.o LIBS := $(libs_softmmu) $(LIBS)
configure +1 -1
@@ -6165,7 +6165,7 @@ fi tools="" if test "$want_tools" = "yes" ; then - tools="qemu-img\$(EXESUF) qemu-io\$(EXESUF) qemu-edid\$(EXESUF) $tools" + tools="qemu-img\$(EXESUF) qemu-vu\$(EXESUF) qemu-io\$(EXESUF) qemu-edid\$(EXESUF) $tools" if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then tools="qemu-nbd\$(EXESUF) $tools" fi
tests/Makefile.include +4 -1
@@ -746,6 +746,7 @@ qos-test-obj-y += tests/libqos/virtio.o qos-test-obj-$(CONFIG_VIRTFS) += tests/libqos/virtio-9p.o qos-test-obj-y += tests/libqos/virtio-balloon.o qos-test-obj-y += tests/libqos/virtio-blk.o +qos-test-obj-y += tests/libqos/vhost-user-blk.o qos-test-obj-y += tests/libqos/virtio-mmio.o qos-test-obj-y += tests/libqos/virtio-net.o qos-test-obj-y += tests/libqos/virtio-pci.o
@@ -788,6 +789,7 @@ qos-test-obj-$(CONFIG_VHOST_NET_USER) += tests/vhost-user-test.o $(chardev-obj-y qos-test-obj-y += tests/virtio-test.o qos-test-obj-$(CONFIG_VIRTFS) += tests/virtio-9p-test.o qos-test-obj-y += tests/virtio-blk-test.o +qos-test-obj-y += tests/vhost-user-blk-test.o qos-test-obj-y += tests/virtio-net-test.o qos-test-obj-y += tests/virtio-rng-test.o qos-test-obj-y += tests/virtio-scsi-test.o
@@ -935,7 +937,8 @@ endef $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: %-softmmu/all $(check-qtest-y) $(call do_test_human,$(check-qtest-$*-y) $(check-qtest-generic-y), \ QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \ - QTEST_QEMU_IMG=qemu-img$(EXESUF)) + QTEST_QEMU_IMG=./qemu-img$(EXESUF) \ + QTEST_QEMU_VU_BINARY=./qemu-vu$(EXESUF)) check-unit: $(check-unit-y) $(call do_test_human, $^)
--
2.24.1
Patchew URL: https://patchew.org/QEMU/[email protected]/
Patchew URL: https://patchew.org/QEMU/[email protected]/
Hi,
This series failed the docker-mingw@fedora build test. Please find the testing commands and their output below. If you have Docker installed, you can probably reproduce it locally.
=== TEST SCRIPT BEGIN ===
#! /bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-mingw@fedora J=14 NETWORK=1
=== TEST SCRIPT END ===
CC chardev/char-ringbuf.o
CC chardev/char-serial.o
CC chardev/char-socket.o
/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.c:26:10: fatal error: sys/socket.h: No such file or directory
#include <sys/socket.h>
^~~~~~~~~~~~~~compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69: contrib/libvhost-user/libvhost-user.o] Error 1
make: *** Waiting for unfinished jobs....
In file included from /tmp/qemu-test/src/include/block/vhost-user.h:3,
from /tmp/qemu-test/src/blockdev-vu.c:2:/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.h:20:10: fatal error: sys/poll.h: No such file or directory
#include <sys/poll.h>
^~~~~~~~~~~~
compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69: blockdev-vu.o] Error 1
In file included from /tmp/qemu-test/src/include/block/vhost-user.h:3,
from /tmp/qemu-test/src/qemu-vu.c:22:
/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.h:20:10: fatal error: sys/poll.h: No such file or directory
#include <sys/poll.h>
^~~~~~~~~~~~compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69: qemu-vu.o] Error 1
In file included from /tmp/qemu-test/src/contrib/libvhost-user/libvhost-user-glib.h:19,
from /tmp/qemu-test/src/contrib/libvhost-user/libvhost-user-glib.c:17:/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.h:20:10: fatal error: sys/poll.h: No such file or directory
#include <sys/poll.h>
^~~~~~~~~~~~compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69: contrib/libvhost-user/libvhost-user-glib.o] Error 1 Traceback (most recent call last):
File "./tests/docker/docker.py", line 662, in <module>
sys.exit(main())
---
raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run', '--label', 'com.qemu.instance.uuid=5a27a9b1f81649c88588b26500a2460e', '-u', '1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e', 'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14', '-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache', '-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-z0m6vw52/src/docker-src.2020-01-13-00.19.17.20483:/var/tmp/qemu:z,ro', 'qemu:fedora', '/var/tmp/qemu/run', 'test-mingw']' returned non-zero exit status 2. filter=--filter=label=com.qemu.instance.uuid=5a27a9b1f81649c88588b26500a2460e
make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-z0m6vw52/src'
make: *** [docker-run-test-mingw@fedora] Error 2
real 2m1.128s
user 0m6.706s
The full log is available at
http://patchew.org/logs/[email protected]/testing.docker-mingw@fedora/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]
It fails MinGW because libvhost-user depends on eventfd which is a Linux-specific feature. Now fixed in v2.
Quoted 89 lines · On Mon, Jan 13, 2020 at 1:21 PM <[email protected]> wrote:
Patchew URL:
https://patchew.org/QEMU/[email protected]/Hi,
This series failed the docker-mingw@fedora build test. Please find the testing commands and
their output below. If you have Docker installed, you can probably reproduce it
locally.=== TEST SCRIPT BEGIN ===
#!/bin/bash
export ARCH=x86_64
make docker-image-fedora V=1 NETWORK=1
time make docker-test-mingw@fedora J=14 NETWORK=1
=== TEST SCRIPT END ===CC chardev/char-ringbuf.o CC chardev/char-serial.o CC chardev/char-socket.o /tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.c:26:10: fatal error: sys/socket.h: No such file or directory #include <sys/socket.h> ^~~~~~~~~~~~~~compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69:contrib/libvhost-user/libvhost-user.o] Error 1
make: *** Waiting for unfinished jobs....
In file included from/tmp/qemu-test/src/include/block/vhost-user.h:3,
from/tmp/qemu-test/src/blockdev-vu.c:2:/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.h:20:10: fatal
error:sys/poll.h: No such file or directory#include <sys/poll.h> ^~~~~~~~~~~~ compilation terminated. make: *** [/tmp/qemu-test/src/rules.mak:69: blockdev-vu.o] Error 1 In file included from /tmp/qemu-test/src/include/block/vhost-user.h:3, from /tmp/qemu-test/src/qemu-vu.c:22: /tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.h:20:10: fatal error: sys/poll.h: No such file or directory #include <sys/poll.h> ^~~~~~~~~~~~compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69: qemu-vu.o] Error 1 In file included from/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user-glib.h:19,
from/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user-glib.c:17:/tmp/qemu-test/src/contrib/libvhost-user/libvhost-user.h:20:10: fatal
error:sys/poll.h: No such file or directory#include <sys/poll.h> ^~~~~~~~~~~~compilation terminated.
make: *** [/tmp/qemu-test/src/rules.mak:69:contrib/libvhost-user/libvhost-user-glib.o] Error 1 Traceback (most recent call last):
File "./tests/docker/docker.py", line 662, in <module>sys.exit(main())---
raiseCalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['sudo', '-n', 'docker', 'run',
'--label', 'com.qemu.instance.uuid=5a27a9b1f81649c88588b26500a2460e', '-u',
'1003', '--security-opt', 'seccomp=unconfined', '--rm', '-e',
'TARGET_LIST=', '-e', 'EXTRA_CONFIGURE_OPTS=', '-e', 'V=', '-e', 'J=14',
'-e', 'DEBUG=', '-e', 'SHOW_ENV=', '-e', 'CCACHE_DIR=/var/tmp/ccache',
'-v', '/home/patchew2/.cache/qemu-docker-ccache:/var/tmp/ccache:z', '-v', '/var/tmp/patchew-tester-tmp-z0m6vw52/src/docker-src.2020-01-13-00.19.17.20483:/var/tmp/qemu:z,ro', 'qemu:fedora', '/var/tmp/qemu/run', 'test-mingw']' returned non-zero exit
status 2.filter=--filter=label=com.qemu.instance.uuid=5a27a9b1f81649c88588b26500a2460e make[1]: *** [docker-run] Error 1
make[1]: Leaving directory `/var/tmp/patchew-tester-tmp-z0m6vw52/src'
make: *** [docker-run-test-mingw@fedora] Error 2real 2m1.128s
user 0m6.706sThe full log is available at
http://patchew.org/logs/[email protected]/testing.docker-mingw@fedora/?type=message
.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to [email protected]
Signature · 2 lines
*Best regards,* *Coiby*