Skip to contentSkip to conversation

vhost-user block device backend implementation

PATCH Superseded Coiby Xu 3 participants 19 messages Jan 14, 2020 – Feb 20, 2020 Kbuild

v2 of 10 Superseded Latest is v10 →

lore
Share & tools
lore

Coiby Xu

author
#0 lore raw

v2 cover vhost-user block device backend implementation

v2:
* Only enable this feauture for Linux for eventfd is a Linux-specific feature

This patch series is an implementation of vhost-user-backend server, thanks to Stefan's guidance after reviewing the draft version and the instructions on https://wiki.qemu.org/Google_Summer_of_Code_2019#vhost-user-blk_device_backend,

Vhost-user-backend server is a UserCreatable object and can be started using object_add,

(qemu) object_add vhost-user-server,id=ID,unix_socket=/tmp/vhost-user-blk_vhost.socket,name=DRIVE_NAME,writable=off (qemu) object_del ID

or appending the "-object" option when starting QEMU,

  $ -object vhost-user-server,id=disk,unix_socket=/tmp/vhost-user-blk_vhost.socket,name=disk,writable=off

Then vhost-user client can connect to the server backend. For example, QEMU could act as a client,

  $ -m 256 -object memory-backend-memfd,id=mem,size=256M,share=on -numa node,memdev=mem -chardev socket,id=char1,path=/tmp/vhost-user-blk_vhost.socket -device vhost-user-blk-pci,id=blk0,chardev=char1

And guest OS could access this vhost-user block device after mouting it.

patches are against commit 035eed4c0d257c905a556fa0f4865a0c077b4e7f.

Coiby Xu (5):
vhost-user block device backend
extend libvhost to support IOThread
a standone-alone tool to directly share disk image file via vhost-user
protocol
new qTest case for the vhost-user-blk device backend building configuration files changes

 Makefile                              |    1 +
 Makefile.objs                         |    2 +-
 Makefile.target                       |    1 +
 blockdev-vu.c                         | 1008 +++++++++++++++++++++++++
 configure                             |    2 +-
 contrib/libvhost-user/libvhost-user.c |   64 +-
 contrib/libvhost-user/libvhost-user.h |   36 +-
 include/block/vhost-user.h            |   46 ++
 qemu-vu.c                             |  264 +++++++
 tests/Makefile.include                |    5 +-
 tests/libqos/vhost-user-blk.c         |  125 +++
 tests/libqos/vhost-user-blk.h         |   44 ++
 tests/vhost-user-blk-test.c           |  691 +++++++++++++++++
 vl.c                                  |    4 +
 14 files changed, 2277 insertions(+), 16 deletions(-)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h
 create mode 100644 qemu-vu.c
 create mode 100644 tests/libqos/vhost-user-blk.c
 create mode 100644 tests/libqos/vhost-user-blk.h
 create mode 100644 tests/vhost-user-blk-test.c

--
2.24.1

author
#1 lore raw

PATCH v2 1/5 vhost-user block device backend

By making use of libvhost, multiple block device drives can be exported and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.

---
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h
blockdev-vu.c new file +1008 -0
new file mode 100644
index 0000000000..45f0bb43a7
@@ -0,0 +1,1008 @@
+#include "qemu/osdep.h"
+#include "block/vhost-user.h"
+#include "qapi/error.h"
+#include "qapi/qapi-types-sockets.h"
+#include "qapi/qapi-commands-block.h"
+
+#include "sysemu/block-backend.h"
+#include "qemu/main-loop.h"
+
+#include "qemu/units.h"
+
+#include "block/block.h"
+
+#include "qom/object_interfaces.h"
+
+#include <sys/eventfd.h>
+
+#include "hw/qdev-properties.h"
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};
+
+struct virtio_blk_inhdr {
+    unsigned char status;
+};
+
+
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;
+    int64_t sector_num;
+    size_t size;
+    struct virtio_blk_inhdr *in;
+    struct virtio_blk_outhdr out;
+    VuClient *client;
+    struct VuVirtq *vq;
+} VubReq;
+
+static void
+remove_watch(VuDev *vu_dev, int fd)
+{
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+
+    client = container_of(vu_dev, VuClient, parent);
+    aio_set_fd_handler(client->blk->ctx, fd, false, NULL, NULL, NULL, NULL);
+}
+
+static void close_client(VuClient *client)
+{
+    vu_deinit(&client->parent);
+    /** g_source_destroy(vub_device->parent.src); */
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->closed = true;
+
+}
+
+static void vub_panic_cb(VuDev *vu_dev, const char *buf)
+{
+    if (buf) {
+        g_warning("vu_panic: %s", buf);
+    }
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    if (client->blk->exit_panic) {
+        client->blk->close = true;
+    }
+    if (!client->closed) {
+        close_client(client);
+    }
+}
+
+
+static void vub_req_complete(VubReq *req)
+{
+    VuDev *vu_dev = &req->client->parent;
+
+    /* IO size with 1 extra status byte */
+    vu_queue_push(vu_dev, req->vq, req->elem,
+                  req->size + 1);
+    vu_queue_notify(vu_dev, req->vq);
+
+    if (req->elem) {
+        free(req->elem);
+    }
+
+    g_free(req);
+}
+
+
+
+static int
+vub_discard_write_zeroes(VubReq *req, struct iovec *iov, uint32_t iovcnt,
+                         uint32_t type)
+{
+    struct virtio_blk_discard_write_zeroes *desc;
+    ssize_t size;
+    void *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (size != sizeof(*desc)) {
+        fprintf(stderr, "Invalid size %ld, expect %ld\n", size, sizeof(*desc));
+        return -1;
+    }
+    buf = g_new0(char, size);
+
+    iov_to_buf_full(iov, iovcnt, 0, buf, size);
+
+
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    VubDev *vdev_blk;
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+    desc = (struct virtio_blk_discard_write_zeroes *)buf;
+    uint64_t range[2] = { le64toh(desc->sector) << 9,
+                          le32toh(desc->num_sectors) << 9 };
+    if (type == VIRTIO_BLK_T_DISCARD) {
+        if (blk_pdiscard(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    } else if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
+        if (blk_pwrite_zeroes(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    }
+    #endif
+
+    g_free(buf);
+    return -1;
+}
+
+
+static void
+vub_flush(VubReq *req)
+{
+    VuClient *client = req->client;
+    blk_co_flush(client->blk->backend);
+}
+
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    void *iobuf;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static int blk_prw(BlockBackend *blk, QEMUIOVector *qiov, int64_t offset,
+                   CoroutineEntry co_entry, BdrvRequestFlags flags)
+{
+
+    BlkRwCo rwco = {
+        .blk    = blk,
+        .offset = offset,
+        .iobuf  = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        co_entry(&rwco);
+    } else {
+        Coroutine *co = qemu_coroutine_create(co_entry, &rwco);
+        bdrv_coroutine_enter(blk_bs(blk), co);
+        BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
+    }
+
+    return rwco.ret;
+}
+
+
+static ssize_t
+vub_rwv(VubReq *req, struct iovec *iov,
+        uint32_t iovcnt,
+        CoroutineEntry co_entry)
+{
+    VuClient *client = req->client;
+    ssize_t rc;
+
+    if (!iovcnt) {
+        fprintf(stderr, "Invalid Read/Write IOV count\n");
+        return -1;
+    }
+
+    int64_t offset = req->sector_num * 512;
+    QEMUIOVector *qiov = g_new0(QEMUIOVector, 1);
+    qemu_iovec_init_external(qiov, iov, iovcnt);
+    rc = blk_prw(client->blk->backend, qiov, offset, co_entry, 0);
+
+    req->size = iov_size(iov, iovcnt);
+    if (rc < 0) {
+        fprintf(stderr, "%s, Sector %"PRIu64", Size %lu failed with %s\n",
+                client->blk->name, req->sector_num, req->size,
+                strerror(errno));
+        return -1;
+    }
+
+    return rc;
+}
+
+static int vub_virtio_process_req(VuClient *client,
+                                     VuVirtq *vq)
+{
+    VuDev *vu_dev = &client->parent;
+    VuVirtqElement *elem;
+    uint32_t type;
+    VubReq *req;
+
+    elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement) + sizeof(VubReq));
+    if (!elem) {
+        return -1;
+    }
+
+    struct iovec *in_iov = elem->in_sg;
+    struct iovec *out_iov = elem->out_sg;
+    unsigned in_num = elem->in_num;
+    unsigned out_num = elem->out_num;
+    /* refer to hw/block/virtio_blk.c */
+    if (elem->out_num < 1 || elem->in_num < 1) {
+        fprintf(stderr, "virtio-blk request missing headers\n");
+        free(elem);
+        return -1;
+    }
+
+    req = g_new0(VubReq, 1);
+    req->client = client;
+    req->vq = vq;
+    req->elem = elem;
+
+    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req->out,
+                            sizeof(req->out)) != sizeof(req->out))) {
+        fprintf(stderr, "virtio-blk request outhdr too short");
+        goto err;
+    }
+
+    iov_discard_front(&out_iov, &out_num, sizeof(req->out));
+
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+        fprintf(stderr, "virtio-blk request inhdr too short");
+        goto err;
+    }
+
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in = (void *)in_iov[in_num - 1].iov_base
+              + in_iov[in_num - 1].iov_len
+              - sizeof(struct virtio_blk_inhdr);
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+
+    type = le32toh(req->out.type);
+    switch (type & ~VIRTIO_BLK_T_BARRIER) {
+    case VIRTIO_BLK_T_IN:
+    case VIRTIO_BLK_T_OUT: {
+        ssize_t ret = 0;
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = le64toh(req->out.sector);
+        if (is_write) {
+            ret = vub_rwv(req, out_iov, out_num, blk_write_entry);
+        } else {
+            ret = vub_rwv(req, in_iov, in_num, blk_read_entry);
+        }
+        if (ret >= 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
+        vub_flush(req);
+        req->in->status = VIRTIO_BLK_S_OK;
+        vub_req_complete(req);
+        break;
+    case VIRTIO_BLK_T_GET_ID: {
+        size_t size = MIN(iov_size(&elem->in_sg[0], in_num),
+                          VIRTIO_BLK_ID_BYTES);
+        snprintf(elem->in_sg[0].iov_base, size, "%s", "vhost_user_blk");
+        req->in->status = VIRTIO_BLK_S_OK;
+        req->size = elem->in_sg[0].iov_len;
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_DISCARD:
+    case VIRTIO_BLK_T_WRITE_ZEROES: {
+        int rc;
+        rc = vub_discard_write_zeroes(req, &elem->out_sg[1], out_num, type);
+        if (rc == 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    default:
+        req->in->status = VIRTIO_BLK_S_UNSUPP;
+        vub_req_complete(req);
+        break;
+    }
+
+    return 0;
+
+err:
+    free(elem);
+    g_free(req);
+    return -1;
+}
+
+
+static void vub_process_vq(VuDev *vu_dev, int idx)
+{
+    VuClient *client;
+    VuVirtq *vq;
+    int ret;
+
+    client = container_of(vu_dev, VuClient, parent);
+    assert(client);
+
+    vq = vu_get_queue(vu_dev, idx);
+    assert(vq);
+
+    while (1) {
+        ret = vub_virtio_process_req(client, vq);
+        if (ret) {
+            break;
+        }
+    }
+}
+
+
+static void vub_queue_set_started(VuDev *vu_dev, int idx, bool started)
+{
+    VuVirtq *vq;
+
+    assert(vu_dev);
+
+    vq = vu_get_queue(vu_dev, idx);
+    vu_set_queue_handler(vu_dev, vq, started ? vub_process_vq : NULL);
+}
+
+static uint64_t
+vub_get_features(VuDev *dev)
+{
+    uint64_t features;
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+
+    features = 1ull << VIRTIO_BLK_F_SIZE_MAX |
+               1ull << VIRTIO_BLK_F_SEG_MAX |
+               1ull << VIRTIO_BLK_F_TOPOLOGY |
+               1ull << VIRTIO_BLK_F_BLK_SIZE |
+               1ull << VIRTIO_BLK_F_FLUSH |
+               #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+               1ull << VIRTIO_BLK_F_DISCARD |
+               1ull << VIRTIO_BLK_F_WRITE_ZEROES |
+               #endif
+               1ull << VIRTIO_BLK_F_CONFIG_WCE |
+               1ull << VIRTIO_F_VERSION_1 |
+               1ull << VIRTIO_RING_F_INDIRECT_DESC |
+               1ull << VIRTIO_RING_F_EVENT_IDX |
+               1ull << VHOST_USER_F_PROTOCOL_FEATURES;
+
+    if (!vdev_blk->writable) {
+        features |= 1ull << VIRTIO_BLK_F_RO;
+    }
+
+    return features;
+}
+
+static uint64_t
+vub_get_protocol_features(VuDev *dev)
+{
+    return 1ull << VHOST_USER_PROTOCOL_F_CONFIG |
+           1ull << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD;
+}
+
+static int
+vub_get_config(VuDev *vu_dev, uint8_t *config, uint32_t len)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    memcpy(config, &vdev_blk->blkcfg, len);
+
+    return 0;
+}
+
+static int
+vub_set_config(VuDev *vu_dev, const uint8_t *data,
+               uint32_t offset, uint32_t size, uint32_t flags)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    uint8_t wce;
+
+    /* don't support live migration */
+    if (flags != VHOST_SET_CONFIG_TYPE_MASTER) {
+        return -1;
+    }
+
+
+    if (offset != offsetof(struct virtio_blk_config, wce) ||
+        size != 1) {
+        return -1;
+    }
+
+    wce = *data;
+    if (wce == vdev_blk->blkcfg.wce) {
+        /* Do nothing as same with old configuration */
+        return 0;
+    }
+
+    vdev_blk->blkcfg.wce = wce;
+    blk_set_enable_write_cache(vdev_blk->backend, true);
+    return 0;
+}
+
+
+/*
+ * When the client disconnects, it send a VHOST_USER_NONE request
+ * and vu_process_message will simple call exit which cause the VM
+ * to exit abruptly.
+ * To avoid this issue,  process VHOST_USER_NONE request ahead
+ * of vu_process_message.
+ *
+ */
+static int vub_process_msg(VuDev *dev, VhostUserMsg *vmsg, int *do_reply)
+{
+    if (vmsg->request == VHOST_USER_NONE) {
+        dev->panic(dev, "disconnect");
+        return true;
+    }
+    return false;
+}
+
+static void
+vmsg_close_fds(VhostUserMsg *vmsg)
+{
+    int i;
+    for (i = 0; i < vmsg->fd_num; i++) {
+        close(vmsg->fds[i]);
+    }
+}
+
+static bool
+vu_message_read_co(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+    char buffer[100];
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    QIOChannel *ioc = client->ioc;
+    do {
+        rc = recvmsg(conn_fd, &msg, 0);
+        if (rc < 0) {
+            if (errno == EAGAIN) {
+                if (qemu_in_coroutine()) {
+                    qio_channel_yield(ioc, G_IO_IN);
+                } else {
+                    qio_channel_wait(ioc, G_IO_IN);
+                }
+                continue;
+            } else if (errno == EINTR) {
+                continue;
+            }
+        }
+        break;
+    } while (true);
+
+    if (rc < 0) {
+        sprintf(buffer, "Error while recvmsg: %s", strerror(errno));
+        vub_panic_cb(vu_dev, buffer);
+        return false;
+    }
+
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);
+
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
+
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        sprintf(buffer,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        vub_panic_cb(vu_dev, buffer);
+        goto fail;
+    }
+
+    if (vmsg->size) {
+        do {
+            rc = read(conn_fd, &vmsg->payload, vmsg->size);
+            if (rc < 0) {
+                if (errno == EAGAIN) {
+                    if (qemu_in_coroutine()) {
+                        qio_channel_yield(ioc, G_IO_IN);
+                    } else {
+                        qio_channel_wait(ioc, G_IO_IN);
+                    }
+                    continue;
+                } else if (errno == EINTR) {
+                    continue;
+                }
+            }
+            break;
+        } while (true);
+
+        if (rc <= 0) {
+            sprintf(buffer, "Error while reading: %s", strerror(errno));
+            vub_panic_cb(vu_dev, buffer);
+            goto fail;
+        }
+
+        assert(rc == vmsg->size);
+    }
+
+    return true;
+
+fail:
+    vmsg_close_fds(vmsg);
+
+    return false;
+}
+
+static void vub_kick_cb(void *opaque)
+{
+    vu_watch_cb_data *data = (vu_watch_cb_data *) opaque;
+    int index = data->index;
+    VuDev *dev = data->vu_dev;
+    VuVirtq *vq = &dev->vq[index];
+    int sock = vq->kick_fd;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        char buffer[100];
+        sprintf(buffer, "kick eventfd_read(): %s", strerror(errno));
+        vub_panic_cb(dev, buffer);
+        g_free(data);
+        dev->remove_watch(dev, dev->vq[index].kick_fd);
+    } else {
+        if (vq->handler) {
+            vq->handler(dev, index);
+        }
+    }
+}
+
+static const VuDevIface vub_iface = {
+    .get_features = vub_get_features,
+    .queue_set_started = vub_queue_set_started,
+    .get_protocol_features = vub_get_protocol_features,
+    .get_config = vub_get_config,
+    .set_config = vub_set_config,
+    .process_msg = vub_process_msg,
+    .read_msg = vu_message_read_co,
+    .kick_callback = vub_kick_cb,
+};
+
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM)
+{
+    if (!vub_dev) {
+        return;
+    }
+
+    blk_unref(vub_dev->backend);
+    g_free(vub_dev->name);
+    g_free(vub_dev->unix_socket);
+
+    if (vub_dev->next.tqe_circ.tql_prev) {
+        /*
+         * if vub_dev->next.tqe_circ.tql_prev = null,
+         * vub_dev hasn't been inserted into the queue and
+         * vub_free is called by obj->instance_finalize.
+         */
+        QTAILQ_REMOVE(&vub_devs, vub_dev, next);
+    }
+    /*
+     * Needn't to free vub_dev if called by QOM
+     * because QOM will do the clean-up work.
+     */
+    if (!called_by_QOM) {
+        g_free(vub_dev);
+    }
+}
+
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+    VuClient *client = opaque;
+
+    while (!client->closed) {
+        vu_dispatch(&client->parent);
+    }
+
+    QTAILQ_REMOVE(&client->blk->clients, client, next);
+
+}
+
+static void vu_client_start(VuClient *client)
+{
+    Coroutine *co = qemu_coroutine_create(vu_client_trip, client);
+    qemu_coroutine_enter(co);
+}
+
+
+G_STATIC_ASSERT((int)G_IO_IN == (int)VU_WATCH_IN);
+G_STATIC_ASSERT((int)G_IO_OUT == (int)VU_WATCH_OUT);
+G_STATIC_ASSERT((int)G_IO_PRI == (int)VU_WATCH_PRI);
+G_STATIC_ASSERT((int)G_IO_ERR == (int)VU_WATCH_ERR);
+G_STATIC_ASSERT((int)G_IO_HUP == (int)VU_WATCH_HUP);
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+          vu_watch_cb_packed_data cb, void *pvt)
+{
+    /*
+     * since aio_dispatch can only pass one user data pointer to the
+     * callback function, pack VuDev, pvt into a struct
+     */
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+    g_assert(cb);
+    client = container_of(vu_dev, VuClient, parent);
+    vu_watch_cb_data *cb_data = g_new0(vu_watch_cb_data, 1);
+    cb_data->index = (intptr_t) pvt;
+    cb_data->vu_dev = vu_dev;
+    aio_set_fd_handler(client->blk->ctx, fd, false, (void *) cb,
+                       NULL, NULL, cb_data);
+}
+
+
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);
+
+    if (!vu_init_packed_data(&client->parent, VHOST_USER_BLK_MAX_QUEUES,
+                             sioc->fd, vub_panic_cb, set_watch,
+                             remove_watch, &vub_iface)) {
+        fprintf(stderr, "Failed to initialized libvhost-user\n");
+        g_free(client);
+        return;
+    }
+
+    client->blk = vub_device;
+    client->refcount = 1;
+    client->sioc = sioc;
+    /*
+     * increase the object reference, so cioc will not freed by
+     * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))
+     */
+    object_ref(OBJECT(client->sioc));
+    qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(QIO_CHANNEL(client->sioc), false, NULL);
+    client->closed = false;
+    QTAILQ_INSERT_TAIL(&client->blk->clients, client, next);
+    vu_client_start(client);
+}
+
+
+void
+vub_initialize_config(BlockDriverState *bs, struct virtio_blk_config *config)
+{
+    config->capacity = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    config->blk_size = BDRV_SECTOR_SIZE;
+    config->size_max = 65536;
+    config->seg_max = 128 - 2;
+    config->min_io_size = 1;
+    config->opt_io_size = 1;
+    config->num_queues = 1;
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    config->max_discard_sectors = 32768;
+    config->max_discard_seg = 1;
+    config->discard_sector_alignment = config->blk_size >> 9;
+    config->max_write_zeroes_sectors = 32768;
+    config->max_write_zeroes_seg = 1;
+    #endif
+}
+
+
+static VubDev *vub_new(VubDev *vub_device, const char *name,
+                       const char *unix_socket, bool writable, Error **errp)
+{
+
+    BlockBackend *blk;
+
+    /*
+     * Don't allow resize while the vhost user server is running,
+     * otherwise we don't care what happens with the node.
+     */
+    uint64_t perm = BLK_PERM_CONSISTENT_READ;
+    int ret;
+
+    AioContext *ctx;
+
+    BlockDriverState *bs = bdrv_lookup_bs(name,
+                                          name,
+                                          errp);
+
+    if (!bs) {
+        error_setg(errp,
+                   "No drive with name '%s'."
+                   " Please find the list of names with "
+                   "'info block'", name);
+        return NULL;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    if (writable) {
+        perm |= BLK_PERM_WRITE;
+    }
+
+    ctx = bdrv_get_aio_context(bs);
+    aio_context_acquire(ctx);
+    bdrv_invalidate_cache(bs, NULL);
+    aio_context_release(ctx);
+
+    blk = blk_new(bdrv_get_aio_context(bs), perm,
+                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
+                  BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(blk, bs, errp);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+
+    blk_set_enable_write_cache(blk, false);
+
+    blk_set_allow_aio_context_change(blk, true);
+
+
+    vub_device->name = g_strdup(name);
+    vub_device->unix_socket = g_strdup(unix_socket);
+    vub_device->writable = writable;
+    vub_device->blkcfg.wce = 0;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    return vub_device;
+
+fail:
+    blk_unref(blk);
+    return NULL;
+}
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM)
+{
+    if (!vub_device) {
+        return;
+    }
+
+    VuClient *client, *next;
+    QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) {
+        if (!client->closed) {
+            close_client(client);
+        }
+    }
+
+    if (vub_device->listener) {
+        qio_net_listener_disconnect(vub_device->listener);
+        object_unref(OBJECT(vub_device->listener));
+    }
+    vub_free(vub_device, called_by_QOM);
+
+}
+
+
+VubDev *vub_dev_find(const char *name)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(name, vub_device->name) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+
+static VubDev *vub_dev_find_by_unix_socket(const char *unix_socket)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(unix_socket, vub_device->unix_socket) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+static void vhost_user_server_start(VubDev *vub_device, const char *unix_socket,
+                                    const char *name, bool writable,
+                                    Error **errp)
+{
+
+    if (vub_dev_find(name) || vub_dev_find_by_unix_socket(unix_socket)) {
+        error_setg(errp, "Vhost user server with name '%s' or "
+                "with socket_path '%s' has already been started",
+                name, unix_socket);
+        return;
+    }
+
+
+    if (!vub_new(vub_device, name, unix_socket, writable, errp)) {
+        return;
+    }
+
+
+    vub_device->listener = qio_net_listener_new();
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);
+    addr->u.q_unix.path = (char *) unix_socket;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp) < 0) {
+        goto error;
+    }
+
+
+    QTAILQ_INSERT_TAIL(&vub_devs, vub_device, next);
+    QTAILQ_INIT(&vub_device->clients);
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    return;
+
+ error:
+    vub_free(vub_device, false);
+}
+
+static void vu_set_block_name(Object *obj, const char *value,
+                                           Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->name) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+
+    vus->name = g_strdup(value);
+}
+
+static char *vu_get_block_name(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    return g_strdup(vus->name);
+}
+
+
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);
+}
+
+static char *vu_get_unix_socket(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return g_strdup(vus->unix_socket);
+}
+
+static bool vu_get_block_writable(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return vus->writable;
+}
+
+static void vu_set_block_writable(Object *obj, bool value, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+
+    vus->writable = value;
+}
+
+static void vhost_user_server_instance_init(Object *obj)
+{
+
+    object_property_add_bool(obj, "writable",
+                            vu_get_block_writable,
+                            vu_set_block_writable, NULL);
+
+    object_property_add_str(obj, "name",
+                            vu_get_block_name,
+                            vu_set_block_name, NULL);
+
+    object_property_add_str(obj, "unix_socket",
+                            vu_get_unix_socket,
+                            vu_set_unix_socket, NULL);
+
+}
+
+static void vhost_user_server_instance_finalize(Object *obj)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    vhost_user_server_free(vus, true);
+    /* object_del shouldn't free this object struct */
+    obj->free = NULL;
+}
+
+static const TypeInfo vhost_user_server_info = {
+    .name = TYPE_VHOST_USER_SERVER,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(VuDev),
+    .instance_init = vhost_user_server_instance_init,
+    .instance_finalize = vhost_user_server_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        {TYPE_USER_CREATABLE},
+        {}
+    },
+};
+
+static void vhost_user_server_register_types(void)
+{
+    type_register_static(&vhost_user_server_info);
+}
+
+type_init(vhost_user_server_register_types)
+
include/block/vhost-user.h new file +46 -0

46 lines not on this page · raw message

vl.c +4 -0
index 86474a55c9..72ac506342 100644
@@ -2553,6 +2553,10 @@ static bool object_create_initial(const char *type, QemuOpts *opts)
     }
 #endif
 
+    /* Reason: vhost-user-server property "name" */
+    if (g_str_equal(type, "vhost-user-server")) {
+        return false;
+    }
     /*
      * Reason: filter-* property "netdev" etc.
      */

--
2.24.1

#2 lore raw

Re: [PATCH v2 1/5] vhost-user block device backend

Quoted 5 lines · Coiby Xu

By making use of libvhost, multiple block device drives can be exported and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.

---
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++

This file contains both vhost-user-blk code and generic vhost-user protocol code. Please split them into two files:
1. backends/vhost-user-blk-server.c
2. util/vhost-user-server.c

(As I read and understand the code better I may have better suggestions about where to put these source files and how to name them.)

Quoted 23 lines
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h
blockdev-vu.c new file +11 -0
new file mode 100644
index 0000000000..45f0bb43a7
@@ -0,0 +1,1008 @@
+#include "qemu/osdep.h"
+#include "block/vhost-user.h"
+#include "qapi/error.h"
+#include "qapi/qapi-types-sockets.h"
+#include "qapi/qapi-commands-block.h"
+
+#include "sysemu/block-backend.h"
+#include "qemu/main-loop.h"
+
+#include "qemu/units.h"
+

blockdev-vu.c line 21

Quoted 10 lines
blockdev-vu.c new file +10 -0
+#include "block/block.h"
+
+#include "qom/object_interfaces.h"
+
+#include <sys/eventfd.h>
+
+#include "hw/qdev-properties.h"
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};

The number of queues is hardcoded to 1 so this constant can be removed for now.

blockdev-vu.c line 28

Quoted 7 lines
blockdev-vu.c new file +7 -0
+
+struct virtio_blk_inhdr {
+    unsigned char status;
+};
+
+
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);

I'm not sure if this list will be necessary. See my comments about replacing the "name" property with Object's built-in "id" property.

Quoted 74 lines
blockdev-vu.c new file +74 -0
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;
+    int64_t sector_num;
+    size_t size;
+    struct virtio_blk_inhdr *in;
+    struct virtio_blk_outhdr out;
+    VuClient *client;
+    struct VuVirtq *vq;
+} VubReq;
+
+static void
+remove_watch(VuDev *vu_dev, int fd)
+{
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+
+    client = container_of(vu_dev, VuClient, parent);
+    aio_set_fd_handler(client->blk->ctx, fd, false, NULL, NULL, NULL, NULL);
+}
+
+static void close_client(VuClient *client)
+{
+    vu_deinit(&client->parent);
+    /** g_source_destroy(vub_device->parent.src); */
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->closed = true;
+
+}
+
+static void vub_panic_cb(VuDev *vu_dev, const char *buf)
+{
+    if (buf) {
+        g_warning("vu_panic: %s", buf);
+    }
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    if (client->blk->exit_panic) {
+        client->blk->close = true;
+    }
+    if (!client->closed) {
+        close_client(client);
+    }
+}
+
+
+static void vub_req_complete(VubReq *req)
+{
+    VuDev *vu_dev = &req->client->parent;
+
+    /* IO size with 1 extra status byte */
+    vu_queue_push(vu_dev, req->vq, req->elem,
+                  req->size + 1);
+    vu_queue_notify(vu_dev, req->vq);
+
+    if (req->elem) {
+        free(req->elem);
+    }
+
+    g_free(req);
+}
+
+
+
+static int
+vub_discard_write_zeroes(VubReq *req, struct iovec *iov, uint32_t iovcnt,
+                         uint32_t type)
+{
+    struct virtio_blk_discard_write_zeroes *desc;
+    ssize_t size;

blockdev-vu.c line 112

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    void *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (size != sizeof(*desc)) {
+        fprintf(stderr, "Invalid size %ld, expect %ld\n", size, sizeof(*desc));
+        return -1;
+    }
+    buf = g_new0(char, size);
+
+    iov_to_buf_full(iov, iovcnt, 0, buf, size);

Simpler:

struct virtio_blk_discard_write_zeroes desc;

  if (unlikely(iov_to_buf(iov, iovcnt, 0, &desc, sizeof(desc)) !=
               sizeof(desc)) {
      /* TODO handle error */
  }

No heap allocation. One variable (desc) instead of two (desc and buf).

Quoted 23 lines
blockdev-vu.c new file +23 -0
+
+
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    VubDev *vdev_blk;
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+    desc = (struct virtio_blk_discard_write_zeroes *)buf;
+    uint64_t range[2] = { le64toh(desc->sector) << 9,
+                          le32toh(desc->num_sectors) << 9 };
+    if (type == VIRTIO_BLK_T_DISCARD) {
+        if (blk_pdiscard(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    } else if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
+        if (blk_pwrite_zeroes(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    }
+    #endif
+
+    g_free(buf);

blockdev-vu.c line 145

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    return -1;
+}
+
+
+static void
+vub_flush(VubReq *req)
+{
+    VuClient *client = req->client;
+    blk_co_flush(client->blk->backend);
+}

Any function that calls a *_co_*() function must be marked coroutine_fn:

static void coroutine_fn
vub_flush(VubReq *req)

Quoted 25 lines
blockdev-vu.c new file +25 -0
+
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    void *iobuf;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;

blockdev-vu.c line 180

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static int blk_prw(BlockBackend *blk, QEMUIOVector *qiov, int64_t offset,
+                   CoroutineEntry co_entry, BdrvRequestFlags flags)

Why is block/block-backend.c:blk_prw() duplicated here?

Quoted 294 lines
blockdev-vu.c new file +294 -0
+{
+
+    BlkRwCo rwco = {
+        .blk    = blk,
+        .offset = offset,
+        .iobuf  = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        co_entry(&rwco);
+    } else {
+        Coroutine *co = qemu_coroutine_create(co_entry, &rwco);
+        bdrv_coroutine_enter(blk_bs(blk), co);
+        BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
+    }
+
+    return rwco.ret;
+}
+
+
+static ssize_t
+vub_rwv(VubReq *req, struct iovec *iov,
+        uint32_t iovcnt,
+        CoroutineEntry co_entry)
+{
+    VuClient *client = req->client;
+    ssize_t rc;
+
+    if (!iovcnt) {
+        fprintf(stderr, "Invalid Read/Write IOV count\n");
+        return -1;
+    }
+
+    int64_t offset = req->sector_num * 512;
+    QEMUIOVector *qiov = g_new0(QEMUIOVector, 1);
+    qemu_iovec_init_external(qiov, iov, iovcnt);
+    rc = blk_prw(client->blk->backend, qiov, offset, co_entry, 0);
+
+    req->size = iov_size(iov, iovcnt);
+    if (rc < 0) {
+        fprintf(stderr, "%s, Sector %"PRIu64", Size %lu failed with %s\n",
+                client->blk->name, req->sector_num, req->size,
+                strerror(errno));
+        return -1;
+    }
+
+    return rc;
+}
+
+static int vub_virtio_process_req(VuClient *client,
+                                     VuVirtq *vq)
+{
+    VuDev *vu_dev = &client->parent;
+    VuVirtqElement *elem;
+    uint32_t type;
+    VubReq *req;
+
+    elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement) + sizeof(VubReq));
+    if (!elem) {
+        return -1;
+    }
+
+    struct iovec *in_iov = elem->in_sg;
+    struct iovec *out_iov = elem->out_sg;
+    unsigned in_num = elem->in_num;
+    unsigned out_num = elem->out_num;
+    /* refer to hw/block/virtio_blk.c */
+    if (elem->out_num < 1 || elem->in_num < 1) {
+        fprintf(stderr, "virtio-blk request missing headers\n");
+        free(elem);
+        return -1;
+    }
+
+    req = g_new0(VubReq, 1);
+    req->client = client;
+    req->vq = vq;
+    req->elem = elem;
+
+    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req->out,
+                            sizeof(req->out)) != sizeof(req->out))) {
+        fprintf(stderr, "virtio-blk request outhdr too short");
+        goto err;
+    }
+
+    iov_discard_front(&out_iov, &out_num, sizeof(req->out));
+
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+        fprintf(stderr, "virtio-blk request inhdr too short");
+        goto err;
+    }
+
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in = (void *)in_iov[in_num - 1].iov_base
+              + in_iov[in_num - 1].iov_len
+              - sizeof(struct virtio_blk_inhdr);
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+
+    type = le32toh(req->out.type);
+    switch (type & ~VIRTIO_BLK_T_BARRIER) {
+    case VIRTIO_BLK_T_IN:
+    case VIRTIO_BLK_T_OUT: {
+        ssize_t ret = 0;
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = le64toh(req->out.sector);
+        if (is_write) {
+            ret = vub_rwv(req, out_iov, out_num, blk_write_entry);
+        } else {
+            ret = vub_rwv(req, in_iov, in_num, blk_read_entry);
+        }
+        if (ret >= 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
+        vub_flush(req);
+        req->in->status = VIRTIO_BLK_S_OK;
+        vub_req_complete(req);
+        break;
+    case VIRTIO_BLK_T_GET_ID: {
+        size_t size = MIN(iov_size(&elem->in_sg[0], in_num),
+                          VIRTIO_BLK_ID_BYTES);
+        snprintf(elem->in_sg[0].iov_base, size, "%s", "vhost_user_blk");
+        req->in->status = VIRTIO_BLK_S_OK;
+        req->size = elem->in_sg[0].iov_len;
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_DISCARD:
+    case VIRTIO_BLK_T_WRITE_ZEROES: {
+        int rc;
+        rc = vub_discard_write_zeroes(req, &elem->out_sg[1], out_num, type);
+        if (rc == 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    default:
+        req->in->status = VIRTIO_BLK_S_UNSUPP;
+        vub_req_complete(req);
+        break;
+    }
+
+    return 0;
+
+err:
+    free(elem);
+    g_free(req);
+    return -1;
+}
+
+
+static void vub_process_vq(VuDev *vu_dev, int idx)
+{
+    VuClient *client;
+    VuVirtq *vq;
+    int ret;
+
+    client = container_of(vu_dev, VuClient, parent);
+    assert(client);
+
+    vq = vu_get_queue(vu_dev, idx);
+    assert(vq);
+
+    while (1) {
+        ret = vub_virtio_process_req(client, vq);
+        if (ret) {
+            break;
+        }
+    }
+}
+
+
+static void vub_queue_set_started(VuDev *vu_dev, int idx, bool started)
+{
+    VuVirtq *vq;
+
+    assert(vu_dev);
+
+    vq = vu_get_queue(vu_dev, idx);
+    vu_set_queue_handler(vu_dev, vq, started ? vub_process_vq : NULL);
+}
+
+static uint64_t
+vub_get_features(VuDev *dev)
+{
+    uint64_t features;
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+
+    features = 1ull << VIRTIO_BLK_F_SIZE_MAX |
+               1ull << VIRTIO_BLK_F_SEG_MAX |
+               1ull << VIRTIO_BLK_F_TOPOLOGY |
+               1ull << VIRTIO_BLK_F_BLK_SIZE |
+               1ull << VIRTIO_BLK_F_FLUSH |
+               #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+               1ull << VIRTIO_BLK_F_DISCARD |
+               1ull << VIRTIO_BLK_F_WRITE_ZEROES |
+               #endif
+               1ull << VIRTIO_BLK_F_CONFIG_WCE |
+               1ull << VIRTIO_F_VERSION_1 |
+               1ull << VIRTIO_RING_F_INDIRECT_DESC |
+               1ull << VIRTIO_RING_F_EVENT_IDX |
+               1ull << VHOST_USER_F_PROTOCOL_FEATURES;
+
+    if (!vdev_blk->writable) {
+        features |= 1ull << VIRTIO_BLK_F_RO;
+    }
+
+    return features;
+}
+
+static uint64_t
+vub_get_protocol_features(VuDev *dev)
+{
+    return 1ull << VHOST_USER_PROTOCOL_F_CONFIG |
+           1ull << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD;
+}
+
+static int
+vub_get_config(VuDev *vu_dev, uint8_t *config, uint32_t len)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    memcpy(config, &vdev_blk->blkcfg, len);
+
+    return 0;
+}
+
+static int
+vub_set_config(VuDev *vu_dev, const uint8_t *data,
+               uint32_t offset, uint32_t size, uint32_t flags)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    uint8_t wce;
+
+    /* don't support live migration */
+    if (flags != VHOST_SET_CONFIG_TYPE_MASTER) {
+        return -1;
+    }
+
+
+    if (offset != offsetof(struct virtio_blk_config, wce) ||
+        size != 1) {
+        return -1;
+    }
+
+    wce = *data;
+    if (wce == vdev_blk->blkcfg.wce) {
+        /* Do nothing as same with old configuration */
+        return 0;
+    }
+
+    vdev_blk->blkcfg.wce = wce;
+    blk_set_enable_write_cache(vdev_blk->backend, true);
+    return 0;
+}
+
+
+/*
+ * When the client disconnects, it send a VHOST_USER_NONE request
+ * and vu_process_message will simple call exit which cause the VM
+ * to exit abruptly.
+ * To avoid this issue,  process VHOST_USER_NONE request ahead
+ * of vu_process_message.
+ *
+ */
+static int vub_process_msg(VuDev *dev, VhostUserMsg *vmsg, int *do_reply)
+{
+    if (vmsg->request == VHOST_USER_NONE) {
+        dev->panic(dev, "disconnect");
+        return true;
+    }
+    return false;
+}
+
+static void

blockdev-vu.c line 484

Quoted 10 lines
blockdev-vu.c new file +10 -0
+vmsg_close_fds(VhostUserMsg *vmsg)
+{
+    int i;
+    for (i = 0; i < vmsg->fd_num; i++) {
+        close(vmsg->fds[i]);
+    }
+}
+
+static bool
+vu_message_read_co(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)

Why is this function called *_co()? It appears to be callable from outside coroutine context too.

Quoted 113 lines
blockdev-vu.c new file +113 -0
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+    char buffer[100];
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    QIOChannel *ioc = client->ioc;
+    do {
+        rc = recvmsg(conn_fd, &msg, 0);
+        if (rc < 0) {
+            if (errno == EAGAIN) {
+                if (qemu_in_coroutine()) {
+                    qio_channel_yield(ioc, G_IO_IN);
+                } else {
+                    qio_channel_wait(ioc, G_IO_IN);
+                }
+                continue;
+            } else if (errno == EINTR) {
+                continue;
+            }
+        }
+        break;
+    } while (true);
+
+    if (rc < 0) {
+        sprintf(buffer, "Error while recvmsg: %s", strerror(errno));
+        vub_panic_cb(vu_dev, buffer);
+        return false;
+    }
+
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);
+
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
+
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        sprintf(buffer,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        vub_panic_cb(vu_dev, buffer);
+        goto fail;
+    }
+
+    if (vmsg->size) {
+        do {
+            rc = read(conn_fd, &vmsg->payload, vmsg->size);
+            if (rc < 0) {
+                if (errno == EAGAIN) {
+                    if (qemu_in_coroutine()) {
+                        qio_channel_yield(ioc, G_IO_IN);
+                    } else {
+                        qio_channel_wait(ioc, G_IO_IN);
+                    }
+                    continue;
+                } else if (errno == EINTR) {
+                    continue;
+                }
+            }
+            break;
+        } while (true);
+
+        if (rc <= 0) {
+            sprintf(buffer, "Error while reading: %s", strerror(errno));
+            vub_panic_cb(vu_dev, buffer);
+            goto fail;
+        }
+
+        assert(rc == vmsg->size);
+    }
+
+    return true;
+
+fail:
+    vmsg_close_fds(vmsg);
+
+    return false;
+}
+
+static void vub_kick_cb(void *opaque)
+{
+    vu_watch_cb_data *data = (vu_watch_cb_data *) opaque;
+    int index = data->index;
+    VuDev *dev = data->vu_dev;
+    VuVirtq *vq = &dev->vq[index];
+    int sock = vq->kick_fd;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        char buffer[100];

blockdev-vu.c line 607

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        sprintf(buffer, "kick eventfd_read(): %s", strerror(errno));
+        vub_panic_cb(dev, buffer);
+        g_free(data);
+        dev->remove_watch(dev, dev->vq[index].kick_fd);
+    } else {
+        if (vq->handler) {
+            vq->handler(dev, index);
+        }
+    }
+}

This is a generic vhost-user function that is not specific to vhost-user-blk. It should be in a different source file.

Quoted 47 lines
blockdev-vu.c new file +47 -0
+
+static const VuDevIface vub_iface = {
+    .get_features = vub_get_features,
+    .queue_set_started = vub_queue_set_started,
+    .get_protocol_features = vub_get_protocol_features,
+    .get_config = vub_get_config,
+    .set_config = vub_set_config,
+    .process_msg = vub_process_msg,
+    .read_msg = vu_message_read_co,
+    .kick_callback = vub_kick_cb,
+};
+
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM)
+{
+    if (!vub_dev) {
+        return;
+    }
+
+    blk_unref(vub_dev->backend);
+    g_free(vub_dev->name);
+    g_free(vub_dev->unix_socket);
+
+    if (vub_dev->next.tqe_circ.tql_prev) {
+        /*
+         * if vub_dev->next.tqe_circ.tql_prev = null,
+         * vub_dev hasn't been inserted into the queue and
+         * vub_free is called by obj->instance_finalize.
+         */
+        QTAILQ_REMOVE(&vub_devs, vub_dev, next);
+    }
+    /*
+     * Needn't to free vub_dev if called by QOM
+     * because QOM will do the clean-up work.
+     */
+    if (!called_by_QOM) {
+        g_free(vub_dev);
+    }
+}
+
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+    VuClient *client = opaque;
+
+    while (!client->closed) {
+        vu_dispatch(&client->parent);
+    }

blockdev-vu.c line 664

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    QTAILQ_REMOVE(&client->blk->clients, client, next);
+
+}
+
+static void vu_client_start(VuClient *client)
+{
+    Coroutine *co = qemu_coroutine_create(vu_client_trip, client);
+    qemu_coroutine_enter(co);
+}

What is the coroutine and threading model here? This launches vu_client_trip() in the current thread's event loop but vu_dispatch() performs blocking I/O and will prevent the event loop from continuing.

Quoted 26 lines
blockdev-vu.c new file +26 -0
+
+
+G_STATIC_ASSERT((int)G_IO_IN == (int)VU_WATCH_IN);
+G_STATIC_ASSERT((int)G_IO_OUT == (int)VU_WATCH_OUT);
+G_STATIC_ASSERT((int)G_IO_PRI == (int)VU_WATCH_PRI);
+G_STATIC_ASSERT((int)G_IO_ERR == (int)VU_WATCH_ERR);
+G_STATIC_ASSERT((int)G_IO_HUP == (int)VU_WATCH_HUP);
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+          vu_watch_cb_packed_data cb, void *pvt)
+{
+    /*
+     * since aio_dispatch can only pass one user data pointer to the
+     * callback function, pack VuDev, pvt into a struct
+     */
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+    g_assert(cb);
+    client = container_of(vu_dev, VuClient, parent);
+    vu_watch_cb_data *cb_data = g_new0(vu_watch_cb_data, 1);
+    cb_data->index = (intptr_t) pvt;
+    cb_data->vu_dev = vu_dev;
+    aio_set_fd_handler(client->blk->ctx, fd, false, (void *) cb,

blockdev-vu.c line 700

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                       NULL, NULL, cb_data);
+}
+
+
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);

Is it helpful to have a separate VuClient struct even though only a single vhost-user client can be connected at a time? It may be simpler to keep connection state directly in VubDev.

Quoted 24 lines
blockdev-vu.c new file +24 -0
+
+    if (!vu_init_packed_data(&client->parent, VHOST_USER_BLK_MAX_QUEUES,
+                             sioc->fd, vub_panic_cb, set_watch,
+                             remove_watch, &vub_iface)) {
+        fprintf(stderr, "Failed to initialized libvhost-user\n");
+        g_free(client);
+        return;
+    }
+
+    client->blk = vub_device;
+    client->refcount = 1;
+    client->sioc = sioc;
+    /*
+     * increase the object reference, so cioc will not freed by
+     * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))
+     */
+    object_ref(OBJECT(client->sioc));
+    qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(QIO_CHANNEL(client->sioc), false, NULL);
+    client->closed = false;

blockdev-vu.c line 734

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    QTAILQ_INSERT_TAIL(&client->blk->clients, client, next);
+    vu_client_start(client);
+}
+
+
+void
+vub_initialize_config(BlockDriverState *bs, struct virtio_blk_config *config)
+{
+    config->capacity = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    config->blk_size = BDRV_SECTOR_SIZE;

At some point this should become a configuration parameter because using 512B sectors on a native 4K sector image file results in poor performance.

blockdev-vu.c line 735

Quoted 1 line
blockdev-vu.c new file +1 -0
+    config->size_max = 65536;

Why 65536? hw/block/virtio-blk.c sets it to 0.

blockdev-vu.c line 736

Quoted 1 line
blockdev-vu.c new file +1 -0
+    config->seg_max = 128 - 2;

This is okay for now but should be changed in the future.

hw/block/virtio-blk.c was recently modified to calculate seg_max based on the virtqueue size (which is where the hardcoded 128 originally came from). Applications that use large buffer sizes (128+ KB) benefit from larger virtqueue sizes and seg_max.

Quoted 5 lines
blockdev-vu.c new file +5 -0
+    config->min_io_size = 1;
+    config->opt_io_size = 1;
+    config->num_queues = 1;
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    config->max_discard_sectors = 32768;

blockdev-vu.c line 751

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    config->max_discard_seg = 1;
+    config->discard_sector_alignment = config->blk_size >> 9;
+    config->max_write_zeroes_sectors = 32768;
+    config->max_write_zeroes_seg = 1;
+    #endif
+}
+
+
+static VubDev *vub_new(VubDev *vub_device, const char *name,
+                       const char *unix_socket, bool writable, Error **errp)

This function does not allocate and return a new VubDev, so vub_new() is not an accurate name for this function. The name vub_init() is clearer.

Why are name, unix_socket, and writable passed in? They are already vub_device fields.

Quoted 128 lines
blockdev-vu.c new file +128 -0
+{
+
+    BlockBackend *blk;
+
+    /*
+     * Don't allow resize while the vhost user server is running,
+     * otherwise we don't care what happens with the node.
+     */
+    uint64_t perm = BLK_PERM_CONSISTENT_READ;
+    int ret;
+
+    AioContext *ctx;
+
+    BlockDriverState *bs = bdrv_lookup_bs(name,
+                                          name,
+                                          errp);
+
+    if (!bs) {
+        error_setg(errp,
+                   "No drive with name '%s'."
+                   " Please find the list of names with "
+                   "'info block'", name);
+        return NULL;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    if (writable) {
+        perm |= BLK_PERM_WRITE;
+    }
+
+    ctx = bdrv_get_aio_context(bs);
+    aio_context_acquire(ctx);
+    bdrv_invalidate_cache(bs, NULL);
+    aio_context_release(ctx);
+
+    blk = blk_new(bdrv_get_aio_context(bs), perm,
+                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
+                  BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(blk, bs, errp);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+
+    blk_set_enable_write_cache(blk, false);
+
+    blk_set_allow_aio_context_change(blk, true);
+
+
+    vub_device->name = g_strdup(name);
+    vub_device->unix_socket = g_strdup(unix_socket);
+    vub_device->writable = writable;
+    vub_device->blkcfg.wce = 0;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    return vub_device;
+
+fail:
+    blk_unref(blk);
+    return NULL;
+}
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM)
+{
+    if (!vub_device) {
+        return;
+    }
+
+    VuClient *client, *next;
+    QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) {
+        if (!client->closed) {
+            close_client(client);
+        }
+    }
+
+    if (vub_device->listener) {
+        qio_net_listener_disconnect(vub_device->listener);
+        object_unref(OBJECT(vub_device->listener));
+    }
+    vub_free(vub_device, called_by_QOM);
+
+}
+
+
+VubDev *vub_dev_find(const char *name)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(name, vub_device->name) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+
+static VubDev *vub_dev_find_by_unix_socket(const char *unix_socket)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(unix_socket, vub_device->unix_socket) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+static void vhost_user_server_start(VubDev *vub_device, const char *unix_socket,
+                                    const char *name, bool writable,
+                                    Error **errp)
+{
+
+    if (vub_dev_find(name) || vub_dev_find_by_unix_socket(unix_socket)) {
+        error_setg(errp, "Vhost user server with name '%s' or "
+                "with socket_path '%s' has already been started",
+                name, unix_socket);
+        return;
+    }
+
+
+    if (!vub_new(vub_device, name, unix_socket, writable, errp)) {

blockdev-vu.c line 889

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        return;
+    }
+
+
+    vub_device->listener = qio_net_listener_new();
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);

addr is never freed. This variable could live on the stack instead:

  SocketAddress addr = {};
Quoted 44 lines
blockdev-vu.c new file +44 -0
+    addr->u.q_unix.path = (char *) unix_socket;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp) < 0) {
+        goto error;
+    }
+
+
+    QTAILQ_INSERT_TAIL(&vub_devs, vub_device, next);
+    QTAILQ_INIT(&vub_device->clients);
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    return;
+
+ error:
+    vub_free(vub_device, false);
+}
+
+static void vu_set_block_name(Object *obj, const char *value,
+                                           Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->name) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+
+    vus->name = g_strdup(value);
+}
+
+static char *vu_get_block_name(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    return g_strdup(vus->name);
+}
+
+
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{

blockdev-vu.c line 943

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);

Property setters should only perform input validation and store the data. Actions like creating network connections, opening files, etc should happen later in a UserCreatableClass->complete() callback.

This is necessary because vus->writable is also a property and may be set after unix_socket. The ->complete() callback is called after all setters so it can access the final values of all properties.

See iothread_class_init() and iothread_complete() for an example.

Quoted 98 lines
blockdev-vu.c new file +65 -0
+}
+
+static char *vu_get_unix_socket(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return g_strdup(vus->unix_socket);
+}
+
+static bool vu_get_block_writable(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return vus->writable;
+}
+
+static void vu_set_block_writable(Object *obj, bool value, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+
+    vus->writable = value;
+}
+
+static void vhost_user_server_instance_init(Object *obj)
+{
+
+    object_property_add_bool(obj, "writable",
+                            vu_get_block_writable,
+                            vu_set_block_writable, NULL);
+
+    object_property_add_str(obj, "name",
+                            vu_get_block_name,
+                            vu_set_block_name, NULL);
+
+    object_property_add_str(obj, "unix_socket",
+                            vu_get_unix_socket,
+                            vu_set_unix_socket, NULL);
+
+}
+
+static void vhost_user_server_instance_finalize(Object *obj)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    vhost_user_server_free(vus, true);
+    /* object_del shouldn't free this object struct */
+    obj->free = NULL;
+}
+
+static const TypeInfo vhost_user_server_info = {
+    .name = TYPE_VHOST_USER_SERVER,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(VuDev),
+    .instance_init = vhost_user_server_instance_init,
+    .instance_finalize = vhost_user_server_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        {TYPE_USER_CREATABLE},
+        {}
+    },
+};
+
+static void vhost_user_server_register_types(void)
+{
+    type_register_static(&vhost_user_server_info);
+}
+
+type_init(vhost_user_server_register_types)
+
include/block/vhost-user.h new file +27 -0
new file mode 100644
index 0000000000..ef6d695244
@@ -0,0 +1,46 @@
+#include "io/channel-socket.h"
+#include "io/net-listener.h"
+#include "contrib/libvhost-user/libvhost-user.h"
+#include "standard-headers/linux/virtio_blk.h"
+typedef struct VubDev VubDev;
+typedef struct VuClient VuClient;
+#define TYPE_VHOST_USER_SERVER "vhost-user-server"
+
+#define VHOST_USER_SERVER(obj) \
+   OBJECT_CHECK(VubDev, obj, TYPE_VHOST_USER_SERVER)
+/* vhost user block device */
+struct VubDev {
+    Object parent_obj;
+    char *name;
+    char *unix_socket;
+    bool exit_panic;
+    bool close;
+    BlockBackend *backend;
+    AioContext *ctx;
+    QIONetListener *listener;
+    QIOChannelSocket *sioc;
+    QTAILQ_HEAD(, VuClient) clients;
+    QTAILQ_ENTRY(VubDev) next;
+    struct virtio_blk_config blkcfg;
+    bool writable;
+};
+

include/block/vhost-user.h line 37

Quoted 10 lines
include/block/vhost-user.h new file +10 -0
+struct VuClient {
+    VuDev parent;
+    int refcount;
+    VubDev *blk;
+    QIOChannelSocket *sioc; /* The underlying data channel */
+    QIOChannel *ioc; /* The current I/O channel */
+    QTAILQ_ENTRY(VuClient) next;
+    bool closed;
+};
+VubDev *vub_dev_find(const char *name);

All -object instances already have an id=ID property. There is no need to declare a separate "name" property. Please look at iothread_by_id() and iothread_get_id() for examples.

Quoted 14 lines
include/block/vhost-user.h new file +9 -0
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM);
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque);
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM);
+
+void vub_initialize_config(BlockDriverState *bs,
+                           struct virtio_blk_config *config);
vl.c +0 -0
index 86474a55c9..72ac506342 100644

vl.c static bool object_create_initial(const char *type, QemuOpts *opts) · line 2559

Quoted 7 lines
vl.c +4 -0
@@ -2553,6 +2553,10 @@ static bool object_create_initial(const char *type, QemuOpts *opts)
     }
 #endif
 
+    /* Reason: vhost-user-server property "name" */
+    if (g_str_equal(type, "vhost-user-server")) {
+        return false;
+    }

I don't understand why the "name" property introduces a creation order dependency. It's just a string and has no dependency on other command-line objects. Can you explain why this change is necessary?

Quoted 6 lines
vl.c +0 -1
     /*
      * Reason: filter-* property "netdev" etc.
      */
--

2.24.1

#3 lore raw

Re: [PATCH v2 1/5] vhost-user block device backend

Am 16.01.2020 um 14:51 hat Stefan Hajnoczi geschrieben:

Quoted 18 lines

blockdev-vu.c line 943

Quoted 13 lines
blockdev-vu.c new file +13 -0
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);

Property setters should only perform input validation and store the data. Actions like creating network connections, opening files, etc should happen later in a UserCreatableClass->complete() callback.

Quoted 5 lines

This is necessary because vus->writable is also a property and may be set after unix_socket. The ->complete() callback is called after all setters so it can access the final values of all properties.

See iothread_class_init() and iothread_complete() for an example.

Ah, right, this is the correct way. I forgot about the existence of .complete(), so please ignore what I wrote.

Quoted 8 lines
Quoted 8 lines
vl.c +0 -0
index 86474a55c9..72ac506342 100644
@@ -2553,6 +2553,10 @@ static bool object_create_initial(const char *type, QemuOpts *opts)
     }
 #endif
 
Quoted 8 lines

vl.c static bool object_create_initial(const char *type, QemuOpts *opts) · line 2559

Quoted 4 lines
vl.c +4 -0
+    /* Reason: vhost-user-server property "name" */
+    if (g_str_equal(type, "vhost-user-server")) {
+        return false;
+    }

I don't understand why the "name" property introduces a creation order dependency. It's just a string and has no dependency on other command-line objects. Can you explain why this change is necessary?

I was confused at first, too, but it's just a naming problem: "name" is what points to the block device to be exported. It should really be "node-name".

Quoted 6 lines
Quoted 6 lines
vl.c +6 -0
+struct VuClient {
+    VuDev parent;
+    int refcount;
+    VubDev *blk;
+    QIOChannelSocket *sioc; /* The underlying data channel */
+    QIOChannel *ioc; /* The current I/O channel */
Quoted 8 lines

vl.c

Quoted 4 lines
vl.c +4 -0
+    QTAILQ_ENTRY(VuClient) next;
+    bool closed;
+};
+VubDev *vub_dev_find(const char *name);

All -object instances already have an id=ID property. There is no need to declare a separate "name" property. Please look at iothread_by_id() and iothread_get_id() for examples.

It's questionable to me if this function is needed at all. It is only used in vhost_user_server_start() to make sure that you don't export a node twice - but what's even the problem with exporting it twice?

Kevin

author
#4 lore raw

Re: [PATCH v2 1/5] vhost-user block device backend

In reply to Stefan Hajnoczi

Thank you for reviewing this patch! I'm already posted v3 based on your feedback.

Quoted 6 lines

blockdev-vu.c line 21

Quoted 4 lines
blockdev-vu.c new file +4 -0
+#include "hw/qdev-properties.h"
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};

The number of queues is hardcoded to 1 so this constant can be removed

for now.

Quoted 5 lines

Review

Quoted 2 lines
(quoted diff) +2 -0
+
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);

I'm not sure if this list will be necessary. See my comments about replacing the "name" property with Object's built-in "id" property.

There is no need to start vhost-user-blk-server with the same exported drive since vhost-user-blk-server can serve multiple clients simutanously. And we shoudn't started two vhost-user-blk-servers with the same unix socket path. So we need this list to avoid dupliate servers. And as pointed out by Kevin, "name" property actually means node-name which is used in v3.

Quoted 6 lines

blockdev-vu.c line 21

Quoted 4 lines
blockdev-vu.c new file +4 -0
+#include "hw/qdev-properties.h"
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};

The number of queues is hardcoded to 1 so this constant can be removed

for now.

I've set VHOST_USER_BLK_MAX_QUEUES = 1 in v3 to avoid magic number.

Quoted 8 lines
Quoted 1 line

+ config->seg_max = 128 - 2;

This is okay for now but should be changed in the future.

hw/block/virtio-blk.c was recently modified to calculate seg_max based on the virtqueue size (which is where the hardcoded 128 originally came from). Applications that use large buffer sizes (128+ KB) benefit from larger virtqueue sizes and seg_max.

I've looked at the implementation of "-device vhost-user-blk-pci,queue-size=512" regarding seg_max and found out vhost-user-blk-server doesn' have the chance to caculate seg_max based on the virtqueue size and report it back to QEMU as vhost-user client in time.

QEMU as vhost-user client will create virtqueues with the size of queue-size parameter. Later it will get the configureation including seg_max from vhost-user-blk-server by sending VHOST_USER_SET_CONFIG message and this seg_max vallue will be sent to guest OS. Guest OS will set the real size of virtqueue which will be sent to vhost-user-blk-server through VHOST_USER_SET_VRING_NUM message. After that QEMU as vhost-user client will never send VHOST_USER_SET_CONFIG again.

There could be three ways to address this issue,
1. Add seg_max_adjust and queue-size object property for vhost-user-blk device in a way similar to virtio-blk device. And QEMU as vhost-user client will ignore seg_max parameter from vhost-user-blk-server. It will caculate seg_max based on queue-size and report it to guest OS.
2. Add seg_max_adjust and queue-size object property for vhost-user-blk server and let vhost-user-blk server calculate seg_max based on queue-size
3. Let QEMU as vhost-user client tell vhost-user-blk-server its queue size by sending VHOST_USER_SET_VRING_NUM message first. When vhost-user-blk-server receive the VHOST_USER_SET_CONFIG message, it will calculate seg_max and report it back to QEMU as vhost-user client.

Which way do you is the best?

Quoted 10 lines

blockdev-vu.c line 700

Quoted 6 lines
blockdev-vu.c new file +6 -0
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);

Is it helpful to have a separate VuClient struct even though only a single vhost-user client can be connected at a time? It may be simpler to keep connection state directly in VubDev.

Currently, I don't use chardev as an object property of vhost-user-blk-server. So actually multiple clients can be connected simutaneously.

All the other suggestions have been adopted in v3. Thank you for your advice!

Quoted 1,187 lines · Stefan Hajnoczi
Quoted 5 lines · Coiby Xu

By making use of libvhost, multiple block device drives can be exported and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.

---
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++

This file contains both vhost-user-blk code and generic vhost-user protocol code. Please split them into two files:
1. backends/vhost-user-blk-server.c
2. util/vhost-user-server.c

(As I read and understand the code better I may have better suggestions about where to put these source files and how to name them.)

Quoted 23 lines
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h
blockdev-vu.c new file +11 -0
new file mode 100644
index 0000000000..45f0bb43a7
@@ -0,0 +1,1008 @@
+#include "qemu/osdep.h"
+#include "block/vhost-user.h"
+#include "qapi/error.h"
+#include "qapi/qapi-types-sockets.h"
+#include "qapi/qapi-commands-block.h"
+
+#include "sysemu/block-backend.h"
+#include "qemu/main-loop.h"
+
+#include "qemu/units.h"
+

blockdev-vu.c line 21

Quoted 10 lines
blockdev-vu.c new file +10 -0
+#include "block/block.h"
+
+#include "qom/object_interfaces.h"
+
+#include <sys/eventfd.h>
+
+#include "hw/qdev-properties.h"
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};

The number of queues is hardcoded to 1 so this constant can be removed for now.

blockdev-vu.c line 28

Quoted 7 lines
blockdev-vu.c new file +7 -0
+
+struct virtio_blk_inhdr {
+    unsigned char status;
+};
+
+
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);

I'm not sure if this list will be necessary. See my comments about replacing the "name" property with Object's built-in "id" property.

Quoted 74 lines
blockdev-vu.c new file +74 -0
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;
+    int64_t sector_num;
+    size_t size;
+    struct virtio_blk_inhdr *in;
+    struct virtio_blk_outhdr out;
+    VuClient *client;
+    struct VuVirtq *vq;
+} VubReq;
+
+static void
+remove_watch(VuDev *vu_dev, int fd)
+{
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+
+    client = container_of(vu_dev, VuClient, parent);
+    aio_set_fd_handler(client->blk->ctx, fd, false, NULL, NULL, NULL, NULL);
+}
+
+static void close_client(VuClient *client)
+{
+    vu_deinit(&client->parent);
+    /** g_source_destroy(vub_device->parent.src); */
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->closed = true;
+
+}
+
+static void vub_panic_cb(VuDev *vu_dev, const char *buf)
+{
+    if (buf) {
+        g_warning("vu_panic: %s", buf);
+    }
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    if (client->blk->exit_panic) {
+        client->blk->close = true;
+    }
+    if (!client->closed) {
+        close_client(client);
+    }
+}
+
+
+static void vub_req_complete(VubReq *req)
+{
+    VuDev *vu_dev = &req->client->parent;
+
+    /* IO size with 1 extra status byte */
+    vu_queue_push(vu_dev, req->vq, req->elem,
+                  req->size + 1);
+    vu_queue_notify(vu_dev, req->vq);
+
+    if (req->elem) {
+        free(req->elem);
+    }
+
+    g_free(req);
+}
+
+
+
+static int
+vub_discard_write_zeroes(VubReq *req, struct iovec *iov, uint32_t iovcnt,
+                         uint32_t type)
+{
+    struct virtio_blk_discard_write_zeroes *desc;
+    ssize_t size;

blockdev-vu.c line 112

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    void *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (size != sizeof(*desc)) {
+        fprintf(stderr, "Invalid size %ld, expect %ld\n", size, sizeof(*desc));
+        return -1;
+    }
+    buf = g_new0(char, size);
+
+    iov_to_buf_full(iov, iovcnt, 0, buf, size);

Simpler:

struct virtio_blk_discard_write_zeroes desc;

  if (unlikely(iov_to_buf(iov, iovcnt, 0, &desc, sizeof(desc)) !=
               sizeof(desc)) {
      /* TODO handle error */
  }

No heap allocation. One variable (desc) instead of two (desc and buf).

Quoted 23 lines
blockdev-vu.c new file +23 -0
+
+
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    VubDev *vdev_blk;
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+    desc = (struct virtio_blk_discard_write_zeroes *)buf;
+    uint64_t range[2] = { le64toh(desc->sector) << 9,
+                          le32toh(desc->num_sectors) << 9 };
+    if (type == VIRTIO_BLK_T_DISCARD) {
+        if (blk_pdiscard(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    } else if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
+        if (blk_pwrite_zeroes(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    }
+    #endif
+
+    g_free(buf);

blockdev-vu.c line 145

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    return -1;
+}
+
+
+static void
+vub_flush(VubReq *req)
+{
+    VuClient *client = req->client;
+    blk_co_flush(client->blk->backend);
+}

Any function that calls a *_co_*() function must be marked coroutine_fn:

static void coroutine_fn
vub_flush(VubReq *req)

Quoted 25 lines
blockdev-vu.c new file +25 -0
+
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    void *iobuf;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;

blockdev-vu.c line 180

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static int blk_prw(BlockBackend *blk, QEMUIOVector *qiov, int64_t offset,
+                   CoroutineEntry co_entry, BdrvRequestFlags flags)

Why is block/block-backend.c:blk_prw() duplicated here?

Quoted 294 lines
blockdev-vu.c new file +294 -0
+{
+
+    BlkRwCo rwco = {
+        .blk    = blk,
+        .offset = offset,
+        .iobuf  = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        co_entry(&rwco);
+    } else {
+        Coroutine *co = qemu_coroutine_create(co_entry, &rwco);
+        bdrv_coroutine_enter(blk_bs(blk), co);
+        BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
+    }
+
+    return rwco.ret;
+}
+
+
+static ssize_t
+vub_rwv(VubReq *req, struct iovec *iov,
+        uint32_t iovcnt,
+        CoroutineEntry co_entry)
+{
+    VuClient *client = req->client;
+    ssize_t rc;
+
+    if (!iovcnt) {
+        fprintf(stderr, "Invalid Read/Write IOV count\n");
+        return -1;
+    }
+
+    int64_t offset = req->sector_num * 512;
+    QEMUIOVector *qiov = g_new0(QEMUIOVector, 1);
+    qemu_iovec_init_external(qiov, iov, iovcnt);
+    rc = blk_prw(client->blk->backend, qiov, offset, co_entry, 0);
+
+    req->size = iov_size(iov, iovcnt);
+    if (rc < 0) {
+        fprintf(stderr, "%s, Sector %"PRIu64", Size %lu failed with %s\n",
+                client->blk->name, req->sector_num, req->size,
+                strerror(errno));
+        return -1;
+    }
+
+    return rc;
+}
+
+static int vub_virtio_process_req(VuClient *client,
+                                     VuVirtq *vq)
+{
+    VuDev *vu_dev = &client->parent;
+    VuVirtqElement *elem;
+    uint32_t type;
+    VubReq *req;
+
+    elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement) + sizeof(VubReq));
+    if (!elem) {
+        return -1;
+    }
+
+    struct iovec *in_iov = elem->in_sg;
+    struct iovec *out_iov = elem->out_sg;
+    unsigned in_num = elem->in_num;
+    unsigned out_num = elem->out_num;
+    /* refer to hw/block/virtio_blk.c */
+    if (elem->out_num < 1 || elem->in_num < 1) {
+        fprintf(stderr, "virtio-blk request missing headers\n");
+        free(elem);
+        return -1;
+    }
+
+    req = g_new0(VubReq, 1);
+    req->client = client;
+    req->vq = vq;
+    req->elem = elem;
+
+    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req->out,
+                            sizeof(req->out)) != sizeof(req->out))) {
+        fprintf(stderr, "virtio-blk request outhdr too short");
+        goto err;
+    }
+
+    iov_discard_front(&out_iov, &out_num, sizeof(req->out));
+
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+        fprintf(stderr, "virtio-blk request inhdr too short");
+        goto err;
+    }
+
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in = (void *)in_iov[in_num - 1].iov_base
+              + in_iov[in_num - 1].iov_len
+              - sizeof(struct virtio_blk_inhdr);
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+
+    type = le32toh(req->out.type);
+    switch (type & ~VIRTIO_BLK_T_BARRIER) {
+    case VIRTIO_BLK_T_IN:
+    case VIRTIO_BLK_T_OUT: {
+        ssize_t ret = 0;
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = le64toh(req->out.sector);
+        if (is_write) {
+            ret = vub_rwv(req, out_iov, out_num, blk_write_entry);
+        } else {
+            ret = vub_rwv(req, in_iov, in_num, blk_read_entry);
+        }
+        if (ret >= 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
+        vub_flush(req);
+        req->in->status = VIRTIO_BLK_S_OK;
+        vub_req_complete(req);
+        break;
+    case VIRTIO_BLK_T_GET_ID: {
+        size_t size = MIN(iov_size(&elem->in_sg[0], in_num),
+                          VIRTIO_BLK_ID_BYTES);
+        snprintf(elem->in_sg[0].iov_base, size, "%s", "vhost_user_blk");
+        req->in->status = VIRTIO_BLK_S_OK;
+        req->size = elem->in_sg[0].iov_len;
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_DISCARD:
+    case VIRTIO_BLK_T_WRITE_ZEROES: {
+        int rc;
+        rc = vub_discard_write_zeroes(req, &elem->out_sg[1], out_num, type);
+        if (rc == 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    default:
+        req->in->status = VIRTIO_BLK_S_UNSUPP;
+        vub_req_complete(req);
+        break;
+    }
+
+    return 0;
+
+err:
+    free(elem);
+    g_free(req);
+    return -1;
+}
+
+
+static void vub_process_vq(VuDev *vu_dev, int idx)
+{
+    VuClient *client;
+    VuVirtq *vq;
+    int ret;
+
+    client = container_of(vu_dev, VuClient, parent);
+    assert(client);
+
+    vq = vu_get_queue(vu_dev, idx);
+    assert(vq);
+
+    while (1) {
+        ret = vub_virtio_process_req(client, vq);
+        if (ret) {
+            break;
+        }
+    }
+}
+
+
+static void vub_queue_set_started(VuDev *vu_dev, int idx, bool started)
+{
+    VuVirtq *vq;
+
+    assert(vu_dev);
+
+    vq = vu_get_queue(vu_dev, idx);
+    vu_set_queue_handler(vu_dev, vq, started ? vub_process_vq : NULL);
+}
+
+static uint64_t
+vub_get_features(VuDev *dev)
+{
+    uint64_t features;
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+
+    features = 1ull << VIRTIO_BLK_F_SIZE_MAX |
+               1ull << VIRTIO_BLK_F_SEG_MAX |
+               1ull << VIRTIO_BLK_F_TOPOLOGY |
+               1ull << VIRTIO_BLK_F_BLK_SIZE |
+               1ull << VIRTIO_BLK_F_FLUSH |
+               #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+               1ull << VIRTIO_BLK_F_DISCARD |
+               1ull << VIRTIO_BLK_F_WRITE_ZEROES |
+               #endif
+               1ull << VIRTIO_BLK_F_CONFIG_WCE |
+               1ull << VIRTIO_F_VERSION_1 |
+               1ull << VIRTIO_RING_F_INDIRECT_DESC |
+               1ull << VIRTIO_RING_F_EVENT_IDX |
+               1ull << VHOST_USER_F_PROTOCOL_FEATURES;
+
+    if (!vdev_blk->writable) {
+        features |= 1ull << VIRTIO_BLK_F_RO;
+    }
+
+    return features;
+}
+
+static uint64_t
+vub_get_protocol_features(VuDev *dev)
+{
+    return 1ull << VHOST_USER_PROTOCOL_F_CONFIG |
+           1ull << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD;
+}
+
+static int
+vub_get_config(VuDev *vu_dev, uint8_t *config, uint32_t len)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    memcpy(config, &vdev_blk->blkcfg, len);
+
+    return 0;
+}
+
+static int
+vub_set_config(VuDev *vu_dev, const uint8_t *data,
+               uint32_t offset, uint32_t size, uint32_t flags)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    uint8_t wce;
+
+    /* don't support live migration */
+    if (flags != VHOST_SET_CONFIG_TYPE_MASTER) {
+        return -1;
+    }
+
+
+    if (offset != offsetof(struct virtio_blk_config, wce) ||
+        size != 1) {
+        return -1;
+    }
+
+    wce = *data;
+    if (wce == vdev_blk->blkcfg.wce) {
+        /* Do nothing as same with old configuration */
+        return 0;
+    }
+
+    vdev_blk->blkcfg.wce = wce;
+    blk_set_enable_write_cache(vdev_blk->backend, true);
+    return 0;
+}
+
+
+/*
+ * When the client disconnects, it send a VHOST_USER_NONE request
+ * and vu_process_message will simple call exit which cause the VM
+ * to exit abruptly.
+ * To avoid this issue,  process VHOST_USER_NONE request ahead
+ * of vu_process_message.
+ *
+ */
+static int vub_process_msg(VuDev *dev, VhostUserMsg *vmsg, int *do_reply)
+{
+    if (vmsg->request == VHOST_USER_NONE) {
+        dev->panic(dev, "disconnect");
+        return true;
+    }
+    return false;
+}
+
+static void

blockdev-vu.c line 484

Quoted 10 lines
blockdev-vu.c new file +10 -0
+vmsg_close_fds(VhostUserMsg *vmsg)
+{
+    int i;
+    for (i = 0; i < vmsg->fd_num; i++) {
+        close(vmsg->fds[i]);
+    }
+}
+
+static bool
+vu_message_read_co(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)

Why is this function called *_co()? It appears to be callable from outside coroutine context too.

Quoted 113 lines
blockdev-vu.c new file +113 -0
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+    char buffer[100];
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    QIOChannel *ioc = client->ioc;
+    do {
+        rc = recvmsg(conn_fd, &msg, 0);
+        if (rc < 0) {
+            if (errno == EAGAIN) {
+                if (qemu_in_coroutine()) {
+                    qio_channel_yield(ioc, G_IO_IN);
+                } else {
+                    qio_channel_wait(ioc, G_IO_IN);
+                }
+                continue;
+            } else if (errno == EINTR) {
+                continue;
+            }
+        }
+        break;
+    } while (true);
+
+    if (rc < 0) {
+        sprintf(buffer, "Error while recvmsg: %s", strerror(errno));
+        vub_panic_cb(vu_dev, buffer);
+        return false;
+    }
+
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);
+
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
+
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        sprintf(buffer,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        vub_panic_cb(vu_dev, buffer);
+        goto fail;
+    }
+
+    if (vmsg->size) {
+        do {
+            rc = read(conn_fd, &vmsg->payload, vmsg->size);
+            if (rc < 0) {
+                if (errno == EAGAIN) {
+                    if (qemu_in_coroutine()) {
+                        qio_channel_yield(ioc, G_IO_IN);
+                    } else {
+                        qio_channel_wait(ioc, G_IO_IN);
+                    }
+                    continue;
+                } else if (errno == EINTR) {
+                    continue;
+                }
+            }
+            break;
+        } while (true);
+
+        if (rc <= 0) {
+            sprintf(buffer, "Error while reading: %s", strerror(errno));
+            vub_panic_cb(vu_dev, buffer);
+            goto fail;
+        }
+
+        assert(rc == vmsg->size);
+    }
+
+    return true;
+
+fail:
+    vmsg_close_fds(vmsg);
+
+    return false;
+}
+
+static void vub_kick_cb(void *opaque)
+{
+    vu_watch_cb_data *data = (vu_watch_cb_data *) opaque;
+    int index = data->index;
+    VuDev *dev = data->vu_dev;
+    VuVirtq *vq = &dev->vq[index];
+    int sock = vq->kick_fd;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        char buffer[100];

blockdev-vu.c line 607

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        sprintf(buffer, "kick eventfd_read(): %s", strerror(errno));
+        vub_panic_cb(dev, buffer);
+        g_free(data);
+        dev->remove_watch(dev, dev->vq[index].kick_fd);
+    } else {
+        if (vq->handler) {
+            vq->handler(dev, index);
+        }
+    }
+}

This is a generic vhost-user function that is not specific to vhost-user-blk. It should be in a different source file.

Quoted 47 lines
blockdev-vu.c new file +47 -0
+
+static const VuDevIface vub_iface = {
+    .get_features = vub_get_features,
+    .queue_set_started = vub_queue_set_started,
+    .get_protocol_features = vub_get_protocol_features,
+    .get_config = vub_get_config,
+    .set_config = vub_set_config,
+    .process_msg = vub_process_msg,
+    .read_msg = vu_message_read_co,
+    .kick_callback = vub_kick_cb,
+};
+
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM)
+{
+    if (!vub_dev) {
+        return;
+    }
+
+    blk_unref(vub_dev->backend);
+    g_free(vub_dev->name);
+    g_free(vub_dev->unix_socket);
+
+    if (vub_dev->next.tqe_circ.tql_prev) {
+        /*
+         * if vub_dev->next.tqe_circ.tql_prev = null,
+         * vub_dev hasn't been inserted into the queue and
+         * vub_free is called by obj->instance_finalize.
+         */
+        QTAILQ_REMOVE(&vub_devs, vub_dev, next);
+    }
+    /*
+     * Needn't to free vub_dev if called by QOM
+     * because QOM will do the clean-up work.
+     */
+    if (!called_by_QOM) {
+        g_free(vub_dev);
+    }
+}
+
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+    VuClient *client = opaque;
+
+    while (!client->closed) {
+        vu_dispatch(&client->parent);
+    }

blockdev-vu.c line 664

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    QTAILQ_REMOVE(&client->blk->clients, client, next);
+
+}
+
+static void vu_client_start(VuClient *client)
+{
+    Coroutine *co = qemu_coroutine_create(vu_client_trip, client);
+    qemu_coroutine_enter(co);
+}

What is the coroutine and threading model here? This launches vu_client_trip() in the current thread's event loop but vu_dispatch() performs blocking I/O and will prevent the event loop from continuing.

Quoted 26 lines
blockdev-vu.c new file +26 -0
+
+
+G_STATIC_ASSERT((int)G_IO_IN == (int)VU_WATCH_IN);
+G_STATIC_ASSERT((int)G_IO_OUT == (int)VU_WATCH_OUT);
+G_STATIC_ASSERT((int)G_IO_PRI == (int)VU_WATCH_PRI);
+G_STATIC_ASSERT((int)G_IO_ERR == (int)VU_WATCH_ERR);
+G_STATIC_ASSERT((int)G_IO_HUP == (int)VU_WATCH_HUP);
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+          vu_watch_cb_packed_data cb, void *pvt)
+{
+    /*
+     * since aio_dispatch can only pass one user data pointer to the
+     * callback function, pack VuDev, pvt into a struct
+     */
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+    g_assert(cb);
+    client = container_of(vu_dev, VuClient, parent);
+    vu_watch_cb_data *cb_data = g_new0(vu_watch_cb_data, 1);
+    cb_data->index = (intptr_t) pvt;
+    cb_data->vu_dev = vu_dev;
+    aio_set_fd_handler(client->blk->ctx, fd, false, (void *) cb,

blockdev-vu.c line 700

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                       NULL, NULL, cb_data);
+}
+
+
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);

Is it helpful to have a separate VuClient struct even though only a single vhost-user client can be connected at a time? It may be simpler to keep connection state directly in VubDev.

Quoted 24 lines
blockdev-vu.c new file +24 -0
+
+    if (!vu_init_packed_data(&client->parent, VHOST_USER_BLK_MAX_QUEUES,
+                             sioc->fd, vub_panic_cb, set_watch,
+                             remove_watch, &vub_iface)) {
+        fprintf(stderr, "Failed to initialized libvhost-user\n");
+        g_free(client);
+        return;
+    }
+
+    client->blk = vub_device;
+    client->refcount = 1;
+    client->sioc = sioc;
+    /*
+     * increase the object reference, so cioc will not freed by
+     * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))
+     */
+    object_ref(OBJECT(client->sioc));
+    qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(QIO_CHANNEL(client->sioc), false, NULL);
+    client->closed = false;

blockdev-vu.c line 734

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    QTAILQ_INSERT_TAIL(&client->blk->clients, client, next);
+    vu_client_start(client);
+}
+
+
+void
+vub_initialize_config(BlockDriverState *bs, struct virtio_blk_config *config)
+{
+    config->capacity = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    config->blk_size = BDRV_SECTOR_SIZE;

At some point this should become a configuration parameter because using 512B sectors on a native 4K sector image file results in poor performance.

blockdev-vu.c line 735

Quoted 1 line
blockdev-vu.c new file +1 -0
+    config->size_max = 65536;

Why 65536? hw/block/virtio-blk.c sets it to 0.

blockdev-vu.c line 736

Quoted 1 line
blockdev-vu.c new file +1 -0
+    config->seg_max = 128 - 2;

This is okay for now but should be changed in the future.

hw/block/virtio-blk.c was recently modified to calculate seg_max based on the virtqueue size (which is where the hardcoded 128 originally came from). Applications that use large buffer sizes (128+ KB) benefit from larger virtqueue sizes and seg_max.

Quoted 5 lines
blockdev-vu.c new file +5 -0
+    config->min_io_size = 1;
+    config->opt_io_size = 1;
+    config->num_queues = 1;
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    config->max_discard_sectors = 32768;

blockdev-vu.c line 751

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    config->max_discard_seg = 1;
+    config->discard_sector_alignment = config->blk_size >> 9;
+    config->max_write_zeroes_sectors = 32768;
+    config->max_write_zeroes_seg = 1;
+    #endif
+}
+
+
+static VubDev *vub_new(VubDev *vub_device, const char *name,
+                       const char *unix_socket, bool writable, Error **errp)

This function does not allocate and return a new VubDev, so vub_new() is not an accurate name for this function. The name vub_init() is clearer.

Why are name, unix_socket, and writable passed in? They are already vub_device fields.

Quoted 128 lines
blockdev-vu.c new file +128 -0
+{
+
+    BlockBackend *blk;
+
+    /*
+     * Don't allow resize while the vhost user server is running,
+     * otherwise we don't care what happens with the node.
+     */
+    uint64_t perm = BLK_PERM_CONSISTENT_READ;
+    int ret;
+
+    AioContext *ctx;
+
+    BlockDriverState *bs = bdrv_lookup_bs(name,
+                                          name,
+                                          errp);
+
+    if (!bs) {
+        error_setg(errp,
+                   "No drive with name '%s'."
+                   " Please find the list of names with "
+                   "'info block'", name);
+        return NULL;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    if (writable) {
+        perm |= BLK_PERM_WRITE;
+    }
+
+    ctx = bdrv_get_aio_context(bs);
+    aio_context_acquire(ctx);
+    bdrv_invalidate_cache(bs, NULL);
+    aio_context_release(ctx);
+
+    blk = blk_new(bdrv_get_aio_context(bs), perm,
+                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
+                  BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(blk, bs, errp);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+
+    blk_set_enable_write_cache(blk, false);
+
+    blk_set_allow_aio_context_change(blk, true);
+
+
+    vub_device->name = g_strdup(name);
+    vub_device->unix_socket = g_strdup(unix_socket);
+    vub_device->writable = writable;
+    vub_device->blkcfg.wce = 0;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    return vub_device;
+
+fail:
+    blk_unref(blk);
+    return NULL;
+}
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM)
+{
+    if (!vub_device) {
+        return;
+    }
+
+    VuClient *client, *next;
+    QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) {
+        if (!client->closed) {
+            close_client(client);
+        }
+    }
+
+    if (vub_device->listener) {
+        qio_net_listener_disconnect(vub_device->listener);
+        object_unref(OBJECT(vub_device->listener));
+    }
+    vub_free(vub_device, called_by_QOM);
+
+}
+
+
+VubDev *vub_dev_find(const char *name)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(name, vub_device->name) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+
+static VubDev *vub_dev_find_by_unix_socket(const char *unix_socket)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(unix_socket, vub_device->unix_socket) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+static void vhost_user_server_start(VubDev *vub_device, const char *unix_socket,
+                                    const char *name, bool writable,
+                                    Error **errp)
+{
+
+    if (vub_dev_find(name) || vub_dev_find_by_unix_socket(unix_socket)) {
+        error_setg(errp, "Vhost user server with name '%s' or "
+                "with socket_path '%s' has already been started",
+                name, unix_socket);
+        return;
+    }
+
+
+    if (!vub_new(vub_device, name, unix_socket, writable, errp)) {

blockdev-vu.c line 889

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        return;
+    }
+
+
+    vub_device->listener = qio_net_listener_new();
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);

addr is never freed. This variable could live on the stack instead:

  SocketAddress addr = {};
Quoted 44 lines
blockdev-vu.c new file +44 -0
+    addr->u.q_unix.path = (char *) unix_socket;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp) < 0) {
+        goto error;
+    }
+
+
+    QTAILQ_INSERT_TAIL(&vub_devs, vub_device, next);
+    QTAILQ_INIT(&vub_device->clients);
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    return;
+
+ error:
+    vub_free(vub_device, false);
+}
+
+static void vu_set_block_name(Object *obj, const char *value,
+                                           Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->name) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+
+    vus->name = g_strdup(value);
+}
+
+static char *vu_get_block_name(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    return g_strdup(vus->name);
+}
+
+
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{

blockdev-vu.c line 943

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);

Property setters should only perform input validation and store the data. Actions like creating network connections, opening files, etc should happen later in a UserCreatableClass->complete() callback.

This is necessary because vus->writable is also a property and may be set after unix_socket. The ->complete() callback is called after all setters so it can access the final values of all properties.

See iothread_class_init() and iothread_complete() for an example.

Quoted 98 lines
blockdev-vu.c new file +65 -0
+}
+
+static char *vu_get_unix_socket(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return g_strdup(vus->unix_socket);
+}
+
+static bool vu_get_block_writable(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return vus->writable;
+}
+
+static void vu_set_block_writable(Object *obj, bool value, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+
+    vus->writable = value;
+}
+
+static void vhost_user_server_instance_init(Object *obj)
+{
+
+    object_property_add_bool(obj, "writable",
+                            vu_get_block_writable,
+                            vu_set_block_writable, NULL);
+
+    object_property_add_str(obj, "name",
+                            vu_get_block_name,
+                            vu_set_block_name, NULL);
+
+    object_property_add_str(obj, "unix_socket",
+                            vu_get_unix_socket,
+                            vu_set_unix_socket, NULL);
+
+}
+
+static void vhost_user_server_instance_finalize(Object *obj)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    vhost_user_server_free(vus, true);
+    /* object_del shouldn't free this object struct */
+    obj->free = NULL;
+}
+
+static const TypeInfo vhost_user_server_info = {
+    .name = TYPE_VHOST_USER_SERVER,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(VuDev),
+    .instance_init = vhost_user_server_instance_init,
+    .instance_finalize = vhost_user_server_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        {TYPE_USER_CREATABLE},
+        {}
+    },
+};
+
+static void vhost_user_server_register_types(void)
+{
+    type_register_static(&vhost_user_server_info);
+}
+
+type_init(vhost_user_server_register_types)
+
include/block/vhost-user.h new file +27 -0
new file mode 100644
index 0000000000..ef6d695244
@@ -0,0 +1,46 @@
+#include "io/channel-socket.h"
+#include "io/net-listener.h"
+#include "contrib/libvhost-user/libvhost-user.h"
+#include "standard-headers/linux/virtio_blk.h"
+typedef struct VubDev VubDev;
+typedef struct VuClient VuClient;
+#define TYPE_VHOST_USER_SERVER "vhost-user-server"
+
+#define VHOST_USER_SERVER(obj) \
+   OBJECT_CHECK(VubDev, obj, TYPE_VHOST_USER_SERVER)
+/* vhost user block device */
+struct VubDev {
+    Object parent_obj;
+    char *name;
+    char *unix_socket;
+    bool exit_panic;
+    bool close;
+    BlockBackend *backend;
+    AioContext *ctx;
+    QIONetListener *listener;
+    QIOChannelSocket *sioc;
+    QTAILQ_HEAD(, VuClient) clients;
+    QTAILQ_ENTRY(VubDev) next;
+    struct virtio_blk_config blkcfg;
+    bool writable;
+};
+

include/block/vhost-user.h line 37

Quoted 10 lines
include/block/vhost-user.h new file +10 -0
+struct VuClient {
+    VuDev parent;
+    int refcount;
+    VubDev *blk;
+    QIOChannelSocket *sioc; /* The underlying data channel */
+    QIOChannel *ioc; /* The current I/O channel */
+    QTAILQ_ENTRY(VuClient) next;
+    bool closed;
+};
+VubDev *vub_dev_find(const char *name);

All -object instances already have an id=ID property. There is no need to declare a separate "name" property. Please look at iothread_by_id() and iothread_get_id() for examples.

Quoted 17 lines
include/block/vhost-user.h new file +9 -0
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM);
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque);
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM);
+
+void vub_initialize_config(BlockDriverState *bs,
+                           struct virtio_blk_config *config);
vl.c +0 -0
index 86474a55c9..72ac506342 100644
@@ -2553,6 +2553,10 @@ static bool object_create_initial(const char *type, QemuOpts *opts)
     }
 #endif
 
Quoted 15 lines

vl.c static bool object_create_initial(const char *type, QemuOpts *opts) · line 2559

Quoted 4 lines
vl.c +4 -0
+    /* Reason: vhost-user-server property "name" */
+    if (g_str_equal(type, "vhost-user-server")) {
+        return false;
+    }

I don't understand why the "name" property introduces a creation order dependency. It's just a string and has no dependency on other command-line objects. Can you explain why this change is necessary?

Quoted 6 lines
vl.c +0 -1
     /*
      * Reason: filter-* property "netdev" etc.
      */
--

2.24.1

--
Best regards,
Coiby

#5 lore raw

Re: [PATCH v2 1/5] vhost-user block device backend

In reply to Coiby Xu

Hi,

I'm only doing a quick first review pointing out the more obvious things while I familiarise myself with your code. I intend to review it in more detail later (either in a second pass for this series, or when you post v3).

Am 14.01.2020 um 15:06 hat Coiby Xu geschrieben:

Quoted 3 lines

By making use of libvhost, multiple block device drives can be exported and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.

Please wrap the commit message at 72 characters.

Quoted 6 lines
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h

This adds a single, relatively big source file. I see at least two
parts: The generic vhost-user infrastructure with connection handling etc. and the implementation of the specific vhost-user-blk device. Separating these into two files is probably a good idea.

I would also suggest to put the files in a new subdirectory block/export/ and call them vhost-user.c/vhost-user-blk.c. The new header file can be in the same directory as it shouldn't be used by anyone else.

blockdev-vu.c

Quoted 6 lines
blockdev-vu.c new file +0 -0
new file mode 100644
index 0000000000..45f0bb43a7
@@ -0,0 +1,1008 @@

The LICENSE file clarifies that files without a license header are GPLv2+, so it's not strictly a problem, but I think it is good style to include a license header that explicitly tells so.

Quoted 8 lines
blockdev-vu.c new file +8 -0
+#include "qemu/osdep.h"
+#include "block/vhost-user.h"
+#include "qapi/error.h"
+#include "qapi/qapi-types-sockets.h"
+#include "qapi/qapi-commands-block.h"
+
+#include "sysemu/block-backend.h"
+#include "qemu/main-loop.h"

blockdev-vu.c line 18

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+#include "qemu/units.h"
+
+#include "block/block.h"
+
+#include "qom/object_interfaces.h"
+
+#include <sys/eventfd.h>
+
+#include "hw/qdev-properties.h"

Does the order of includes and the empty lines between them signify anything? If not, I suggest just sorting them alphabetically (and maybe using empty lines between different subdirectories if you like this better than a single large block).

According to CODING_STYLE.rst, system headers like <sys/eventfd.h> come before all QEMU headers (except qemu/osdep.h, which always must come first).

Quoted 4 lines
blockdev-vu.c new file +4 -0
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};
+

blockdev-vu.c line 32

Quoted 10 lines
blockdev-vu.c new file +10 -0
+struct virtio_blk_inhdr {
+    unsigned char status;
+};
+
+
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;

Maybe worth a comment that this was allocated with plain malloc(), so you must use free() rather than g_free() (which would be the default in QEMU)?

blockdev-vu.c line 39

Quoted 7 lines
blockdev-vu.c new file +7 -0
+    int64_t sector_num;
+    size_t size;
+    struct virtio_blk_inhdr *in;
+    struct virtio_blk_outhdr out;
+    VuClient *client;
+    struct VuVirtq *vq;
+} VubReq;

I'm not completely sure yet, but I think I would prefer VuBlock to Vub in the type names. Some may even prefer VhostUserBlock, but I can see that this would be quite lengthy.

Quoted 6 lines
blockdev-vu.c new file +6 -0
+static void
+remove_watch(VuDev *vu_dev, int fd)
+{
+    VuClient *client;
+
+    g_assert(vu_dev);

blockdev-vu.c line 56

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    g_assert(fd >= 0);
+
+    client = container_of(vu_dev, VuClient, parent);
+    aio_set_fd_handler(client->blk->ctx, fd, false, NULL, NULL, NULL, NULL);
+}
+
+static void close_client(VuClient *client)
+{
+    vu_deinit(&client->parent);
+    /** g_source_destroy(vub_device->parent.src); */

Leftover from conversion?

blockdev-vu.c line 63

Quoted 7 lines
blockdev-vu.c new file +7 -0
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->closed = true;
+
+}
+
+static void vub_panic_cb(VuDev *vu_dev, const char *buf)

You use a lot of sprintf() before calling this function. Would it be worth taking a printf-like format parameter instead of buf and using a variable argument list?

blockdev-vu.c line 66

Quoted 3 lines
blockdev-vu.c new file +3 -0
+{
+    if (buf) {
+        g_warning("vu_panic: %s", buf);

I think QEMU proper doesn't use g_warning() anywhere. This could be error_report() or warn_report(). (Or if you use a format string error_vreport() and warn_vreport().)

Quoted 9 lines
blockdev-vu.c new file +9 -0
+    }
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    if (client->blk->exit_panic) {
+        client->blk->close = true;
+    }
+    if (!client->closed) {
+        close_client(client);
+    }

blockdev-vu.c line 85

Quoted 10 lines
blockdev-vu.c new file +10 -0
+}
+
+
+static void vub_req_complete(VubReq *req)
+{
+    VuDev *vu_dev = &req->client->parent;
+
+    /* IO size with 1 extra status byte */
+    vu_queue_push(vu_dev, req->vq, req->elem,
+                  req->size + 1);

I think this fits in a single line.

Quoted 13 lines
blockdev-vu.c new file +13 -0
+    vu_queue_notify(vu_dev, req->vq);
+
+    if (req->elem) {
+        free(req->elem);
+    }
+
+    g_free(req);
+}
+
+
+
+static int
+vub_discard_write_zeroes(VubReq *req, struct iovec *iov, uint32_t iovcnt,

blockdev-vu.c line 108

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                         uint32_t type)
+{
+    struct virtio_blk_discard_write_zeroes *desc;
+    ssize_t size;
+    void *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (size != sizeof(*desc)) {
+        fprintf(stderr, "Invalid size %ld, expect %ld\n", size, sizeof(*desc));
+        return -1;

This would be error_report(), too. (More cases below, I'll ignore them now.)

I would prefer consistent use of -errno instead of -1 for error cases if you don't mind. I guess this would be -EINVAL here. I won't mention it for all the other cases; if you want to make the change, you need to make it everywhere, obviously.

blockdev-vu.c line 112

Quoted 4 lines
blockdev-vu.c new file +4 -0
+    }
+    buf = g_new0(char, size);
+
+    iov_to_buf_full(iov, iovcnt, 0, buf, size);

I think uint8_t describes better than char what we want here: A buffer of bytes.

The empty line would make more sense to me above the g_new0() line than after it because it starts a new section that deals with the buffer. In general, the use of empty lines feels a bit inconsistent in this patch. You may want to go over them again.

blockdev-vu.c line 113

Quoted 2 lines
blockdev-vu.c new file +2 -0
+
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)

Preprocessor directives should be unindented.

However, I don't think any of this code actually depends on Linux, BLKDISCARD or BLKZEROOUT. You can just call blk_pdiscard() and blk_pwrite_zeroes() and they will do whatever is necessary to perform the operation on the backend (which might not be a Linux block device, but could be a regular file or even using a network protocol like NBD).

Quoted 6 lines
blockdev-vu.c new file +6 -0
+    VubDev *vdev_blk;
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+    desc = (struct virtio_blk_discard_write_zeroes *)buf;
+    uint64_t range[2] = { le64toh(desc->sector) << 9,
+                          le32toh(desc->num_sectors) << 9 };

blockdev-vu.c line 131

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    if (type == VIRTIO_BLK_T_DISCARD) {
+        if (blk_pdiscard(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    } else if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
+        if (blk_pwrite_zeroes(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }

blk_pdiscard() and blk_pwrite_zeroes() are synchronous functions. In other words, the guest will be blocked until the I/O is complete. We cannot do this.

I think you should let vub_virtio_process_req() run in a coroutine so that you can call blk_co_pdiscard() and blk_co_pwrite_zeroes() here.

blockdev-vu.c line 144

Quoted 13 lines
blockdev-vu.c new file +13 -0
+    }
+    #endif
+
+    g_free(buf);
+    return -1;
+}
+
+
+static void
+vub_flush(VubReq *req)
+{
+    VuClient *client = req->client;
+    blk_co_flush(client->blk->backend);

You can't call blk_co_flush() from outside coroutine context. This code will be right after you move vub_virtio_process_req() to a coroutine, though (which will make this function a coroutine_fn).

Quoted 47 lines
blockdev-vu.c new file +47 -0
+}
+
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    void *iobuf;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static int blk_prw(BlockBackend *blk, QEMUIOVector *qiov, int64_t offset,
+                   CoroutineEntry co_entry, BdrvRequestFlags flags)
+{
+
+    BlkRwCo rwco = {
+        .blk    = blk,
+        .offset = offset,
+        .iobuf  = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {

blockdev-vu.c line 201

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        /* Fast-path if already in coroutine context */
+        co_entry(&rwco);
+    } else {
+        Coroutine *co = qemu_coroutine_create(co_entry, &rwco);
+        bdrv_coroutine_enter(blk_bs(blk), co);
+        BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
+    }
+
+    return rwco.ret;
+}

This is copy&paste from block-backend.c. We should certainly not do this. I think it will automatically go away when you can use blk_co_preadv() and blk_co_pwritev() directly.

Note that the BDRV_POLL_WHILE() means that like above, we would be waiting for the request to complete. This would block the guest and would also not allow parallel requests, killing the I/O performance of our vhost-user export.

blockdev-vu.c line 207

Quoted 5 lines
blockdev-vu.c new file +5 -0
+
+static ssize_t
+vub_rwv(VubReq *req, struct iovec *iov,
+        uint32_t iovcnt,
+        CoroutineEntry co_entry)

I don't understand the line wrapping here. :-)

Quoted 5 lines
blockdev-vu.c new file +5 -0
+{
+    VuClient *client = req->client;
+    ssize_t rc;
+
+    if (!iovcnt) {

blockdev-vu.c line 222

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        fprintf(stderr, "Invalid Read/Write IOV count\n");
+        return -1;
+    }
+
+    int64_t offset = req->sector_num * 512;
+    QEMUIOVector *qiov = g_new0(QEMUIOVector, 1);
+    qemu_iovec_init_external(qiov, iov, iovcnt);
+    rc = blk_prw(client->blk->backend, qiov, offset, co_entry, 0);
+
+    req->size = iov_size(iov, iovcnt);

You can use qiov->size instead of duplicating this information into a separate VubReq field.

blockdev-vu.c line 234

Quoted 12 lines
blockdev-vu.c new file +12 -0
+    if (rc < 0) {
+        fprintf(stderr, "%s, Sector %"PRIu64", Size %lu failed with %s\n",
+                client->blk->name, req->sector_num, req->size,
+                strerror(errno));
+        return -1;
+    }
+
+    return rc;
+}
+
+static int vub_virtio_process_req(VuClient *client,
+                                     VuVirtq *vq)

Indentation is off. This could be a single line anyway.

Quoted 117 lines
blockdev-vu.c new file +117 -0
+{
+    VuDev *vu_dev = &client->parent;
+    VuVirtqElement *elem;
+    uint32_t type;
+    VubReq *req;
+
+    elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement) + sizeof(VubReq));
+    if (!elem) {
+        return -1;
+    }
+
+    struct iovec *in_iov = elem->in_sg;
+    struct iovec *out_iov = elem->out_sg;
+    unsigned in_num = elem->in_num;
+    unsigned out_num = elem->out_num;
+    /* refer to hw/block/virtio_blk.c */
+    if (elem->out_num < 1 || elem->in_num < 1) {
+        fprintf(stderr, "virtio-blk request missing headers\n");
+        free(elem);
+        return -1;
+    }
+
+    req = g_new0(VubReq, 1);
+    req->client = client;
+    req->vq = vq;
+    req->elem = elem;
+
+    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req->out,
+                            sizeof(req->out)) != sizeof(req->out))) {
+        fprintf(stderr, "virtio-blk request outhdr too short");
+        goto err;
+    }
+
+    iov_discard_front(&out_iov, &out_num, sizeof(req->out));
+
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+        fprintf(stderr, "virtio-blk request inhdr too short");
+        goto err;
+    }
+
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in = (void *)in_iov[in_num - 1].iov_base
+              + in_iov[in_num - 1].iov_len
+              - sizeof(struct virtio_blk_inhdr);
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+
+    type = le32toh(req->out.type);
+    switch (type & ~VIRTIO_BLK_T_BARRIER) {
+    case VIRTIO_BLK_T_IN:
+    case VIRTIO_BLK_T_OUT: {
+        ssize_t ret = 0;
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = le64toh(req->out.sector);
+        if (is_write) {
+            ret = vub_rwv(req, out_iov, out_num, blk_write_entry);
+        } else {
+            ret = vub_rwv(req, in_iov, in_num, blk_read_entry);
+        }
+        if (ret >= 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
+        vub_flush(req);
+        req->in->status = VIRTIO_BLK_S_OK;
+        vub_req_complete(req);
+        break;
+    case VIRTIO_BLK_T_GET_ID: {
+        size_t size = MIN(iov_size(&elem->in_sg[0], in_num),
+                          VIRTIO_BLK_ID_BYTES);
+        snprintf(elem->in_sg[0].iov_base, size, "%s", "vhost_user_blk");
+        req->in->status = VIRTIO_BLK_S_OK;
+        req->size = elem->in_sg[0].iov_len;
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_DISCARD:
+    case VIRTIO_BLK_T_WRITE_ZEROES: {
+        int rc;
+        rc = vub_discard_write_zeroes(req, &elem->out_sg[1], out_num, type);
+        if (rc == 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    default:
+        req->in->status = VIRTIO_BLK_S_UNSUPP;
+        vub_req_complete(req);
+        break;
+    }
+
+    return 0;
+
+err:
+    free(elem);
+    g_free(req);
+    return -1;
+}
+
+
+static void vub_process_vq(VuDev *vu_dev, int idx)
+{
+    VuClient *client;
+    VuVirtq *vq;
+    int ret;
+
+    client = container_of(vu_dev, VuClient, parent);
+    assert(client);
+

blockdev-vu.c line 361

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    vq = vu_get_queue(vu_dev, idx);
+    assert(vq);
+
+    while (1) {
+        ret = vub_virtio_process_req(client, vq);
+        if (ret) {
+            break;
+        }
+    }
+}

I mentioned above moving vub_virtio_process_req() into a coroutine. Just creating a coroutine here and entering it wouldn't work, though.

The best design is probably to get requests from the virtqueue in this function (what is currently the first half of vub_virtio_process_req()) and then spawn a coroutine per request to actually execute them (roughly the switch in vub_virtio_process_req()).

This way you'll get parallel requests and won't have to think about synchronising accesses to the virtqueue from multiple coroutines.

Quoted 86 lines
blockdev-vu.c new file +86 -0
+
+static void vub_queue_set_started(VuDev *vu_dev, int idx, bool started)
+{
+    VuVirtq *vq;
+
+    assert(vu_dev);
+
+    vq = vu_get_queue(vu_dev, idx);
+    vu_set_queue_handler(vu_dev, vq, started ? vub_process_vq : NULL);
+}
+
+static uint64_t
+vub_get_features(VuDev *dev)
+{
+    uint64_t features;
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+
+    features = 1ull << VIRTIO_BLK_F_SIZE_MAX |
+               1ull << VIRTIO_BLK_F_SEG_MAX |
+               1ull << VIRTIO_BLK_F_TOPOLOGY |
+               1ull << VIRTIO_BLK_F_BLK_SIZE |
+               1ull << VIRTIO_BLK_F_FLUSH |
+               #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+               1ull << VIRTIO_BLK_F_DISCARD |
+               1ull << VIRTIO_BLK_F_WRITE_ZEROES |
+               #endif
+               1ull << VIRTIO_BLK_F_CONFIG_WCE |
+               1ull << VIRTIO_F_VERSION_1 |
+               1ull << VIRTIO_RING_F_INDIRECT_DESC |
+               1ull << VIRTIO_RING_F_EVENT_IDX |
+               1ull << VHOST_USER_F_PROTOCOL_FEATURES;
+
+    if (!vdev_blk->writable) {
+        features |= 1ull << VIRTIO_BLK_F_RO;
+    }
+
+    return features;
+}
+
+static uint64_t
+vub_get_protocol_features(VuDev *dev)
+{
+    return 1ull << VHOST_USER_PROTOCOL_F_CONFIG |
+           1ull << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD;
+}
+
+static int
+vub_get_config(VuDev *vu_dev, uint8_t *config, uint32_t len)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    memcpy(config, &vdev_blk->blkcfg, len);
+
+    return 0;
+}
+
+static int
+vub_set_config(VuDev *vu_dev, const uint8_t *data,
+               uint32_t offset, uint32_t size, uint32_t flags)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    uint8_t wce;
+
+    /* don't support live migration */
+    if (flags != VHOST_SET_CONFIG_TYPE_MASTER) {
+        return -1;
+    }
+
+
+    if (offset != offsetof(struct virtio_blk_config, wce) ||
+        size != 1) {
+        return -1;
+    }
+
+    wce = *data;
+    if (wce == vdev_blk->blkcfg.wce) {
+        /* Do nothing as same with old configuration */
+        return 0;

blockdev-vu.c line 458

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    }
+
+    vdev_blk->blkcfg.wce = wce;
+    blk_set_enable_write_cache(vdev_blk->backend, true);
+    return 0;
+}
+
+
+/*
+ * When the client disconnects, it send a VHOST_USER_NONE request

s/send/sends/

Quoted 36 lines
blockdev-vu.c new file +36 -0
+ * and vu_process_message will simple call exit which cause the VM
+ * to exit abruptly.
+ * To avoid this issue,  process VHOST_USER_NONE request ahead
+ * of vu_process_message.
+ *
+ */
+static int vub_process_msg(VuDev *dev, VhostUserMsg *vmsg, int *do_reply)
+{
+    if (vmsg->request == VHOST_USER_NONE) {
+        dev->panic(dev, "disconnect");
+        return true;
+    }
+    return false;
+}
+
+static void
+vmsg_close_fds(VhostUserMsg *vmsg)
+{
+    int i;
+    for (i = 0; i < vmsg->fd_num; i++) {
+        close(vmsg->fds[i]);
+    }
+}
+
+static bool
+vu_message_read_co(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,

blockdev-vu.c line 504

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+    char buffer[100];
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    QIOChannel *ioc = client->ioc;
+    do {
+        rc = recvmsg(conn_fd, &msg, 0);

This should certainly use qio_channel_readv_full() rather than working directly with a socket fd?

Quoted 12 lines
blockdev-vu.c new file +12 -0
+        if (rc < 0) {
+            if (errno == EAGAIN) {
+                if (qemu_in_coroutine()) {
+                    qio_channel_yield(ioc, G_IO_IN);
+                } else {
+                    qio_channel_wait(ioc, G_IO_IN);
+                }
+                continue;
+            } else if (errno == EINTR) {
+                continue;
+            }
+        }

blockdev-vu.c line 526

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        break;
+    } while (true);
+
+    if (rc < 0) {
+        sprintf(buffer, "Error while recvmsg: %s", strerror(errno));
+        vub_panic_cb(vu_dev, buffer);
+        return false;
+    }
+
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);

Why do you think that there can't be short reads?

blockdev-vu.c line 539

Quoted 12 lines
blockdev-vu.c new file +12 -0
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }

I think the fd passing part becomes easier when you use the proper qio_channel_readv_full() function. Its implementation is also a bit more careful than yours. For example, you forgot checking fd_size against VHOST_MEMORY_MAX_NREGIONS, allowing a buffer overflow in the memcpy(), and you don't adjust fd flags for the new file descriptors.

blockdev-vu.c line 552

Quoted 12 lines
blockdev-vu.c new file +12 -0
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        sprintf(buffer,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        vub_panic_cb(vu_dev, buffer);
+        goto fail;
+    }
+
+    if (vmsg->size) {
+        do {
+            rc = read(conn_fd, &vmsg->payload, vmsg->size);

qio_channel_readv_all_eof() already implements this whole loop and correctly handles short reads, too.

Quoted 56 lines
blockdev-vu.c new file +56 -0
+            if (rc < 0) {
+                if (errno == EAGAIN) {
+                    if (qemu_in_coroutine()) {
+                        qio_channel_yield(ioc, G_IO_IN);
+                    } else {
+                        qio_channel_wait(ioc, G_IO_IN);
+                    }
+                    continue;
+                } else if (errno == EINTR) {
+                    continue;
+                }
+            }
+            break;
+        } while (true);
+
+        if (rc <= 0) {
+            sprintf(buffer, "Error while reading: %s", strerror(errno));
+            vub_panic_cb(vu_dev, buffer);
+            goto fail;
+        }
+
+        assert(rc == vmsg->size);
+    }
+
+    return true;
+
+fail:
+    vmsg_close_fds(vmsg);
+
+    return false;
+}
+
+static void vub_kick_cb(void *opaque)
+{
+    vu_watch_cb_data *data = (vu_watch_cb_data *) opaque;
+    int index = data->index;
+    VuDev *dev = data->vu_dev;
+    VuVirtq *vq = &dev->vq[index];
+    int sock = vq->kick_fd;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        char buffer[100];
+        sprintf(buffer, "kick eventfd_read(): %s", strerror(errno));
+        vub_panic_cb(dev, buffer);
+        g_free(data);
+        dev->remove_watch(dev, dev->vq[index].kick_fd);
+    } else {
+        if (vq->handler) {
+            vq->handler(dev, index);
+        }
+    }
+}
+

blockdev-vu.c line 618

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static const VuDevIface vub_iface = {
+    .get_features = vub_get_features,
+    .queue_set_started = vub_queue_set_started,
+    .get_protocol_features = vub_get_protocol_features,
+    .get_config = vub_get_config,
+    .set_config = vub_set_config,
+    .process_msg = vub_process_msg,
+    .read_msg = vu_message_read_co,
+    .kick_callback = vub_kick_cb,
+};

I would prefer the = signs to be aligned to the same column.

Quoted 28 lines
blockdev-vu.c new file +28 -0
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM)
+{
+    if (!vub_dev) {
+        return;
+    }
+
+    blk_unref(vub_dev->backend);
+    g_free(vub_dev->name);
+    g_free(vub_dev->unix_socket);
+
+    if (vub_dev->next.tqe_circ.tql_prev) {
+        /*
+         * if vub_dev->next.tqe_circ.tql_prev = null,
+         * vub_dev hasn't been inserted into the queue and
+         * vub_free is called by obj->instance_finalize.
+         */
+        QTAILQ_REMOVE(&vub_devs, vub_dev, next);
+    }
+    /*
+     * Needn't to free vub_dev if called by QOM
+     * because QOM will do the clean-up work.
+     */
+    if (!called_by_QOM) {
+        g_free(vub_dev);
+    }
+}
+

blockdev-vu.c line 657

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+    VuClient *client = opaque;
+
+    while (!client->closed) {
+        vu_dispatch(&client->parent);
+    }
+
+    QTAILQ_REMOVE(&client->blk->clients, client, next);
+

Extra empty line.

Quoted 100 lines
blockdev-vu.c new file +100 -0
+}
+
+static void vu_client_start(VuClient *client)
+{
+    Coroutine *co = qemu_coroutine_create(vu_client_trip, client);
+    qemu_coroutine_enter(co);
+}
+
+
+G_STATIC_ASSERT((int)G_IO_IN == (int)VU_WATCH_IN);
+G_STATIC_ASSERT((int)G_IO_OUT == (int)VU_WATCH_OUT);
+G_STATIC_ASSERT((int)G_IO_PRI == (int)VU_WATCH_PRI);
+G_STATIC_ASSERT((int)G_IO_ERR == (int)VU_WATCH_ERR);
+G_STATIC_ASSERT((int)G_IO_HUP == (int)VU_WATCH_HUP);
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+          vu_watch_cb_packed_data cb, void *pvt)
+{
+    /*
+     * since aio_dispatch can only pass one user data pointer to the
+     * callback function, pack VuDev, pvt into a struct
+     */
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+    g_assert(cb);
+    client = container_of(vu_dev, VuClient, parent);
+    vu_watch_cb_data *cb_data = g_new0(vu_watch_cb_data, 1);
+    cb_data->index = (intptr_t) pvt;
+    cb_data->vu_dev = vu_dev;
+    aio_set_fd_handler(client->blk->ctx, fd, false, (void *) cb,
+                       NULL, NULL, cb_data);
+}
+
+
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);
+
+    if (!vu_init_packed_data(&client->parent, VHOST_USER_BLK_MAX_QUEUES,
+                             sioc->fd, vub_panic_cb, set_watch,
+                             remove_watch, &vub_iface)) {
+        fprintf(stderr, "Failed to initialized libvhost-user\n");
+        g_free(client);
+        return;
+    }
+
+    client->blk = vub_device;
+    client->refcount = 1;
+    client->sioc = sioc;
+    /*
+     * increase the object reference, so cioc will not freed by
+     * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))
+     */
+    object_ref(OBJECT(client->sioc));
+    qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(QIO_CHANNEL(client->sioc), false, NULL);
+    client->closed = false;
+    QTAILQ_INSERT_TAIL(&client->blk->clients, client, next);
+    vu_client_start(client);
+}
+
+
+void
+vub_initialize_config(BlockDriverState *bs, struct virtio_blk_config *config)
+{
+    config->capacity = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    config->blk_size = BDRV_SECTOR_SIZE;
+    config->size_max = 65536;
+    config->seg_max = 128 - 2;
+    config->min_io_size = 1;
+    config->opt_io_size = 1;
+    config->num_queues = 1;
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    config->max_discard_sectors = 32768;
+    config->max_discard_seg = 1;
+    config->discard_sector_alignment = config->blk_size >> 9;
+    config->max_write_zeroes_sectors = 32768;
+    config->max_write_zeroes_seg = 1;
+    #endif
+}
+
+
+static VubDev *vub_new(VubDev *vub_device, const char *name,
+                       const char *unix_socket, bool writable, Error **errp)
+{
+
+    BlockBackend *blk;
+
+    /*
+     * Don't allow resize while the vhost user server is running,

blockdev-vu.c line 767

Quoted 10 lines
blockdev-vu.c new file +10 -0
+     * otherwise we don't care what happens with the node.
+     */
+    uint64_t perm = BLK_PERM_CONSISTENT_READ;
+    int ret;
+
+    AioContext *ctx;
+
+    BlockDriverState *bs = bdrv_lookup_bs(name,
+                                          name,
+                                          errp);

This fits in a single line.

blockdev-vu.c line 773

Quoted 6 lines
blockdev-vu.c new file +6 -0
+
+    if (!bs) {
+        error_setg(errp,
+                   "No drive with name '%s'."
+                   " Please find the list of names with "
+                   "'info block'", name);

This can probably be two lines instead of four.

Quoted 161 lines
blockdev-vu.c new file +161 -0
+        return NULL;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    if (writable) {
+        perm |= BLK_PERM_WRITE;
+    }
+
+    ctx = bdrv_get_aio_context(bs);
+    aio_context_acquire(ctx);
+    bdrv_invalidate_cache(bs, NULL);
+    aio_context_release(ctx);
+
+    blk = blk_new(bdrv_get_aio_context(bs), perm,
+                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
+                  BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(blk, bs, errp);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+
+    blk_set_enable_write_cache(blk, false);
+
+    blk_set_allow_aio_context_change(blk, true);
+
+
+    vub_device->name = g_strdup(name);
+    vub_device->unix_socket = g_strdup(unix_socket);
+    vub_device->writable = writable;
+    vub_device->blkcfg.wce = 0;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    return vub_device;
+
+fail:
+    blk_unref(blk);
+    return NULL;
+}
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM)
+{
+    if (!vub_device) {
+        return;
+    }
+
+    VuClient *client, *next;
+    QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) {
+        if (!client->closed) {
+            close_client(client);
+        }
+    }
+
+    if (vub_device->listener) {
+        qio_net_listener_disconnect(vub_device->listener);
+        object_unref(OBJECT(vub_device->listener));
+    }
+    vub_free(vub_device, called_by_QOM);
+
+}
+
+
+VubDev *vub_dev_find(const char *name)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(name, vub_device->name) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+
+static VubDev *vub_dev_find_by_unix_socket(const char *unix_socket)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(unix_socket, vub_device->unix_socket) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+static void vhost_user_server_start(VubDev *vub_device, const char *unix_socket,
+                                    const char *name, bool writable,
+                                    Error **errp)
+{
+
+    if (vub_dev_find(name) || vub_dev_find_by_unix_socket(unix_socket)) {
+        error_setg(errp, "Vhost user server with name '%s' or "
+                "with socket_path '%s' has already been started",
+                name, unix_socket);
+        return;
+    }
+
+
+    if (!vub_new(vub_device, name, unix_socket, writable, errp)) {
+        return;
+    }
+
+
+    vub_device->listener = qio_net_listener_new();
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);
+    addr->u.q_unix.path = (char *) unix_socket;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp) < 0) {
+        goto error;
+    }
+
+
+    QTAILQ_INSERT_TAIL(&vub_devs, vub_device, next);
+    QTAILQ_INIT(&vub_device->clients);
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    return;
+
+ error:
+    vub_free(vub_device, false);
+}
+
+static void vu_set_block_name(Object *obj, const char *value,
+                                           Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->name) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+
+    vus->name = g_strdup(value);
+}
+
+static char *vu_get_block_name(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    return g_strdup(vus->name);
+}
+
+
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;

blockdev-vu.c line 944

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);
+}

This makes the unix-socket property magic in that it starts the server with the properties specified at this point. This means that this property must always be specified last.

Maybe it would be better to use a boolean property (similar to qdev's "realized") that explicitly start and possibly stops the export.

Writing to other properties should probably result in an error while the server is already running because these property changes won't take effect any more then.

Quoted 23 lines
blockdev-vu.c new file +23 -0
+static char *vu_get_unix_socket(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return g_strdup(vus->unix_socket);
+}
+
+static bool vu_get_block_writable(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return vus->writable;
+}
+
+static void vu_set_block_writable(Object *obj, bool value, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+
+    vus->writable = value;
+}
+
+static void vhost_user_server_instance_init(Object *obj)
+{
+
+    object_property_add_bool(obj, "writable",

blockdev-vu.c line 978

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                            vu_get_block_writable,
+                            vu_set_block_writable, NULL);
+
+    object_property_add_str(obj, "name",
+                            vu_get_block_name,
+                            vu_set_block_name, NULL);
+
+    object_property_add_str(obj, "unix_socket",
+                            vu_get_unix_socket,
+                            vu_set_unix_socket, NULL);

These should probably be object_class_property_add_*() and be called in .class_init rather than .instance_init.

"name" suggests that it's the name of the export rather than the block device to be exported. I would suggest "node-name" (and then actually only pass it as node-name to bdrv_lookup_bs()).

I expect that in the long run, we'll want to accept a full SocketAddress rather than just a filename like in "unix_socket".

Quoted 19 lines
blockdev-vu.c new file +19 -0
+}
+
+static void vhost_user_server_instance_finalize(Object *obj)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    vhost_user_server_free(vus, true);
+    /* object_del shouldn't free this object struct */
+    obj->free = NULL;
+}
+
+static const TypeInfo vhost_user_server_info = {
+    .name = TYPE_VHOST_USER_SERVER,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(VuDev),
+    .instance_init = vhost_user_server_instance_init,
+    .instance_finalize = vhost_user_server_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        {TYPE_USER_CREATABLE},
+        {}

blockdev-vu.c line 1008

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    },
+};
+
+static void vhost_user_server_register_types(void)
+{
+    type_register_static(&vhost_user_server_info);
+}
+
+type_init(vhost_user_server_register_types)
+

Extra empty line at the file end.

In summary, I can see this going in the right direction, though in detail some more work will be needed.

Kevin

author
#6 lore raw

Re: [PATCH v2 1/5] vhost-user block device backend

Hi Kevin,

Thank you for reviewing my work in a rather detailed way.

Quoted 8 lines
Quoted 6 lines
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h

This adds a single, relatively big source file. I see at least two

parts: The generic vhost-user infrastructure with connection handling etc. and the implementation of the specific vhost-user-blk device. Separating these into two files is probably a good idea.

Quoted 1 line

I would also suggest to put the files in a new subdirectory

block/export/ and call them vhost-user.c/vhost-user-blk.c. The new header file can be in the same directory as it shouldn't be used by anyone else.

I've split blockdev-vu.c in two separate files but in a different subdirectory
- backends/vhost-user-blk-server.c
- util/vhost-user-server.c

Quoted 9 lines

blockdev-vu.c line 32

Quoted 5 lines
blockdev-vu.c new file +5 -0
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;

Maybe worth a comment that this was allocated with plain malloc(), so you must use free() rather than g_free() (which would be the default in QEMU)?

Although VuVirtqElement is created using malloc, VubReq is created using g_new0.

I missed several suggestions in v3 but in v4 all suggestions have been applied. Thank you!

Quoted 1,209 lines · Kevin Wolf

Hi,

I'm only doing a quick first review pointing out the more obvious things while I familiarise myself with your code. I intend to review it in more detail later (either in a second pass for this series, or when you post v3).

Am 14.01.2020 um 15:06 hat Coiby Xu geschrieben:

Quoted 3 lines

By making use of libvhost, multiple block device drives can be exported and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.

Please wrap the commit message at 72 characters.

Quoted 6 lines
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h

This adds a single, relatively big source file. I see at least two
parts: The generic vhost-user infrastructure with connection handling etc. and the implementation of the specific vhost-user-blk device. Separating these into two files is probably a good idea.

I would also suggest to put the files in a new subdirectory block/export/ and call them vhost-user.c/vhost-user-blk.c. The new header file can be in the same directory as it shouldn't be used by anyone else.

blockdev-vu.c

Quoted 6 lines
blockdev-vu.c new file +0 -0
new file mode 100644
index 0000000000..45f0bb43a7
@@ -0,0 +1,1008 @@

The LICENSE file clarifies that files without a license header are GPLv2+, so it's not strictly a problem, but I think it is good style to include a license header that explicitly tells so.

Quoted 8 lines
blockdev-vu.c new file +8 -0
+#include "qemu/osdep.h"
+#include "block/vhost-user.h"
+#include "qapi/error.h"
+#include "qapi/qapi-types-sockets.h"
+#include "qapi/qapi-commands-block.h"
+
+#include "sysemu/block-backend.h"
+#include "qemu/main-loop.h"

blockdev-vu.c line 18

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+#include "qemu/units.h"
+
+#include "block/block.h"
+
+#include "qom/object_interfaces.h"
+
+#include <sys/eventfd.h>
+
+#include "hw/qdev-properties.h"

Does the order of includes and the empty lines between them signify anything? If not, I suggest just sorting them alphabetically (and maybe using empty lines between different subdirectories if you like this better than a single large block).

According to CODING_STYLE.rst, system headers like <sys/eventfd.h> come before all QEMU headers (except qemu/osdep.h, which always must come first).

Quoted 4 lines
blockdev-vu.c new file +4 -0
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};
+

blockdev-vu.c line 32

Quoted 10 lines
blockdev-vu.c new file +10 -0
+struct virtio_blk_inhdr {
+    unsigned char status;
+};
+
+
+static QTAILQ_HEAD(, VubDev) vub_devs = QTAILQ_HEAD_INITIALIZER(vub_devs);
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;

Maybe worth a comment that this was allocated with plain malloc(), so you must use free() rather than g_free() (which would be the default in QEMU)?

blockdev-vu.c line 39

Quoted 7 lines
blockdev-vu.c new file +7 -0
+    int64_t sector_num;
+    size_t size;
+    struct virtio_blk_inhdr *in;
+    struct virtio_blk_outhdr out;
+    VuClient *client;
+    struct VuVirtq *vq;
+} VubReq;

I'm not completely sure yet, but I think I would prefer VuBlock to Vub in the type names. Some may even prefer VhostUserBlock, but I can see that this would be quite lengthy.

Quoted 6 lines
blockdev-vu.c new file +6 -0
+static void
+remove_watch(VuDev *vu_dev, int fd)
+{
+    VuClient *client;
+
+    g_assert(vu_dev);

blockdev-vu.c line 56

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    g_assert(fd >= 0);
+
+    client = container_of(vu_dev, VuClient, parent);
+    aio_set_fd_handler(client->blk->ctx, fd, false, NULL, NULL, NULL, NULL);
+}
+
+static void close_client(VuClient *client)
+{
+    vu_deinit(&client->parent);
+    /** g_source_destroy(vub_device->parent.src); */

Leftover from conversion?

blockdev-vu.c line 63

Quoted 7 lines
blockdev-vu.c new file +7 -0
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->closed = true;
+
+}
+
+static void vub_panic_cb(VuDev *vu_dev, const char *buf)

You use a lot of sprintf() before calling this function. Would it be worth taking a printf-like format parameter instead of buf and using a variable argument list?

blockdev-vu.c line 66

Quoted 3 lines
blockdev-vu.c new file +3 -0
+{
+    if (buf) {
+        g_warning("vu_panic: %s", buf);

I think QEMU proper doesn't use g_warning() anywhere. This could be error_report() or warn_report(). (Or if you use a format string error_vreport() and warn_vreport().)

Quoted 9 lines
blockdev-vu.c new file +9 -0
+    }
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    if (client->blk->exit_panic) {
+        client->blk->close = true;
+    }
+    if (!client->closed) {
+        close_client(client);
+    }

blockdev-vu.c line 85

Quoted 10 lines
blockdev-vu.c new file +10 -0
+}
+
+
+static void vub_req_complete(VubReq *req)
+{
+    VuDev *vu_dev = &req->client->parent;
+
+    /* IO size with 1 extra status byte */
+    vu_queue_push(vu_dev, req->vq, req->elem,
+                  req->size + 1);

I think this fits in a single line.

Quoted 13 lines
blockdev-vu.c new file +13 -0
+    vu_queue_notify(vu_dev, req->vq);
+
+    if (req->elem) {
+        free(req->elem);
+    }
+
+    g_free(req);
+}
+
+
+
+static int
+vub_discard_write_zeroes(VubReq *req, struct iovec *iov, uint32_t iovcnt,

blockdev-vu.c line 108

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                         uint32_t type)
+{
+    struct virtio_blk_discard_write_zeroes *desc;
+    ssize_t size;
+    void *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (size != sizeof(*desc)) {
+        fprintf(stderr, "Invalid size %ld, expect %ld\n", size, sizeof(*desc));
+        return -1;

This would be error_report(), too. (More cases below, I'll ignore them now.)

I would prefer consistent use of -errno instead of -1 for error cases if you don't mind. I guess this would be -EINVAL here. I won't mention it for all the other cases; if you want to make the change, you need to make it everywhere, obviously.

blockdev-vu.c line 112

Quoted 4 lines
blockdev-vu.c new file +4 -0
+    }
+    buf = g_new0(char, size);
+
+    iov_to_buf_full(iov, iovcnt, 0, buf, size);

I think uint8_t describes better than char what we want here: A buffer of bytes.

The empty line would make more sense to me above the g_new0() line than after it because it starts a new section that deals with the buffer. In general, the use of empty lines feels a bit inconsistent in this patch. You may want to go over them again.

blockdev-vu.c line 113

Quoted 2 lines
blockdev-vu.c new file +2 -0
+
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)

Preprocessor directives should be unindented.

However, I don't think any of this code actually depends on Linux, BLKDISCARD or BLKZEROOUT. You can just call blk_pdiscard() and blk_pwrite_zeroes() and they will do whatever is necessary to perform the operation on the backend (which might not be a Linux block device, but could be a regular file or even using a network protocol like NBD).

Quoted 6 lines
blockdev-vu.c new file +6 -0
+    VubDev *vdev_blk;
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+    desc = (struct virtio_blk_discard_write_zeroes *)buf;
+    uint64_t range[2] = { le64toh(desc->sector) << 9,
+                          le32toh(desc->num_sectors) << 9 };

blockdev-vu.c line 131

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    if (type == VIRTIO_BLK_T_DISCARD) {
+        if (blk_pdiscard(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    } else if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
+        if (blk_pwrite_zeroes(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }

blk_pdiscard() and blk_pwrite_zeroes() are synchronous functions. In other words, the guest will be blocked until the I/O is complete. We cannot do this.

I think you should let vub_virtio_process_req() run in a coroutine so that you can call blk_co_pdiscard() and blk_co_pwrite_zeroes() here.

blockdev-vu.c line 144

Quoted 13 lines
blockdev-vu.c new file +13 -0
+    }
+    #endif
+
+    g_free(buf);
+    return -1;
+}
+
+
+static void
+vub_flush(VubReq *req)
+{
+    VuClient *client = req->client;
+    blk_co_flush(client->blk->backend);

You can't call blk_co_flush() from outside coroutine context. This code will be right after you move vub_virtio_process_req() to a coroutine, though (which will make this function a coroutine_fn).

Quoted 47 lines
blockdev-vu.c new file +47 -0
+}
+
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    void *iobuf;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static int blk_prw(BlockBackend *blk, QEMUIOVector *qiov, int64_t offset,
+                   CoroutineEntry co_entry, BdrvRequestFlags flags)
+{
+
+    BlkRwCo rwco = {
+        .blk    = blk,
+        .offset = offset,
+        .iobuf  = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {

blockdev-vu.c line 201

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        /* Fast-path if already in coroutine context */
+        co_entry(&rwco);
+    } else {
+        Coroutine *co = qemu_coroutine_create(co_entry, &rwco);
+        bdrv_coroutine_enter(blk_bs(blk), co);
+        BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
+    }
+
+    return rwco.ret;
+}

This is copy&paste from block-backend.c. We should certainly not do this. I think it will automatically go away when you can use blk_co_preadv() and blk_co_pwritev() directly.

Note that the BDRV_POLL_WHILE() means that like above, we would be waiting for the request to complete. This would block the guest and would also not allow parallel requests, killing the I/O performance of our vhost-user export.

blockdev-vu.c line 207

Quoted 5 lines
blockdev-vu.c new file +5 -0
+
+static ssize_t
+vub_rwv(VubReq *req, struct iovec *iov,
+        uint32_t iovcnt,
+        CoroutineEntry co_entry)

I don't understand the line wrapping here. :-)

Quoted 5 lines
blockdev-vu.c new file +5 -0
+{
+    VuClient *client = req->client;
+    ssize_t rc;
+
+    if (!iovcnt) {

blockdev-vu.c line 222

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        fprintf(stderr, "Invalid Read/Write IOV count\n");
+        return -1;
+    }
+
+    int64_t offset = req->sector_num * 512;
+    QEMUIOVector *qiov = g_new0(QEMUIOVector, 1);
+    qemu_iovec_init_external(qiov, iov, iovcnt);
+    rc = blk_prw(client->blk->backend, qiov, offset, co_entry, 0);
+
+    req->size = iov_size(iov, iovcnt);

You can use qiov->size instead of duplicating this information into a separate VubReq field.

blockdev-vu.c line 234

Quoted 12 lines
blockdev-vu.c new file +12 -0
+    if (rc < 0) {
+        fprintf(stderr, "%s, Sector %"PRIu64", Size %lu failed with %s\n",
+                client->blk->name, req->sector_num, req->size,
+                strerror(errno));
+        return -1;
+    }
+
+    return rc;
+}
+
+static int vub_virtio_process_req(VuClient *client,
+                                     VuVirtq *vq)

Indentation is off. This could be a single line anyway.

Quoted 117 lines
blockdev-vu.c new file +117 -0
+{
+    VuDev *vu_dev = &client->parent;
+    VuVirtqElement *elem;
+    uint32_t type;
+    VubReq *req;
+
+    elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement) + sizeof(VubReq));
+    if (!elem) {
+        return -1;
+    }
+
+    struct iovec *in_iov = elem->in_sg;
+    struct iovec *out_iov = elem->out_sg;
+    unsigned in_num = elem->in_num;
+    unsigned out_num = elem->out_num;
+    /* refer to hw/block/virtio_blk.c */
+    if (elem->out_num < 1 || elem->in_num < 1) {
+        fprintf(stderr, "virtio-blk request missing headers\n");
+        free(elem);
+        return -1;
+    }
+
+    req = g_new0(VubReq, 1);
+    req->client = client;
+    req->vq = vq;
+    req->elem = elem;
+
+    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req->out,
+                            sizeof(req->out)) != sizeof(req->out))) {
+        fprintf(stderr, "virtio-blk request outhdr too short");
+        goto err;
+    }
+
+    iov_discard_front(&out_iov, &out_num, sizeof(req->out));
+
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+        fprintf(stderr, "virtio-blk request inhdr too short");
+        goto err;
+    }
+
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in = (void *)in_iov[in_num - 1].iov_base
+              + in_iov[in_num - 1].iov_len
+              - sizeof(struct virtio_blk_inhdr);
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+
+    type = le32toh(req->out.type);
+    switch (type & ~VIRTIO_BLK_T_BARRIER) {
+    case VIRTIO_BLK_T_IN:
+    case VIRTIO_BLK_T_OUT: {
+        ssize_t ret = 0;
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = le64toh(req->out.sector);
+        if (is_write) {
+            ret = vub_rwv(req, out_iov, out_num, blk_write_entry);
+        } else {
+            ret = vub_rwv(req, in_iov, in_num, blk_read_entry);
+        }
+        if (ret >= 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
+        vub_flush(req);
+        req->in->status = VIRTIO_BLK_S_OK;
+        vub_req_complete(req);
+        break;
+    case VIRTIO_BLK_T_GET_ID: {
+        size_t size = MIN(iov_size(&elem->in_sg[0], in_num),
+                          VIRTIO_BLK_ID_BYTES);
+        snprintf(elem->in_sg[0].iov_base, size, "%s", "vhost_user_blk");
+        req->in->status = VIRTIO_BLK_S_OK;
+        req->size = elem->in_sg[0].iov_len;
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_DISCARD:
+    case VIRTIO_BLK_T_WRITE_ZEROES: {
+        int rc;
+        rc = vub_discard_write_zeroes(req, &elem->out_sg[1], out_num, type);
+        if (rc == 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    default:
+        req->in->status = VIRTIO_BLK_S_UNSUPP;
+        vub_req_complete(req);
+        break;
+    }
+
+    return 0;
+
+err:
+    free(elem);
+    g_free(req);
+    return -1;
+}
+
+
+static void vub_process_vq(VuDev *vu_dev, int idx)
+{
+    VuClient *client;
+    VuVirtq *vq;
+    int ret;
+
+    client = container_of(vu_dev, VuClient, parent);
+    assert(client);
+

blockdev-vu.c line 361

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    vq = vu_get_queue(vu_dev, idx);
+    assert(vq);
+
+    while (1) {
+        ret = vub_virtio_process_req(client, vq);
+        if (ret) {
+            break;
+        }
+    }
+}

I mentioned above moving vub_virtio_process_req() into a coroutine. Just creating a coroutine here and entering it wouldn't work, though.

The best design is probably to get requests from the virtqueue in this function (what is currently the first half of vub_virtio_process_req()) and then spawn a coroutine per request to actually execute them (roughly the switch in vub_virtio_process_req()).

This way you'll get parallel requests and won't have to think about synchronising accesses to the virtqueue from multiple coroutines.

Quoted 86 lines
blockdev-vu.c new file +86 -0
+
+static void vub_queue_set_started(VuDev *vu_dev, int idx, bool started)
+{
+    VuVirtq *vq;
+
+    assert(vu_dev);
+
+    vq = vu_get_queue(vu_dev, idx);
+    vu_set_queue_handler(vu_dev, vq, started ? vub_process_vq : NULL);
+}
+
+static uint64_t
+vub_get_features(VuDev *dev)
+{
+    uint64_t features;
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+
+    features = 1ull << VIRTIO_BLK_F_SIZE_MAX |
+               1ull << VIRTIO_BLK_F_SEG_MAX |
+               1ull << VIRTIO_BLK_F_TOPOLOGY |
+               1ull << VIRTIO_BLK_F_BLK_SIZE |
+               1ull << VIRTIO_BLK_F_FLUSH |
+               #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+               1ull << VIRTIO_BLK_F_DISCARD |
+               1ull << VIRTIO_BLK_F_WRITE_ZEROES |
+               #endif
+               1ull << VIRTIO_BLK_F_CONFIG_WCE |
+               1ull << VIRTIO_F_VERSION_1 |
+               1ull << VIRTIO_RING_F_INDIRECT_DESC |
+               1ull << VIRTIO_RING_F_EVENT_IDX |
+               1ull << VHOST_USER_F_PROTOCOL_FEATURES;
+
+    if (!vdev_blk->writable) {
+        features |= 1ull << VIRTIO_BLK_F_RO;
+    }
+
+    return features;
+}
+
+static uint64_t
+vub_get_protocol_features(VuDev *dev)
+{
+    return 1ull << VHOST_USER_PROTOCOL_F_CONFIG |
+           1ull << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD;
+}
+
+static int
+vub_get_config(VuDev *vu_dev, uint8_t *config, uint32_t len)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    memcpy(config, &vdev_blk->blkcfg, len);
+
+    return 0;
+}
+
+static int
+vub_set_config(VuDev *vu_dev, const uint8_t *data,
+               uint32_t offset, uint32_t size, uint32_t flags)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    uint8_t wce;
+
+    /* don't support live migration */
+    if (flags != VHOST_SET_CONFIG_TYPE_MASTER) {
+        return -1;
+    }
+
+
+    if (offset != offsetof(struct virtio_blk_config, wce) ||
+        size != 1) {
+        return -1;
+    }
+
+    wce = *data;
+    if (wce == vdev_blk->blkcfg.wce) {
+        /* Do nothing as same with old configuration */
+        return 0;

blockdev-vu.c line 458

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    }
+
+    vdev_blk->blkcfg.wce = wce;
+    blk_set_enable_write_cache(vdev_blk->backend, true);
+    return 0;
+}
+
+
+/*
+ * When the client disconnects, it send a VHOST_USER_NONE request

s/send/sends/

Quoted 36 lines
blockdev-vu.c new file +36 -0
+ * and vu_process_message will simple call exit which cause the VM
+ * to exit abruptly.
+ * To avoid this issue,  process VHOST_USER_NONE request ahead
+ * of vu_process_message.
+ *
+ */
+static int vub_process_msg(VuDev *dev, VhostUserMsg *vmsg, int *do_reply)
+{
+    if (vmsg->request == VHOST_USER_NONE) {
+        dev->panic(dev, "disconnect");
+        return true;
+    }
+    return false;
+}
+
+static void
+vmsg_close_fds(VhostUserMsg *vmsg)
+{
+    int i;
+    for (i = 0; i < vmsg->fd_num; i++) {
+        close(vmsg->fds[i]);
+    }
+}
+
+static bool
+vu_message_read_co(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,

blockdev-vu.c line 504

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+    char buffer[100];
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    QIOChannel *ioc = client->ioc;
+    do {
+        rc = recvmsg(conn_fd, &msg, 0);

This should certainly use qio_channel_readv_full() rather than working directly with a socket fd?

Quoted 12 lines
blockdev-vu.c new file +12 -0
+        if (rc < 0) {
+            if (errno == EAGAIN) {
+                if (qemu_in_coroutine()) {
+                    qio_channel_yield(ioc, G_IO_IN);
+                } else {
+                    qio_channel_wait(ioc, G_IO_IN);
+                }
+                continue;
+            } else if (errno == EINTR) {
+                continue;
+            }
+        }

blockdev-vu.c line 526

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        break;
+    } while (true);
+
+    if (rc < 0) {
+        sprintf(buffer, "Error while recvmsg: %s", strerror(errno));
+        vub_panic_cb(vu_dev, buffer);
+        return false;
+    }
+
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);

Why do you think that there can't be short reads?

blockdev-vu.c line 539

Quoted 12 lines
blockdev-vu.c new file +12 -0
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }

I think the fd passing part becomes easier when you use the proper qio_channel_readv_full() function. Its implementation is also a bit more careful than yours. For example, you forgot checking fd_size against VHOST_MEMORY_MAX_NREGIONS, allowing a buffer overflow in the memcpy(), and you don't adjust fd flags for the new file descriptors.

blockdev-vu.c line 552

Quoted 12 lines
blockdev-vu.c new file +12 -0
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        sprintf(buffer,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        vub_panic_cb(vu_dev, buffer);
+        goto fail;
+    }
+
+    if (vmsg->size) {
+        do {
+            rc = read(conn_fd, &vmsg->payload, vmsg->size);

qio_channel_readv_all_eof() already implements this whole loop and correctly handles short reads, too.

Quoted 56 lines
blockdev-vu.c new file +56 -0
+            if (rc < 0) {
+                if (errno == EAGAIN) {
+                    if (qemu_in_coroutine()) {
+                        qio_channel_yield(ioc, G_IO_IN);
+                    } else {
+                        qio_channel_wait(ioc, G_IO_IN);
+                    }
+                    continue;
+                } else if (errno == EINTR) {
+                    continue;
+                }
+            }
+            break;
+        } while (true);
+
+        if (rc <= 0) {
+            sprintf(buffer, "Error while reading: %s", strerror(errno));
+            vub_panic_cb(vu_dev, buffer);
+            goto fail;
+        }
+
+        assert(rc == vmsg->size);
+    }
+
+    return true;
+
+fail:
+    vmsg_close_fds(vmsg);
+
+    return false;
+}
+
+static void vub_kick_cb(void *opaque)
+{
+    vu_watch_cb_data *data = (vu_watch_cb_data *) opaque;
+    int index = data->index;
+    VuDev *dev = data->vu_dev;
+    VuVirtq *vq = &dev->vq[index];
+    int sock = vq->kick_fd;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        char buffer[100];
+        sprintf(buffer, "kick eventfd_read(): %s", strerror(errno));
+        vub_panic_cb(dev, buffer);
+        g_free(data);
+        dev->remove_watch(dev, dev->vq[index].kick_fd);
+    } else {
+        if (vq->handler) {
+            vq->handler(dev, index);
+        }
+    }
+}
+

blockdev-vu.c line 618

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static const VuDevIface vub_iface = {
+    .get_features = vub_get_features,
+    .queue_set_started = vub_queue_set_started,
+    .get_protocol_features = vub_get_protocol_features,
+    .get_config = vub_get_config,
+    .set_config = vub_set_config,
+    .process_msg = vub_process_msg,
+    .read_msg = vu_message_read_co,
+    .kick_callback = vub_kick_cb,
+};

I would prefer the = signs to be aligned to the same column.

Quoted 28 lines
blockdev-vu.c new file +28 -0
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM)
+{
+    if (!vub_dev) {
+        return;
+    }
+
+    blk_unref(vub_dev->backend);
+    g_free(vub_dev->name);
+    g_free(vub_dev->unix_socket);
+
+    if (vub_dev->next.tqe_circ.tql_prev) {
+        /*
+         * if vub_dev->next.tqe_circ.tql_prev = null,
+         * vub_dev hasn't been inserted into the queue and
+         * vub_free is called by obj->instance_finalize.
+         */
+        QTAILQ_REMOVE(&vub_devs, vub_dev, next);
+    }
+    /*
+     * Needn't to free vub_dev if called by QOM
+     * because QOM will do the clean-up work.
+     */
+    if (!called_by_QOM) {
+        g_free(vub_dev);
+    }
+}
+

blockdev-vu.c line 657

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+    VuClient *client = opaque;
+
+    while (!client->closed) {
+        vu_dispatch(&client->parent);
+    }
+
+    QTAILQ_REMOVE(&client->blk->clients, client, next);
+

Extra empty line.

Quoted 100 lines
blockdev-vu.c new file +100 -0
+}
+
+static void vu_client_start(VuClient *client)
+{
+    Coroutine *co = qemu_coroutine_create(vu_client_trip, client);
+    qemu_coroutine_enter(co);
+}
+
+
+G_STATIC_ASSERT((int)G_IO_IN == (int)VU_WATCH_IN);
+G_STATIC_ASSERT((int)G_IO_OUT == (int)VU_WATCH_OUT);
+G_STATIC_ASSERT((int)G_IO_PRI == (int)VU_WATCH_PRI);
+G_STATIC_ASSERT((int)G_IO_ERR == (int)VU_WATCH_ERR);
+G_STATIC_ASSERT((int)G_IO_HUP == (int)VU_WATCH_HUP);
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+          vu_watch_cb_packed_data cb, void *pvt)
+{
+    /*
+     * since aio_dispatch can only pass one user data pointer to the
+     * callback function, pack VuDev, pvt into a struct
+     */
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+    g_assert(cb);
+    client = container_of(vu_dev, VuClient, parent);
+    vu_watch_cb_data *cb_data = g_new0(vu_watch_cb_data, 1);
+    cb_data->index = (intptr_t) pvt;
+    cb_data->vu_dev = vu_dev;
+    aio_set_fd_handler(client->blk->ctx, fd, false, (void *) cb,
+                       NULL, NULL, cb_data);
+}
+
+
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);
+
+    if (!vu_init_packed_data(&client->parent, VHOST_USER_BLK_MAX_QUEUES,
+                             sioc->fd, vub_panic_cb, set_watch,
+                             remove_watch, &vub_iface)) {
+        fprintf(stderr, "Failed to initialized libvhost-user\n");
+        g_free(client);
+        return;
+    }
+
+    client->blk = vub_device;
+    client->refcount = 1;
+    client->sioc = sioc;
+    /*
+     * increase the object reference, so cioc will not freed by
+     * qio_net_listener_channel_func which will call object_unref(OBJECT(sioc))
+     */
+    object_ref(OBJECT(client->sioc));
+    qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(QIO_CHANNEL(client->sioc), false, NULL);
+    client->closed = false;
+    QTAILQ_INSERT_TAIL(&client->blk->clients, client, next);
+    vu_client_start(client);
+}
+
+
+void
+vub_initialize_config(BlockDriverState *bs, struct virtio_blk_config *config)
+{
+    config->capacity = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    config->blk_size = BDRV_SECTOR_SIZE;
+    config->size_max = 65536;
+    config->seg_max = 128 - 2;
+    config->min_io_size = 1;
+    config->opt_io_size = 1;
+    config->num_queues = 1;
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    config->max_discard_sectors = 32768;
+    config->max_discard_seg = 1;
+    config->discard_sector_alignment = config->blk_size >> 9;
+    config->max_write_zeroes_sectors = 32768;
+    config->max_write_zeroes_seg = 1;
+    #endif
+}
+
+
+static VubDev *vub_new(VubDev *vub_device, const char *name,
+                       const char *unix_socket, bool writable, Error **errp)
+{
+
+    BlockBackend *blk;
+
+    /*
+     * Don't allow resize while the vhost user server is running,

blockdev-vu.c line 767

Quoted 10 lines
blockdev-vu.c new file +10 -0
+     * otherwise we don't care what happens with the node.
+     */
+    uint64_t perm = BLK_PERM_CONSISTENT_READ;
+    int ret;
+
+    AioContext *ctx;
+
+    BlockDriverState *bs = bdrv_lookup_bs(name,
+                                          name,
+                                          errp);

This fits in a single line.

blockdev-vu.c line 773

Quoted 6 lines
blockdev-vu.c new file +6 -0
+
+    if (!bs) {
+        error_setg(errp,
+                   "No drive with name '%s'."
+                   " Please find the list of names with "
+                   "'info block'", name);

This can probably be two lines instead of four.

Quoted 161 lines
blockdev-vu.c new file +161 -0
+        return NULL;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    if (writable) {
+        perm |= BLK_PERM_WRITE;
+    }
+
+    ctx = bdrv_get_aio_context(bs);
+    aio_context_acquire(ctx);
+    bdrv_invalidate_cache(bs, NULL);
+    aio_context_release(ctx);
+
+    blk = blk_new(bdrv_get_aio_context(bs), perm,
+                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
+                  BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(blk, bs, errp);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+
+    blk_set_enable_write_cache(blk, false);
+
+    blk_set_allow_aio_context_change(blk, true);
+
+
+    vub_device->name = g_strdup(name);
+    vub_device->unix_socket = g_strdup(unix_socket);
+    vub_device->writable = writable;
+    vub_device->blkcfg.wce = 0;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    return vub_device;
+
+fail:
+    blk_unref(blk);
+    return NULL;
+}
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM)
+{
+    if (!vub_device) {
+        return;
+    }
+
+    VuClient *client, *next;
+    QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) {
+        if (!client->closed) {
+            close_client(client);
+        }
+    }
+
+    if (vub_device->listener) {
+        qio_net_listener_disconnect(vub_device->listener);
+        object_unref(OBJECT(vub_device->listener));
+    }
+    vub_free(vub_device, called_by_QOM);
+
+}
+
+
+VubDev *vub_dev_find(const char *name)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(name, vub_device->name) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+
+static VubDev *vub_dev_find_by_unix_socket(const char *unix_socket)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(unix_socket, vub_device->unix_socket) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+static void vhost_user_server_start(VubDev *vub_device, const char *unix_socket,
+                                    const char *name, bool writable,
+                                    Error **errp)
+{
+
+    if (vub_dev_find(name) || vub_dev_find_by_unix_socket(unix_socket)) {
+        error_setg(errp, "Vhost user server with name '%s' or "
+                "with socket_path '%s' has already been started",
+                name, unix_socket);
+        return;
+    }
+
+
+    if (!vub_new(vub_device, name, unix_socket, writable, errp)) {
+        return;
+    }
+
+
+    vub_device->listener = qio_net_listener_new();
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);
+    addr->u.q_unix.path = (char *) unix_socket;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp) < 0) {
+        goto error;
+    }
+
+
+    QTAILQ_INSERT_TAIL(&vub_devs, vub_device, next);
+    QTAILQ_INIT(&vub_device->clients);
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    return;
+
+ error:
+    vub_free(vub_device, false);
+}
+
+static void vu_set_block_name(Object *obj, const char *value,
+                                           Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->name) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+
+    vus->name = g_strdup(value);
+}
+
+static char *vu_get_block_name(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    return g_strdup(vus->name);
+}
+
+
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;

blockdev-vu.c line 944

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);
+}

This makes the unix-socket property magic in that it starts the server with the properties specified at this point. This means that this property must always be specified last.

Maybe it would be better to use a boolean property (similar to qdev's "realized") that explicitly start and possibly stops the export.

Writing to other properties should probably result in an error while the server is already running because these property changes won't take effect any more then.

Quoted 23 lines
blockdev-vu.c new file +23 -0
+static char *vu_get_unix_socket(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return g_strdup(vus->unix_socket);
+}
+
+static bool vu_get_block_writable(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return vus->writable;
+}
+
+static void vu_set_block_writable(Object *obj, bool value, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+
+    vus->writable = value;
+}
+
+static void vhost_user_server_instance_init(Object *obj)
+{
+
+    object_property_add_bool(obj, "writable",

blockdev-vu.c line 978

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                            vu_get_block_writable,
+                            vu_set_block_writable, NULL);
+
+    object_property_add_str(obj, "name",
+                            vu_get_block_name,
+                            vu_set_block_name, NULL);
+
+    object_property_add_str(obj, "unix_socket",
+                            vu_get_unix_socket,
+                            vu_set_unix_socket, NULL);

These should probably be object_class_property_add_*() and be called in .class_init rather than .instance_init.

"name" suggests that it's the name of the export rather than the block device to be exported. I would suggest "node-name" (and then actually only pass it as node-name to bdrv_lookup_bs()).

I expect that in the long run, we'll want to accept a full SocketAddress rather than just a filename like in "unix_socket".

Quoted 19 lines
blockdev-vu.c new file +19 -0
+}
+
+static void vhost_user_server_instance_finalize(Object *obj)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    vhost_user_server_free(vus, true);
+    /* object_del shouldn't free this object struct */
+    obj->free = NULL;
+}
+
+static const TypeInfo vhost_user_server_info = {
+    .name = TYPE_VHOST_USER_SERVER,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(VuDev),
+    .instance_init = vhost_user_server_instance_init,
+    .instance_finalize = vhost_user_server_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        {TYPE_USER_CREATABLE},
+        {}

blockdev-vu.c line 1008

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    },
+};
+
+static void vhost_user_server_register_types(void)
+{
+    type_register_static(&vhost_user_server_info);
+}
+
+type_init(vhost_user_server_register_types)
+

Extra empty line at the file end.

Quoted 5 lines

In summary, I can see this going in the right direction, though in detail some more work will be needed.

Kevin

--
Best regards,
Coiby

author
#7 lore raw

Re: [PATCH v2 1/5] vhost-user block device backend

In reply to Kevin Wolf
Quoted 6 lines

blockdev-vu.c

Quoted 6 lines
blockdev-vu.c new file +6 -0
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type ==

SCM_RIGHTS) {

Quoted 7 lines

blockdev-vu.c line 539

Quoted 6 lines
blockdev-vu.c new file +6 -0
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
Quoted 5 lines

I think the fd passing part becomes easier when you use the proper qio_channel_readv_full() function. Its implementation is also a bit more careful than yours. For example, you forgot checking fd_size against VHOST_MEMORY_MAX_NREGIONS, allowing a buffer overflow in the memcpy(), and you don't adjust fd flags for the new file descriptors.

Oh, I used qio_channel_readv_full in v3&v4. But I still forgot checking fd_size against VHOST_MEMORY_MAX_NREGIONS. I'll fix this buffer overflow issue in v5.

Quoted 1,237 lines · Kevin Wolf

Hi,

I'm only doing a quick first review pointing out the more obvious things while I familiarise myself with your code. I intend to review it in more detail later (either in a second pass for this series, or when you post v3).

Am 14.01.2020 um 15:06 hat Coiby Xu geschrieben:

Quoted 1 line

By making use of libvhost, multiple block device drives can be exported

and each drive can serve multiple clients simultaneously. Since vhost-user-server needs a block drive to be created first, delay the creation of this object.

Quoted 2 lines

Please wrap the commit message at 72 characters.

Quoted 6 lines
 blockdev-vu.c              | 1008 ++++++++++++++++++++++++++++++++++++
 include/block/vhost-user.h |   46 ++
 vl.c                       |    4 +
 3 files changed, 1058 insertions(+)
 create mode 100644 blockdev-vu.c
 create mode 100644 include/block/vhost-user.h

This adds a single, relatively big source file. I see at least two
parts: The generic vhost-user infrastructure with connection handling etc. and the implementation of the specific vhost-user-blk device. Separating these into two files is probably a good idea.

I would also suggest to put the files in a new subdirectory block/export/ and call them vhost-user.c/vhost-user-blk.c. The new header file can be in the same directory as it shouldn't be used by anyone else.

blockdev-vu.c

Quoted 6 lines
blockdev-vu.c new file +0 -0
new file mode 100644
index 0000000000..45f0bb43a7
@@ -0,0 +1,1008 @@

The LICENSE file clarifies that files without a license header are GPLv2+, so it's not strictly a problem, but I think it is good style to include a license header that explicitly tells so.

Quoted 8 lines
blockdev-vu.c new file +8 -0
+#include "qemu/osdep.h"
+#include "block/vhost-user.h"
+#include "qapi/error.h"
+#include "qapi/qapi-types-sockets.h"
+#include "qapi/qapi-commands-block.h"
+
+#include "sysemu/block-backend.h"
+#include "qemu/main-loop.h"

blockdev-vu.c line 18

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+#include "qemu/units.h"
+
+#include "block/block.h"
+
+#include "qom/object_interfaces.h"
+
+#include <sys/eventfd.h>
+
+#include "hw/qdev-properties.h"

Does the order of includes and the empty lines between them signify anything? If not, I suggest just sorting them alphabetically (and maybe using empty lines between different subdirectories if you like this better than a single large block).

According to CODING_STYLE.rst, system headers like <sys/eventfd.h> come before all QEMU headers (except qemu/osdep.h, which always must come first).

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+enum {
+    VHOST_USER_BLK_MAX_QUEUES = 8,
+};
+
+struct virtio_blk_inhdr {
+    unsigned char status;
+};
+
+
+static QTAILQ_HEAD(, VubDev) vub_devs =

QTAILQ_HEAD_INITIALIZER(vub_devs);

blockdev-vu.c line 32

Quoted 4 lines
blockdev-vu.c new file +4 -0
+
+
+typedef struct VubReq {
+    VuVirtqElement *elem;

Maybe worth a comment that this was allocated with plain malloc(), so you must use free() rather than g_free() (which would be the default in QEMU)?

blockdev-vu.c line 39

Quoted 7 lines
blockdev-vu.c new file +7 -0
+    int64_t sector_num;
+    size_t size;
+    struct virtio_blk_inhdr *in;
+    struct virtio_blk_outhdr out;
+    VuClient *client;
+    struct VuVirtq *vq;
+} VubReq;

I'm not completely sure yet, but I think I would prefer VuBlock to Vub in the type names. Some may even prefer VhostUserBlock, but I can see that this would be quite lengthy.

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static void
+remove_watch(VuDev *vu_dev, int fd)
+{
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+
+    client = container_of(vu_dev, VuClient, parent);
+    aio_set_fd_handler(client->blk->ctx, fd, false, NULL, NULL, NULL,

NULL);

blockdev-vu.c line 56

Quoted 6 lines
blockdev-vu.c new file +6 -0
+}
+
+static void close_client(VuClient *client)
+{
+    vu_deinit(&client->parent);
+    /** g_source_destroy(vub_device->parent.src); */

Leftover from conversion?

blockdev-vu.c line 63

Quoted 7 lines
blockdev-vu.c new file +7 -0
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->closed = true;
+
+}
+
+static void vub_panic_cb(VuDev *vu_dev, const char *buf)

You use a lot of sprintf() before calling this function. Would it be worth taking a printf-like format parameter instead of buf and using a variable argument list?

blockdev-vu.c line 66

Quoted 3 lines
blockdev-vu.c new file +3 -0
+{
+    if (buf) {
+        g_warning("vu_panic: %s", buf);

I think QEMU proper doesn't use g_warning() anywhere. This could be error_report() or warn_report(). (Or if you use a format string error_vreport() and warn_vreport().)

Quoted 9 lines
blockdev-vu.c new file +9 -0
+    }
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    if (client->blk->exit_panic) {
+        client->blk->close = true;
+    }
+    if (!client->closed) {
+        close_client(client);
+    }

blockdev-vu.c line 85

Quoted 10 lines
blockdev-vu.c new file +10 -0
+}
+
+
+static void vub_req_complete(VubReq *req)
+{
+    VuDev *vu_dev = &req->client->parent;
+
+    /* IO size with 1 extra status byte */
+    vu_queue_push(vu_dev, req->vq, req->elem,
+                  req->size + 1);

I think this fits in a single line.

blockdev-vu.c

Quoted 13 lines
blockdev-vu.c new file +13 -0
+    vu_queue_notify(vu_dev, req->vq);
+
+    if (req->elem) {
+        free(req->elem);
+    }
+
+    g_free(req);
+}
+
+
+
+static int
+vub_discard_write_zeroes(VubReq *req, struct iovec *iov, uint32_t

iovcnt,

blockdev-vu.c

Quoted 9 lines
blockdev-vu.c new file +9 -0
+                         uint32_t type)
+{
+    struct virtio_blk_discard_write_zeroes *desc;
+    ssize_t size;
+    void *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (size != sizeof(*desc)) {
+        fprintf(stderr, "Invalid size %ld, expect %ld\n", size,

sizeof(*desc));

blockdev-vu.c line 107

Quoted 1 line
blockdev-vu.c new file +1 -0
+        return -1;

This would be error_report(), too. (More cases below, I'll ignore them now.)

I would prefer consistent use of -errno instead of -1 for error cases if you don't mind. I guess this would be -EINVAL here. I won't mention it for all the other cases; if you want to make the change, you need to make it everywhere, obviously.

blockdev-vu.c line 112

Quoted 4 lines
blockdev-vu.c new file +4 -0
+    }
+    buf = g_new0(char, size);
+
+    iov_to_buf_full(iov, iovcnt, 0, buf, size);

I think uint8_t describes better than char what we want here: A buffer of bytes.

The empty line would make more sense to me above the g_new0() line than after it because it starts a new section that deals with the buffer. In general, the use of empty lines feels a bit inconsistent in this patch. You may want to go over them again.

blockdev-vu.c line 113

Quoted 2 lines
blockdev-vu.c new file +2 -0
+
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)

Preprocessor directives should be unindented.

However, I don't think any of this code actually depends on Linux, BLKDISCARD or BLKZEROOUT. You can just call blk_pdiscard() and blk_pwrite_zeroes() and they will do whatever is necessary to perform the operation on the backend (which might not be a Linux block device, but could be a regular file or even using a network protocol like NBD).

Quoted 6 lines
blockdev-vu.c new file +6 -0
+    VubDev *vdev_blk;
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+    desc = (struct virtio_blk_discard_write_zeroes *)buf;
+    uint64_t range[2] = { le64toh(desc->sector) << 9,
+                          le32toh(desc->num_sectors) << 9 };

blockdev-vu.c line 131

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    if (type == VIRTIO_BLK_T_DISCARD) {
+        if (blk_pdiscard(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }
+    } else if (type == VIRTIO_BLK_T_WRITE_ZEROES) {
+        if (blk_pwrite_zeroes(vdev_blk->blk, range[0], range[1]) == 0) {
+            g_free(buf);
+            return 0;
+        }

blk_pdiscard() and blk_pwrite_zeroes() are synchronous functions. In other words, the guest will be blocked until the I/O is complete. We cannot do this.

I think you should let vub_virtio_process_req() run in a coroutine so that you can call blk_co_pdiscard() and blk_co_pwrite_zeroes() here.

blockdev-vu.c line 144

Quoted 13 lines
blockdev-vu.c new file +13 -0
+    }
+    #endif
+
+    g_free(buf);
+    return -1;
+}
+
+
+static void
+vub_flush(VubReq *req)
+{
+    VuClient *client = req->client;
+    blk_co_flush(client->blk->backend);

You can't call blk_co_flush() from outside coroutine context. This code will be right after you move vub_virtio_process_req() to a coroutine, though (which will make this function a coroutine_fn).

blockdev-vu.c

Quoted 4 lines
blockdev-vu.c new file +4 -0
+}
+
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in

progress */

Quoted 21 lines
blockdev-vu.c new file +21 -0
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    void *iobuf;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static void blk_write_entry(void *opaque)
+{

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    BlkRwCo *rwco = opaque;
+    QEMUIOVector *qiov = rwco->iobuf;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, qiov->size,
+                              qiov, rwco->flags);
+    aio_wait_kick();
+}
+
+
+static int blk_prw(BlockBackend *blk, QEMUIOVector *qiov, int64_t

offset,

Quoted 12 lines
blockdev-vu.c new file +12 -0
+                   CoroutineEntry co_entry, BdrvRequestFlags flags)
+{
+
+    BlkRwCo rwco = {
+        .blk    = blk,
+        .offset = offset,
+        .iobuf  = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {

blockdev-vu.c line 201

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        /* Fast-path if already in coroutine context */
+        co_entry(&rwco);
+    } else {
+        Coroutine *co = qemu_coroutine_create(co_entry, &rwco);
+        bdrv_coroutine_enter(blk_bs(blk), co);
+        BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
+    }
+
+    return rwco.ret;
+}

This is copy&paste from block-backend.c. We should certainly not do this. I think it will automatically go away when you can use blk_co_preadv() and blk_co_pwritev() directly.

Note that the BDRV_POLL_WHILE() means that like above, we would be waiting for the request to complete. This would block the guest and would also not allow parallel requests, killing the I/O performance of our vhost-user export.

blockdev-vu.c line 207

Quoted 5 lines
blockdev-vu.c new file +5 -0
+
+static ssize_t
+vub_rwv(VubReq *req, struct iovec *iov,
+        uint32_t iovcnt,
+        CoroutineEntry co_entry)

I don't understand the line wrapping here. :-)

Quoted 5 lines
blockdev-vu.c new file +5 -0
+{
+    VuClient *client = req->client;
+    ssize_t rc;
+
+    if (!iovcnt) {

blockdev-vu.c line 222

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        fprintf(stderr, "Invalid Read/Write IOV count\n");
+        return -1;
+    }
+
+    int64_t offset = req->sector_num * 512;
+    QEMUIOVector *qiov = g_new0(QEMUIOVector, 1);
+    qemu_iovec_init_external(qiov, iov, iovcnt);
+    rc = blk_prw(client->blk->backend, qiov, offset, co_entry, 0);
+
+    req->size = iov_size(iov, iovcnt);

You can use qiov->size instead of duplicating this information into a separate VubReq field.

blockdev-vu.c line 221

Quoted 2 lines
blockdev-vu.c new file +2 -0
+    if (rc < 0) {
+        fprintf(stderr, "%s, Sector %"PRIu64", Size %lu failed with

%s\n",

blockdev-vu.c line 234

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                client->blk->name, req->sector_num, req->size,
+                strerror(errno));
+        return -1;
+    }
+
+    return rc;
+}
+
+static int vub_virtio_process_req(VuClient *client,
+                                     VuVirtq *vq)

Indentation is off. This could be a single line anyway.

blockdev-vu.c

Quoted 7 lines
blockdev-vu.c new file +7 -0
+{
+    VuDev *vu_dev = &client->parent;
+    VuVirtqElement *elem;
+    uint32_t type;
+    VubReq *req;
+
+    elem = vu_queue_pop(vu_dev, vq, sizeof(VuVirtqElement) +

sizeof(VubReq));

Quoted 68 lines
blockdev-vu.c new file +68 -0
+    if (!elem) {
+        return -1;
+    }
+
+    struct iovec *in_iov = elem->in_sg;
+    struct iovec *out_iov = elem->out_sg;
+    unsigned in_num = elem->in_num;
+    unsigned out_num = elem->out_num;
+    /* refer to hw/block/virtio_blk.c */
+    if (elem->out_num < 1 || elem->in_num < 1) {
+        fprintf(stderr, "virtio-blk request missing headers\n");
+        free(elem);
+        return -1;
+    }
+
+    req = g_new0(VubReq, 1);
+    req->client = client;
+    req->vq = vq;
+    req->elem = elem;
+
+    if (unlikely(iov_to_buf(out_iov, out_num, 0, &req->out,
+                            sizeof(req->out)) != sizeof(req->out))) {
+        fprintf(stderr, "virtio-blk request outhdr too short");
+        goto err;
+    }
+
+    iov_discard_front(&out_iov, &out_num, sizeof(req->out));
+
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+        fprintf(stderr, "virtio-blk request inhdr too short");
+        goto err;
+    }
+
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in = (void *)in_iov[in_num - 1].iov_base
+              + in_iov[in_num - 1].iov_len
+              - sizeof(struct virtio_blk_inhdr);
+    iov_discard_back(in_iov, &in_num, sizeof(struct virtio_blk_inhdr));
+
+
+    type = le32toh(req->out.type);
+    switch (type & ~VIRTIO_BLK_T_BARRIER) {
+    case VIRTIO_BLK_T_IN:
+    case VIRTIO_BLK_T_OUT: {
+        ssize_t ret = 0;
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = le64toh(req->out.sector);
+        if (is_write) {
+            ret = vub_rwv(req, out_iov, out_num, blk_write_entry);
+        } else {
+            ret = vub_rwv(req, in_iov, in_num, blk_read_entry);
+        }
+        if (ret >= 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
+        vub_flush(req);
+        req->in->status = VIRTIO_BLK_S_OK;
+        vub_req_complete(req);
+        break;
+    case VIRTIO_BLK_T_GET_ID: {
+        size_t size = MIN(iov_size(&elem->in_sg[0], in_num),
+                          VIRTIO_BLK_ID_BYTES);

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        snprintf(elem->in_sg[0].iov_base, size, "%s", "vhost_user_blk");
+        req->in->status = VIRTIO_BLK_S_OK;
+        req->size = elem->in_sg[0].iov_len;
+        vub_req_complete(req);
+        break;
+    }
+    case VIRTIO_BLK_T_DISCARD:
+    case VIRTIO_BLK_T_WRITE_ZEROES: {
+        int rc;
+        rc = vub_discard_write_zeroes(req, &elem->out_sg[1], out_num,

type);

Quoted 32 lines
blockdev-vu.c new file +32 -0
+        if (rc == 0) {
+            req->in->status = VIRTIO_BLK_S_OK;
+        } else {
+            req->in->status = VIRTIO_BLK_S_IOERR;
+        }
+        vub_req_complete(req);
+        break;
+    }
+    default:
+        req->in->status = VIRTIO_BLK_S_UNSUPP;
+        vub_req_complete(req);
+        break;
+    }
+
+    return 0;
+
+err:
+    free(elem);
+    g_free(req);
+    return -1;
+}
+
+
+static void vub_process_vq(VuDev *vu_dev, int idx)
+{
+    VuClient *client;
+    VuVirtq *vq;
+    int ret;
+
+    client = container_of(vu_dev, VuClient, parent);
+    assert(client);
+

blockdev-vu.c line 361

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    vq = vu_get_queue(vu_dev, idx);
+    assert(vq);
+
+    while (1) {
+        ret = vub_virtio_process_req(client, vq);
+        if (ret) {
+            break;
+        }
+    }
+}

I mentioned above moving vub_virtio_process_req() into a coroutine. Just creating a coroutine here and entering it wouldn't work, though.

The best design is probably to get requests from the virtqueue in this function (what is currently the first half of vub_virtio_process_req()) and then spawn a coroutine per request to actually execute them (roughly the switch in vub_virtio_process_req()).

This way you'll get parallel requests and won't have to think about synchronising accesses to the virtqueue from multiple coroutines.

Quoted 16 lines
blockdev-vu.c new file +16 -0
+
+static void vub_queue_set_started(VuDev *vu_dev, int idx, bool started)
+{
+    VuVirtq *vq;
+
+    assert(vu_dev);
+
+    vq = vu_get_queue(vu_dev, idx);
+    vu_set_queue_handler(vu_dev, vq, started ? vub_process_vq : NULL);
+}
+
+static uint64_t
+vub_get_features(VuDev *dev)
+{
+    uint64_t features;
+    VubDev *vdev_blk;

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    VuClient *client = container_of(dev, VuClient, parent);
+    vdev_blk = client->blk;
+
+    features = 1ull << VIRTIO_BLK_F_SIZE_MAX |
+               1ull << VIRTIO_BLK_F_SEG_MAX |
+               1ull << VIRTIO_BLK_F_TOPOLOGY |
+               1ull << VIRTIO_BLK_F_BLK_SIZE |
+               1ull << VIRTIO_BLK_F_FLUSH |
+               #if defined(__linux__) && defined(BLKDISCARD) &&

defined(BLKZEROOUT)

Quoted 60 lines
blockdev-vu.c new file +60 -0
+               1ull << VIRTIO_BLK_F_DISCARD |
+               1ull << VIRTIO_BLK_F_WRITE_ZEROES |
+               #endif
+               1ull << VIRTIO_BLK_F_CONFIG_WCE |
+               1ull << VIRTIO_F_VERSION_1 |
+               1ull << VIRTIO_RING_F_INDIRECT_DESC |
+               1ull << VIRTIO_RING_F_EVENT_IDX |
+               1ull << VHOST_USER_F_PROTOCOL_FEATURES;
+
+    if (!vdev_blk->writable) {
+        features |= 1ull << VIRTIO_BLK_F_RO;
+    }
+
+    return features;
+}
+
+static uint64_t
+vub_get_protocol_features(VuDev *dev)
+{
+    return 1ull << VHOST_USER_PROTOCOL_F_CONFIG |
+           1ull << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD;
+}
+
+static int
+vub_get_config(VuDev *vu_dev, uint8_t *config, uint32_t len)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    memcpy(config, &vdev_blk->blkcfg, len);
+
+    return 0;
+}
+
+static int
+vub_set_config(VuDev *vu_dev, const uint8_t *data,
+               uint32_t offset, uint32_t size, uint32_t flags)
+{
+    VubDev *vdev_blk;
+
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    vdev_blk = client->blk;
+    uint8_t wce;
+
+    /* don't support live migration */
+    if (flags != VHOST_SET_CONFIG_TYPE_MASTER) {
+        return -1;
+    }
+
+
+    if (offset != offsetof(struct virtio_blk_config, wce) ||
+        size != 1) {
+        return -1;
+    }
+
+    wce = *data;
+    if (wce == vdev_blk->blkcfg.wce) {
+        /* Do nothing as same with old configuration */
+        return 0;

blockdev-vu.c line 458

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    }
+
+    vdev_blk->blkcfg.wce = wce;
+    blk_set_enable_write_cache(vdev_blk->backend, true);
+    return 0;
+}
+
+
+/*
+ * When the client disconnects, it send a VHOST_USER_NONE request

s/send/sends/

blockdev-vu.c

Quoted 7 lines
blockdev-vu.c new file +7 -0
+ * and vu_process_message will simple call exit which cause the VM
+ * to exit abruptly.
+ * To avoid this issue,  process VHOST_USER_NONE request ahead
+ * of vu_process_message.
+ *
+ */
+static int vub_process_msg(VuDev *dev, VhostUserMsg *vmsg, int

*do_reply)

Quoted 11 lines
blockdev-vu.c new file +11 -0
+{
+    if (vmsg->request == VHOST_USER_NONE) {
+        dev->panic(dev, "disconnect");
+        return true;
+    }
+    return false;
+}
+
+static void
+vmsg_close_fds(VhostUserMsg *vmsg)
+{

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    int i;
+    for (i = 0; i < vmsg->fd_num; i++) {
+        close(vmsg->fds[i]);
+    }
+}
+
+static bool
+vu_message_read_co(VuDev *vu_dev, int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] =

{ };

Quoted 8 lines
blockdev-vu.c new file +8 -0
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,

blockdev-vu.c line 504

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+    char buffer[100];
+    VuClient *client = container_of(vu_dev, VuClient, parent);
+    QIOChannel *ioc = client->ioc;
+    do {
+        rc = recvmsg(conn_fd, &msg, 0);

This should certainly use qio_channel_readv_full() rather than working directly with a socket fd?

Quoted 12 lines
blockdev-vu.c new file +12 -0
+        if (rc < 0) {
+            if (errno == EAGAIN) {
+                if (qemu_in_coroutine()) {
+                    qio_channel_yield(ioc, G_IO_IN);
+                } else {
+                    qio_channel_wait(ioc, G_IO_IN);
+                }
+                continue;
+            } else if (errno == EINTR) {
+                continue;
+            }
+        }

blockdev-vu.c line 526

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        break;
+    } while (true);
+
+    if (rc < 0) {
+        sprintf(buffer, "Error while recvmsg: %s", strerror(errno));
+        vub_panic_cb(vu_dev, buffer);
+        return false;
+    }
+
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);

Why do you think that there can't be short reads?

blockdev-vu.c

Quoted 6 lines
blockdev-vu.c new file +6 -0
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type ==

SCM_RIGHTS) {

blockdev-vu.c line 539

Quoted 6 lines
blockdev-vu.c new file +6 -0
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }

I think the fd passing part becomes easier when you use the proper qio_channel_readv_full() function. Its implementation is also a bit more careful than yours. For example, you forgot checking fd_size against VHOST_MEMORY_MAX_NREGIONS, allowing a buffer overflow in the memcpy(), and you don't adjust fd flags for the new file descriptors.

blockdev-vu.c

Quoted 3 lines
blockdev-vu.c new file +3 -0
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        sprintf(buffer,
+                "Error: too big message request: %d, size: vmsg->size:

%u, "

blockdev-vu.c line 552

Quoted 9 lines
blockdev-vu.c new file +9 -0
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        vub_panic_cb(vu_dev, buffer);
+        goto fail;
+    }
+
+    if (vmsg->size) {
+        do {
+            rc = read(conn_fd, &vmsg->payload, vmsg->size);

qio_channel_readv_all_eof() already implements this whole loop and correctly handles short reads, too.

Quoted 56 lines
blockdev-vu.c new file +56 -0
+            if (rc < 0) {
+                if (errno == EAGAIN) {
+                    if (qemu_in_coroutine()) {
+                        qio_channel_yield(ioc, G_IO_IN);
+                    } else {
+                        qio_channel_wait(ioc, G_IO_IN);
+                    }
+                    continue;
+                } else if (errno == EINTR) {
+                    continue;
+                }
+            }
+            break;
+        } while (true);
+
+        if (rc <= 0) {
+            sprintf(buffer, "Error while reading: %s", strerror(errno));
+            vub_panic_cb(vu_dev, buffer);
+            goto fail;
+        }
+
+        assert(rc == vmsg->size);
+    }
+
+    return true;
+
+fail:
+    vmsg_close_fds(vmsg);
+
+    return false;
+}
+
+static void vub_kick_cb(void *opaque)
+{
+    vu_watch_cb_data *data = (vu_watch_cb_data *) opaque;
+    int index = data->index;
+    VuDev *dev = data->vu_dev;
+    VuVirtq *vq = &dev->vq[index];
+    int sock = vq->kick_fd;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        char buffer[100];
+        sprintf(buffer, "kick eventfd_read(): %s", strerror(errno));
+        vub_panic_cb(dev, buffer);
+        g_free(data);
+        dev->remove_watch(dev, dev->vq[index].kick_fd);
+    } else {
+        if (vq->handler) {
+            vq->handler(dev, index);
+        }
+    }
+}
+

blockdev-vu.c line 618

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static const VuDevIface vub_iface = {
+    .get_features = vub_get_features,
+    .queue_set_started = vub_queue_set_started,
+    .get_protocol_features = vub_get_protocol_features,
+    .get_config = vub_get_config,
+    .set_config = vub_set_config,
+    .process_msg = vub_process_msg,
+    .read_msg = vu_message_read_co,
+    .kick_callback = vub_kick_cb,
+};

I would prefer the = signs to be aligned to the same column.

Quoted 28 lines
blockdev-vu.c new file +28 -0
+
+void vub_free(VubDev *vub_dev, bool called_by_QOM)
+{
+    if (!vub_dev) {
+        return;
+    }
+
+    blk_unref(vub_dev->backend);
+    g_free(vub_dev->name);
+    g_free(vub_dev->unix_socket);
+
+    if (vub_dev->next.tqe_circ.tql_prev) {
+        /*
+         * if vub_dev->next.tqe_circ.tql_prev = null,
+         * vub_dev hasn't been inserted into the queue and
+         * vub_free is called by obj->instance_finalize.
+         */
+        QTAILQ_REMOVE(&vub_devs, vub_dev, next);
+    }
+    /*
+     * Needn't to free vub_dev if called by QOM
+     * because QOM will do the clean-up work.
+     */
+    if (!called_by_QOM) {
+        g_free(vub_dev);
+    }
+}
+

blockdev-vu.c line 657

Quoted 10 lines
blockdev-vu.c new file +10 -0
+static coroutine_fn void vu_client_trip(void *opaque)
+{
+    VuClient *client = opaque;
+
+    while (!client->closed) {
+        vu_dispatch(&client->parent);
+    }
+
+    QTAILQ_REMOVE(&client->blk->clients, client, next);
+

Extra empty line.

Quoted 48 lines
blockdev-vu.c new file +48 -0
+}
+
+static void vu_client_start(VuClient *client)
+{
+    Coroutine *co = qemu_coroutine_create(vu_client_trip, client);
+    qemu_coroutine_enter(co);
+}
+
+
+G_STATIC_ASSERT((int)G_IO_IN == (int)VU_WATCH_IN);
+G_STATIC_ASSERT((int)G_IO_OUT == (int)VU_WATCH_OUT);
+G_STATIC_ASSERT((int)G_IO_PRI == (int)VU_WATCH_PRI);
+G_STATIC_ASSERT((int)G_IO_ERR == (int)VU_WATCH_ERR);
+G_STATIC_ASSERT((int)G_IO_HUP == (int)VU_WATCH_HUP);
+
+static void
+set_watch(VuDev *vu_dev, int fd, int vu_evt,
+          vu_watch_cb_packed_data cb, void *pvt)
+{
+    /*
+     * since aio_dispatch can only pass one user data pointer to the
+     * callback function, pack VuDev, pvt into a struct
+     */
+    VuClient *client;
+
+    g_assert(vu_dev);
+    g_assert(fd >= 0);
+    g_assert(cb);
+    client = container_of(vu_dev, VuClient, parent);
+    vu_watch_cb_data *cb_data = g_new0(vu_watch_cb_data, 1);
+    cb_data->index = (intptr_t) pvt;
+    cb_data->vu_dev = vu_dev;
+    aio_set_fd_handler(client->blk->ctx, fd, false, (void *) cb,
+                       NULL, NULL, cb_data);
+}
+
+
+void vub_accept(QIONetListener *listener, QIOChannelSocket *sioc,
+                gpointer opaque)
+{
+    VuClient *client;
+    VubDev *vub_device = opaque;
+    client = g_new0(VuClient, 1);
+
+    if (!vu_init_packed_data(&client->parent, VHOST_USER_BLK_MAX_QUEUES,
+                             sioc->fd, vub_panic_cb, set_watch,
+                             remove_watch, &vub_iface)) {
+        fprintf(stderr, "Failed to initialized libvhost-user\n");

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+        g_free(client);
+        return;
+    }
+
+    client->blk = vub_device;
+    client->refcount = 1;
+    client->sioc = sioc;
+    /*
+     * increase the object reference, so cioc will not freed by
+     * qio_net_listener_channel_func which will call

object_unref(OBJECT(sioc))

Quoted 6 lines
blockdev-vu.c new file +6 -0
+     */
+    object_ref(OBJECT(client->sioc));
+    qio_channel_set_name(QIO_CHANNEL(sioc), "vhost-user client");
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    object_ref(OBJECT(sioc));

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    qio_channel_set_blocking(QIO_CHANNEL(client->sioc), false, NULL);
+    client->closed = false;
+    QTAILQ_INSERT_TAIL(&client->blk->clients, client, next);
+    vu_client_start(client);
+}
+
+
+void
+vub_initialize_config(BlockDriverState *bs, struct virtio_blk_config

*config)

Quoted 10 lines
blockdev-vu.c new file +10 -0
+{
+    config->capacity = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
+    config->blk_size = BDRV_SECTOR_SIZE;
+    config->size_max = 65536;
+    config->seg_max = 128 - 2;
+    config->min_io_size = 1;
+    config->opt_io_size = 1;
+    config->num_queues = 1;
+    #if defined(__linux__) && defined(BLKDISCARD) && defined(BLKZEROOUT)
+    config->max_discard_sectors = 32768;

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    config->max_discard_seg = 1;
+    config->discard_sector_alignment = config->blk_size >> 9;
+    config->max_write_zeroes_sectors = 32768;
+    config->max_write_zeroes_seg = 1;
+    #endif
+}
+
+
+static VubDev *vub_new(VubDev *vub_device, const char *name,
+                       const char *unix_socket, bool writable, Error

**errp)

Quoted 6 lines
blockdev-vu.c new file +6 -0
+{
+
+    BlockBackend *blk;
+
+    /*
+     * Don't allow resize while the vhost user server is running,

blockdev-vu.c line 767

Quoted 10 lines
blockdev-vu.c new file +10 -0
+     * otherwise we don't care what happens with the node.
+     */
+    uint64_t perm = BLK_PERM_CONSISTENT_READ;
+    int ret;
+
+    AioContext *ctx;
+
+    BlockDriverState *bs = bdrv_lookup_bs(name,
+                                          name,
+                                          errp);

This fits in a single line.

blockdev-vu.c line 773

Quoted 6 lines
blockdev-vu.c new file +6 -0
+
+    if (!bs) {
+        error_setg(errp,
+                   "No drive with name '%s'."
+                   " Please find the list of names with "
+                   "'info block'", name);

This can probably be two lines instead of four.

Quoted 83 lines
blockdev-vu.c new file +83 -0
+        return NULL;
+    }
+
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    if (writable) {
+        perm |= BLK_PERM_WRITE;
+    }
+
+    ctx = bdrv_get_aio_context(bs);
+    aio_context_acquire(ctx);
+    bdrv_invalidate_cache(bs, NULL);
+    aio_context_release(ctx);
+
+    blk = blk_new(bdrv_get_aio_context(bs), perm,
+                  BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
+                  BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD);
+    ret = blk_insert_bs(blk, bs, errp);
+
+    if (ret < 0) {
+        goto fail;
+    }
+
+
+    blk_set_enable_write_cache(blk, false);
+
+    blk_set_allow_aio_context_change(blk, true);
+
+
+    vub_device->name = g_strdup(name);
+    vub_device->unix_socket = g_strdup(unix_socket);
+    vub_device->writable = writable;
+    vub_device->blkcfg.wce = 0;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    return vub_device;
+
+fail:
+    blk_unref(blk);
+    return NULL;
+}
+
+void vhost_user_server_free(VubDev *vub_device, bool called_by_QOM)
+{
+    if (!vub_device) {
+        return;
+    }
+
+    VuClient *client, *next;
+    QTAILQ_FOREACH_SAFE(client, &vub_device->clients, next, next) {
+        if (!client->closed) {
+            close_client(client);
+        }
+    }
+
+    if (vub_device->listener) {
+        qio_net_listener_disconnect(vub_device->listener);
+        object_unref(OBJECT(vub_device->listener));
+    }
+    vub_free(vub_device, called_by_QOM);
+
+}
+
+
+VubDev *vub_dev_find(const char *name)
+{
+    VubDev *vub_device;
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(name, vub_device->name) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+
+static VubDev *vub_dev_find_by_unix_socket(const char *unix_socket)
+{
+    VubDev *vub_device;

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    QTAILQ_FOREACH(vub_device, &vub_devs, next) {
+        if (strcmp(unix_socket, vub_device->unix_socket) == 0) {
+            return vub_device;
+        }
+    }
+
+    return NULL;
+}
+
+static void vhost_user_server_start(VubDev *vub_device, const char

*unix_socket,

blockdev-vu.c

Quoted 5 lines
blockdev-vu.c new file +5 -0
+                                    const char *name, bool writable,
+                                    Error **errp)
+{
+
+    if (vub_dev_find(name) || vub_dev_find_by_unix_socket(unix_socket))

{

Quoted 11 lines
blockdev-vu.c new file +11 -0
+        error_setg(errp, "Vhost user server with name '%s' or "
+                "with socket_path '%s' has already been started",
+                name, unix_socket);
+        return;
+    }
+
+
+    if (!vub_new(vub_device, name, unix_socket, writable, errp)) {
+        return;
+    }
+

blockdev-vu.c

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    vub_device->listener = qio_net_listener_new();
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);
+    addr->u.q_unix.path = (char *) unix_socket;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp)

< 0) {

Quoted 42 lines
blockdev-vu.c new file +42 -0
+        goto error;
+    }
+
+
+    QTAILQ_INSERT_TAIL(&vub_devs, vub_device, next);
+    QTAILQ_INIT(&vub_device->clients);
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    return;
+
+ error:
+    vub_free(vub_device, false);
+}
+
+static void vu_set_block_name(Object *obj, const char *value,
+                                           Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+
+    if (vus->name) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+
+    vus->name = g_strdup(value);
+}
+
+static char *vu_get_block_name(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    return g_strdup(vus->name);
+}
+
+
+static void vu_set_unix_socket(Object *obj, const char *value,
+                                            Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;

blockdev-vu.c line 944

Quoted 10 lines
blockdev-vu.c new file +10 -0
+
+    if (vus->unix_socket) {
+        error_setg(errp, "unix_socket property already set");
+        return;
+    }
+
+    vus->unix_socket = g_strdup(value);
+    vhost_user_server_start(vus, value, vus->name,
+                            vus->writable, errp);
+}

This makes the unix-socket property magic in that it starts the server with the properties specified at this point. This means that this property must always be specified last.

Maybe it would be better to use a boolean property (similar to qdev's "realized") that explicitly start and possibly stops the export.

Writing to other properties should probably result in an error while the server is already running because these property changes won't take effect any more then.

Quoted 23 lines
blockdev-vu.c new file +23 -0
+static char *vu_get_unix_socket(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return g_strdup(vus->unix_socket);
+}
+
+static bool vu_get_block_writable(Object *obj, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);;
+    return vus->writable;
+}
+
+static void vu_set_block_writable(Object *obj, bool value, Error **errp)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+
+    vus->writable = value;
+}
+
+static void vhost_user_server_instance_init(Object *obj)
+{
+
+    object_property_add_bool(obj, "writable",

blockdev-vu.c line 978

Quoted 10 lines
blockdev-vu.c new file +10 -0
+                            vu_get_block_writable,
+                            vu_set_block_writable, NULL);
+
+    object_property_add_str(obj, "name",
+                            vu_get_block_name,
+                            vu_set_block_name, NULL);
+
+    object_property_add_str(obj, "unix_socket",
+                            vu_get_unix_socket,
+                            vu_set_unix_socket, NULL);

These should probably be object_class_property_add_*() and be called in .class_init rather than .instance_init.

"name" suggests that it's the name of the export rather than the block device to be exported. I would suggest "node-name" (and then actually only pass it as node-name to bdrv_lookup_bs()).

I expect that in the long run, we'll want to accept a full SocketAddress rather than just a filename like in "unix_socket".

Quoted 19 lines
blockdev-vu.c new file +19 -0
+}
+
+static void vhost_user_server_instance_finalize(Object *obj)
+{
+    VubDev *vus = VHOST_USER_SERVER(obj);
+    vhost_user_server_free(vus, true);
+    /* object_del shouldn't free this object struct */
+    obj->free = NULL;
+}
+
+static const TypeInfo vhost_user_server_info = {
+    .name = TYPE_VHOST_USER_SERVER,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(VuDev),
+    .instance_init = vhost_user_server_instance_init,
+    .instance_finalize = vhost_user_server_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        {TYPE_USER_CREATABLE},
+        {}

blockdev-vu.c line 1008

Quoted 10 lines
blockdev-vu.c new file +10 -0
+    },
+};
+
+static void vhost_user_server_register_types(void)
+{
+    type_register_static(&vhost_user_server_info);
+}
+
+type_init(vhost_user_server_register_types)
+

Extra empty line at the file end.

In summary, I can see this going in the right direction, though in detail some more work will be needed.

Kevin

Signature · 2 lines
*Best regards,*
*Coiby*
author
#8 lore raw

PATCH v2 2/5 extend libvhost to support IOThread

In reply to Coiby Xu

Previously libvhost dispatch events in its own GMainContext. Now vhost-user client's kick event can be dispatched in block device drive's AioContext thus IOThread is supported.

---
 contrib/libvhost-user/libvhost-user.c | 64 ++++++++++++++++++++++-----
 contrib/libvhost-user/libvhost-user.h | 36 ++++++++++++++-
 2 files changed, 87 insertions(+), 13 deletions(-)
contrib/libvhost-user/libvhost-user.c +53 -11
index ec27b78ff1..cd328c1509 100644
@@ -67,7 +67,6 @@
 /* The version of inflight buffer */
 #define INFLIGHT_VERSION 1
 
-#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
 
 /* The version of the protocol we support */
 #define VHOST_USER_VERSION 1
@@ -260,7 +259,7 @@ have_userfault(void)
 }
 
 static bool
-vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
+vu_message_read_(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
 {
     char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
     struct iovec iov = {
@@ -286,6 +285,8 @@ vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
         return false;
     }
 
+    assert(rc == VHOST_USER_HDR_SIZE || rc == 0);
+
     vmsg->fd_num = 0;
     for (cmsg = CMSG_FIRSTHDR(&msg);
          cmsg != NULL;
@@ -328,6 +329,17 @@ fail:
     return false;
 }
 
+static bool vu_message_read(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
+{
+    vu_read_msg_cb read_msg;
+    if (dev->iface->read_msg) {
+        read_msg = dev->iface->read_msg;
+    } else {
+        read_msg = vu_message_read_;
+    }
+    return read_msg(dev, conn_fd, vmsg);
+}
+
 static bool
 vu_message_write(VuDev *dev, int conn_fd, VhostUserMsg *vmsg)
 {
@@ -400,7 +412,6 @@ vu_process_message_reply(VuDev *dev, const VhostUserMsg *vmsg)
     if ((vmsg->flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
         return true;
     }
-
     if (!vu_message_read(dev, dev->slave_fd, &msg_reply)) {
         return false;
     }
@@ -644,7 +655,8 @@ vu_set_mem_table_exec_postcopy(VuDev *dev, VhostUserMsg *vmsg)
                     "%s: Failed to madvise(DONTNEED) region %d: %s\n",
                     __func__, i, strerror(errno));
         }
-        /* Turn off transparent hugepages so we dont get lose wakeups
+        /*
+         * Turn off transparent hugepages so we don't get lose wakeups
          * in neighbouring pages.
          * TODO: Turn this backon later.
          */
@@ -1047,9 +1059,13 @@ vu_set_vring_kick_exec(VuDev *dev, VhostUserMsg *vmsg)
     }
 
     if (dev->vq[index].kick_fd != -1 && dev->vq[index].handler) {
-        dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
-                       vu_kick_cb, (void *)(long)index);
-
+        if (dev->set_watch_packed_data) {
+            dev->set_watch_packed_data(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
+                           dev->iface->kick_callback, (void *)(long)index);
+        } else {
+            dev->set_watch(dev, dev->vq[index].kick_fd, VU_WATCH_IN,
+                           vu_kick_cb, (void *)(long)index);
+        }
         DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
                dev->vq[index].kick_fd, index);
     }
@@ -1069,8 +1085,13 @@ void vu_set_queue_handler(VuDev *dev, VuVirtq *vq,
     vq->handler = handler;
     if (vq->kick_fd >= 0) {
         if (handler) {
-            dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
-                           vu_kick_cb, (void *)(long)qidx);
+            if (dev->set_watch_packed_data) {
+                dev->set_watch_packed_data(dev, vq->kick_fd, VU_WATCH_IN,
+                        dev->iface->kick_callback, (void *)(long)qidx);
+            } else {
+                dev->set_watch(dev, vq->kick_fd, VU_WATCH_IN,
+                        vu_kick_cb, (void *)(long)qidx);
+            }
         } else {
             dev->remove_watch(dev, vq->kick_fd);
         }
@@ -1596,6 +1617,12 @@ vu_deinit(VuDev *dev)
         }
 
         if (vq->kick_fd != -1) {
+            /* remove watch for kick_fd
+             * When client process is running in gdb and
+             * quit command is run in gdb, QEMU will still dispatch the event
+             * which will cause segment fault in the callback function
+             */
+            dev->remove_watch(dev, vq->kick_fd);
             close(vq->kick_fd);
             vq->kick_fd = -1;
         }
@@ -1647,10 +1674,9 @@ vu_init(VuDev *dev,
         const VuDevIface *iface)
 {
     uint16_t i;
-
     assert(max_queues > 0);
     assert(socket >= 0);
-    assert(set_watch);
+    /* assert(set_watch); */
     assert(remove_watch);
     assert(iface);
     assert(panic);
@@ -1682,6 +1708,22 @@ vu_init(VuDev *dev,
     return true;
 }
 
+bool
+vu_init_packed_data(VuDev *dev,
+        uint16_t max_queues,
+        int socket,
+        vu_panic_cb panic,
+        vu_set_watch_cb_packed_data set_watch_packed_data,
+        vu_remove_watch_cb remove_watch,
+        const VuDevIface *iface)
+{
+    if (vu_init(dev, max_queues, socket, panic, NULL, remove_watch, iface)) {
+        dev->set_watch_packed_data = set_watch_packed_data;
+        return true;
+    }
+    return false;
+}
+
 VuVirtq *
 vu_get_queue(VuDev *dev, int qidx)
 {
contrib/libvhost-user/libvhost-user.h +34 -2
index 46b600799b..5230d55092 100644
@@ -34,6 +34,9 @@ typedef enum VhostSetConfigType {
     VHOST_SET_CONFIG_TYPE_MIGRATION = 1,
 } VhostSetConfigType;
 
+
+#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
+
 /*
  * Maximum size of virtio device config space
  */
@@ -200,6 +203,7 @@ typedef uint64_t (*vu_get_features_cb) (VuDev *dev);
 typedef void (*vu_set_features_cb) (VuDev *dev, uint64_t features);
 typedef int (*vu_process_msg_cb) (VuDev *dev, VhostUserMsg *vmsg,
                                   int *do_reply);
+typedef bool (*vu_read_msg_cb) (VuDev *dev, int sock, VhostUserMsg *vmsg);
 typedef void (*vu_queue_set_started_cb) (VuDev *dev, int qidx, bool started);
 typedef bool (*vu_queue_is_processed_in_order_cb) (VuDev *dev, int qidx);
 typedef int (*vu_get_config_cb) (VuDev *dev, uint8_t *config, uint32_t len);
@@ -207,6 +211,15 @@ typedef int (*vu_set_config_cb) (VuDev *dev, const uint8_t *data,
                                  uint32_t offset, uint32_t size,
                                  uint32_t flags);
 
+typedef struct vu_watch_cb_data {
+   long index;
+   VuDev *vu_dev;
+} vu_watch_cb_data;
+typedef void (*vu_watch_cb_packed_data) (void *packed_data);
+
+typedef void (*vu_set_watch_cb_packed_data) (VuDev *dev, int fd, int condition,
+                                 vu_watch_cb_packed_data cb, void *data);
+
 typedef struct VuDevIface {
     /* called by VHOST_USER_GET_FEATURES to get the features bitmask */
     vu_get_features_cb get_features;
@@ -220,8 +233,11 @@ typedef struct VuDevIface {
     /* process_msg is called for each vhost-user message received */
     /* skip libvhost-user processing if return value != 0 */
     vu_process_msg_cb process_msg;
+    vu_read_msg_cb read_msg;
+    vu_watch_cb_packed_data kick_callback;
     /* tells when queues can be processed */
     vu_queue_set_started_cb queue_set_started;
+
     /*
      * If the queue is processed in order, in which case it will be
      * resumed to vring.used->idx. This can help to support resuming
@@ -366,7 +382,8 @@ struct VuDev {
     /* @set_watch: add or update the given fd to the watch set,
      * call cb when condition is met */
     vu_set_watch_cb set_watch;
-
+    /* AIO dispatch will only one data pointer to callback function */
+    vu_set_watch_cb_packed_data set_watch_packed_data;
     /* @remove_watch: remove the given fd from the watch set */
     vu_remove_watch_cb remove_watch;
 
@@ -398,7 +415,7 @@ typedef struct VuVirtqElement {
  * @remove_watch: a remove_watch callback
  * @iface: a VuDevIface structure with vhost-user device callbacks
  *
- * Intializes a VuDev vhost-user context.
+ * Initializes a VuDev vhost-user context.
  *
  * Returns: true on success, false on failure.
  **/
@@ -411,6 +428,21 @@ bool vu_init(VuDev *dev,
              const VuDevIface *iface);
 
 
+/**
+ * vu_init_packed_data:
+ * Same as vu_init except for set_watch_packed_data which will pack
+ * two parameters into a struct thus QEMU aio_dispatch can pass the
+ * required data to callback function.
+ *
+ * Returns: true on success, false on failure.
+ **/
+bool vu_init_packed_data(VuDev *dev,
+             uint16_t max_queues,
+             int socket,
+             vu_panic_cb panic,
+             vu_set_watch_cb_packed_data set_watch_packed_data,
+             vu_remove_watch_cb remove_watch,
+             const VuDevIface *iface);
 /**
  * vu_deinit:
  * @dev: a VuDev context

--
2.24.1

author
#9 lore raw

PATCH v2 3/5 a standone-alone tool to directly share disk image file via vhost-user protocol

In reply to Coiby Xu

vhost-user-blk can have played as vhost-user backend but it only supports raw file and don't support VIRTIO_BLK_T_DISCARD and VIRTIO_BLK_T_WRITE_ZEROES operations on raw file (ioctl(fd, BLKDISCARD) is only valid for real block device).

---
 qemu-vu.c | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 264 insertions(+)
 create mode 100644 qemu-vu.c
qemu-vu.c new file +264 -0
new file mode 100644
index 0000000000..25c32c2c6d
@@ -0,0 +1,264 @@
+/*
+ *  Copyright (C) 2020  Coiby Xu <[email protected]>
+ *
+ *  Vhost-user-blk device backend
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include <getopt.h>
+#include <libgen.h>
+#include "block/vhost-user.h"
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+#include "sysemu/block-backend.h"
+#include "block/block_int.h"
+#include "qemu/main-loop.h"
+#include "qemu/module.h"
+#include "qemu/option.h"
+#include "qemu/error-report.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
+#include "qom/object_interfaces.h"
+#include "io/net-listener.h"
+#include "qemu-version.h"
+
+#define QEMU_VU_OPT_CACHE         256
+
+#define QEMU_VU_OPT_AIO           257
+
+static char *srcpath;
+
+static void usage(const char *name)
+{
+    (printf) (
+"Usage: %s [OPTIONS] FILE\n"
+"  or:  %s -L [OPTIONS]\n"
+"QEMU Vhost-user Server Utility\n"
+"\n"
+"  -h, --help                display this help and exit\n"
+"  -V, --version             output version information and exit\n"
+"\n"
+"Connection properties:\n"
+"  -k, --socket=PATH         path to the unix socket\n"
+"\n"
+"General purpose options:\n"
+"  -e, -- exit-panic         When the panic callback is called, the program\n"
+"                            will exit. Useful for make check-qtest.\n"
+"\n"
+"Block device options:\n"
+"  -f, --format=FORMAT       set image format (raw, qcow2, ...)\n"
+"  -r, --read-only           export read-only\n"
+"  -n, --nocache             disable host cache\n"
+"      --cache=MODE          set cache mode (none, writeback, ...)\n"
+"      --aio=MODE            set AIO mode (native or threads)\n"
+"\n"
+QEMU_HELP_BOTTOM "\n"
+    , name, name);
+}
+
+static void version(const char *name)
+{
+    printf(
+"%s " QEMU_FULL_VERSION "\n"
+"Written by Coiby Xu, based on qemu-nbd by Anthony Liguori\n"
+"\n"
+QEMU_COPYRIGHT "\n"
+"This is free software; see the source for copying conditions.  There is NO\n"
+"warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n"
+    , name);
+}
+
+static VubDev *vub_device;
+
+static void vus_shutdown(void)
+{
+    job_cancel_sync_all();
+    bdrv_close_all();
+    vub_free(vub_device, false);
+}
+
+int main(int argc, char **argv)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs;
+    bool readonly = false;
+    char *sockpath = NULL;
+    int64_t fd_size;
+    const char *sopt = "hVrnvek:f:";
+    struct option lopt[] = {
+        { "help", no_argument, NULL, 'h' },
+        { "version", no_argument, NULL, 'V' },
+        { "exit-panic", no_argument, NULL, 'e' },
+        { "socket", required_argument, NULL, 'k' },
+        { "read-only", no_argument, NULL, 'r' },
+        { "nocache", no_argument, NULL, 'n' },
+        { "cache", required_argument, NULL, QEMU_VU_OPT_CACHE },
+        { "aio", required_argument, NULL, QEMU_VU_OPT_AIO },
+        { "format", required_argument, NULL, 'f' },
+        { NULL, 0, NULL, 0 }
+    };
+    int ch;
+    int opt_ind = 0;
+    int flags = BDRV_O_RDWR;
+    bool seen_cache = false;
+    bool seen_aio = false;
+    const char *fmt = NULL;
+    Error *local_err = NULL;
+    QDict *options = NULL;
+    bool writethrough = true;
+    bool exit_panic = false;
+
+    error_init(argv[0]);
+
+    module_call_init(MODULE_INIT_QOM);
+    qemu_init_exec_dir(argv[0]);
+
+    while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
+        switch (ch) {
+        case 'e':
+            exit_panic = true;
+            break;
+        case 'n':
+            optarg = (char *) "none";
+            /* fallthrough */
+        case QEMU_VU_OPT_CACHE:
+            if (seen_cache) {
+                error_report("-n and --cache can only be specified once");
+                exit(EXIT_FAILURE);
+            }
+            seen_cache = true;
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) == -1) {
+                error_report("Invalid cache mode `%s'", optarg);
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case QEMU_VU_OPT_AIO:
+            if (seen_aio) {
+                error_report("--aio can only be specified once");
+                exit(EXIT_FAILURE);
+            }
+            seen_aio = true;
+            if (!strcmp(optarg, "native")) {
+                flags |= BDRV_O_NATIVE_AIO;
+            } else if (!strcmp(optarg, "threads")) {
+                /* this is the default */
+            } else {
+               error_report("invalid aio mode `%s'", optarg);
+               exit(EXIT_FAILURE);
+            }
+            break;
+        case 'r':
+            readonly = true;
+            flags &= ~BDRV_O_RDWR;
+            break;
+        case 'k':
+            sockpath = optarg;
+            if (sockpath[0] != '/') {
+                error_report("socket path must be absolute");
+                exit(EXIT_FAILURE);
+            }
+            break;
+        case 'f':
+            fmt = optarg;
+            break;
+        case 'V':
+            version(argv[0]);
+            exit(0);
+            break;
+        case 'h':
+            usage(argv[0]);
+            exit(0);
+            break;
+        case '?':
+            error_report("Try `%s --help' for more information.", argv[0]);
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    if ((argc - optind) != 1) {
+        error_report("Invalid number of arguments");
+        error_printf("Try `%s --help' for more information.\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    if (qemu_init_main_loop(&local_err)) {
+        error_report_err(local_err);
+        exit(EXIT_FAILURE);
+    }
+    bdrv_init();
+
+    srcpath = argv[optind];
+    if (fmt) {
+        options = qdict_new();
+        qdict_put_str(options, "driver", fmt);
+    }
+    blk = blk_new_open(srcpath, NULL, options, flags, &local_err);
+
+    if (!blk) {
+        error_reportf_err(local_err, "Failed to blk_new_open '%s': ",
+                          argv[optind]);
+        exit(EXIT_FAILURE);
+    }
+    bs = blk_bs(blk);
+
+    blk_set_enable_write_cache(blk, !writethrough);
+
+    fd_size = blk_getlength(blk);
+    if (fd_size < 0) {
+        error_report("Failed to determine the image length: %s",
+                     strerror(-fd_size));
+        exit(EXIT_FAILURE);
+    }
+
+    AioContext *ctx = bdrv_get_aio_context(bs);
+    bdrv_invalidate_cache(bs, NULL);
+
+    vub_device = g_new0(VubDev, 1);
+    vub_device->unix_socket = g_strdup(sockpath);
+    vub_device->writable = !readonly;
+    vub_device->blkcfg.wce = !writethrough;
+    vub_device->backend = blk;
+    vub_device->ctx = ctx;
+    vub_initialize_config(bs, &vub_device->blkcfg);
+    vub_device->listener = qio_net_listener_new();
+    vub_device->exit_panic = exit_panic;
+
+    qio_net_listener_set_name(vub_device->listener,
+                              "vhost-user-backend-listener");
+
+    SocketAddress *addr = g_new0(SocketAddress, 1);
+    addr->u.q_unix.path = (char *) sockpath;
+    addr->type = SOCKET_ADDRESS_TYPE_UNIX;
+    Error **errp = NULL;
+    if (qio_net_listener_open_sync(vub_device->listener, addr, 1, errp) < 0) {
+        goto error;
+    }
+
+    qio_net_listener_set_client_func(vub_device->listener,
+                                     vub_accept,
+                                     vub_device,
+                                     NULL);
+
+    QTAILQ_INIT(&vub_device->clients);
+
+    do {
+        main_loop_wait(false);
+    } while (!vub_device->exit_panic || !vub_device->close);
+
+ error:
+    vus_shutdown();
+    exit(EXIT_SUCCESS);
+}

--
2.24.1

#10 lore raw

Re: [PATCH v2 3/5] a standone-alone tool to directly share disk image file via vhost-user protocol

Quoted 7 lines · Coiby Xu

vhost-user-blk can have played as vhost-user backend but it only supports raw file and don't support VIRTIO_BLK_T_DISCARD and VIRTIO_BLK_T_WRITE_ZEROES operations on raw file (ioctl(fd, BLKDISCARD) is only valid for real block device).

---
 qemu-vu.c | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 264 insertions(+)
 create mode 100644 qemu-vu.c

Kevin has been working on qemu-storage-daemon, a tool for running NBD exports, block jobs, and other storage features that are not part of a guest. I think qemu-storage-daemon would be the appropriate tool for running vhost-user-blk servers. A dedicated binary is not necessary.

Stefan

author
#11 lore raw

Re: [PATCH v2 3/5] a standone-alone tool to directly share disk image file via vhost-user protocol

Excellent! I will add an option (or object property) for vhost-user-blk server oject which will tell the daemon process to exit when the client disconnects, thus "make check-qtest" will not get held by this daemon process. After that since Kevin's qemu-storage-daemon support "-object" option
(https://patchew.org/QEMU/[email protected]/[email protected]/)
and vhost-user-server is a user-creatable QOM object, it will work out of the box.

I'm curious when will be formal version of qemu-storage-daemon finished so I can take advantage of it? Or should I apply the RFC PATCHes to my working branch directly and submit them together with the patches on vhost-user-blk server feature when posting v3?

Quoted 10 lines · Stefan Hajnoczi
Quoted 7 lines · Coiby Xu

vhost-user-blk can have played as vhost-user backend but it only supports raw file and don't support VIRTIO_BLK_T_DISCARD and VIRTIO_BLK_T_WRITE_ZEROES operations on raw file (ioctl(fd, BLKDISCARD) is only valid for real block device).

---
 qemu-vu.c | 264 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 264 insertions(+)
 create mode 100644 qemu-vu.c
Quoted 6 lines

Kevin has been working on qemu-storage-daemon, a tool for running NBD exports, block jobs, and other storage features that are not part of a guest. I think qemu-storage-daemon would be the appropriate tool for running vhost-user-blk servers. A dedicated binary is not necessary.

Stefan

--
Best regards,
Coiby

#12 lore raw

Re: [PATCH v2 3/5] a standone-alone tool to directly share disk image file via vhost-user protocol

Am 17.01.2020 um 09:12 hat Coiby Xu geschrieben:

Quoted 8 lines

Excellent! I will add an option (or object property) for vhost-user-blk server oject which will tell the daemon process to exit when the client disconnects, thus "make check-qtest" will not get held by this daemon process. After that since Kevin's qemu-storage-daemon support "-object" option
(https://patchew.org/QEMU/[email protected]/[email protected]/)
and vhost-user-server is a user-creatable QOM object, it will work out of the box.

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Would it make sense to prefix the object type name with "x-" so we can later retire it from the external user interface without a deprecation period?

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Quoted 4 lines

I'm curious when will be formal version of qemu-storage-daemon finished so I can take advantage of it? Or should I apply the RFC PATCHes to my working branch directly and submit them together with the patches on vhost-user-blk server feature when posting v3?

It's the next thing I'm planning to work on after completing the coroutine-base QMP handlers (which I hope to get finished very soon).

For the time being I would suggest that you put any patches that depend on qemu-storage-daemon (if you do need it) at the end of your series so that we could apply the first part even if the storage daemon isn't in yet.

The latest version of my patches is at:

git://repo.or.cz/qemu/kevin.git storage-daemon

But if you just need something for testing your code, I think it would even make sense if you kept your standalone tool around (even though we'll never merge it) and we'll deal with integration in the storage daemon once both parts are ready.

Kevin

author
#13 lore raw

Re: [PATCH v2 3/5] a standone-alone tool to directly share disk image file via vhost-user protocol

Hi Kevin,

Quoted 4 lines

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Does it mean TYPE_USER_CREATABLE interface in QOM will become deprecated in the future? I'm curious what are the reasons for making QOM object no user creatable? Because we may still need to start vhost-user block device backend through HMP or QMP instead of stating it as a standalone-alone daemon.

Quoted 2 lines

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Accroding to https://patchew.org/QEMU/[email protected]/[email protected]/,

qemu-vu.c line 70

Quoted 6 lines
qemu-vu.c new file +6 -0
+static bool exit_requested = false;
+
+void qemu_system_killed(int signal, pid_t pid)
+{
+    exit_requested = true;
+}

if exit_requested = true, qemu-storage-daemon will exit the main loop and then quit. So is calling qemu_system_killed by what you means "to send an explicit 'quit' command to the daemon"?

Quoted 47 lines · Kevin Wolf

Am 17.01.2020 um 09:12 hat Coiby Xu geschrieben:

Quoted 8 lines

Excellent! I will add an option (or object property) for vhost-user-blk server oject which will tell the daemon process to exit when the client disconnects, thus "make check-qtest" will not get held by this daemon process. After that since Kevin's qemu-storage-daemon support "-object" option
(https://patchew.org/QEMU/[email protected]/[email protected]/)
and vhost-user-server is a user-creatable QOM object, it will work out of the box.

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Would it make sense to prefix the object type name with "x-" so we can later retire it from the external user interface without a deprecation period?

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Quoted 4 lines

I'm curious when will be formal version of qemu-storage-daemon finished so I can take advantage of it? Or should I apply the RFC PATCHes to my working branch directly and submit them together with the patches on vhost-user-blk server feature when posting v3?

It's the next thing I'm planning to work on after completing the coroutine-base QMP handlers (which I hope to get finished very soon).

For the time being I would suggest that you put any patches that depend on qemu-storage-daemon (if you do need it) at the end of your series so that we could apply the first part even if the storage daemon isn't in yet.

The latest version of my patches is at:

git://repo.or.cz/qemu/kevin.git storage-daemon

But if you just need something for testing your code, I think it would even make sense if you kept your standalone tool around (even though we'll never merge it) and we'll deal with integration in the storage daemon once both parts are ready.

Kevin

Signature · 2 lines
Best regards,
Coiby
#14 lore raw

Re: [PATCH v2 3/5] a standone-alone tool to directly share disk image file via vhost-user protocol

Am 31.01.2020 um 17:42 hat Coiby Xu geschrieben:

Quoted 10 lines
Quoted 4 lines

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Does it mean TYPE_USER_CREATABLE interface in QOM will become deprecated in the future? I'm curious what are the reasons for making QOM object no user creatable? Because we may still need to start vhost-user block device backend through HMP or QMP instead of stating it as a standalone-alone daemon.

Not in general, but if we have something like a block-export-add QMP command, the QOM interface would be redundant. We could still leave it there and have both a low-level and a high-level interface, but whether we would want to is something we still have to decide.

Quoted 5 lines
Quoted 2 lines

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Accroding to https://patchew.org/QEMU/[email protected]/[email protected]/,

Quoted 10 lines

qemu-vu.c line 70

Quoted 6 lines
qemu-vu.c new file +6 -0
+static bool exit_requested = false;
+
+void qemu_system_killed(int signal, pid_t pid)
+{
+    exit_requested = true;
+}

if exit_requested = true, qemu-storage-daemon will exit the main loop and then quit. So is calling qemu_system_killed by what you means "to send an explicit 'quit' command to the daemon"?

qemu_system_killed() is call in the signal handlers for, amongst others, SIGTERM and SIGINT. This is one way to stop the storage daemon (for manual use, sending SIGINT with Ctrl-C is probably the easiest way).

What I actually meant is the 'quit' QMP command which will cause qmp_quit() to be run, which contains the same code. But if sending a signal is more convenient, that's just as good.

Kevin

Quoted 54 lines · Kevin Wolf
Quoted 47 lines

Am 17.01.2020 um 09:12 hat Coiby Xu geschrieben:

Quoted 8 lines

Excellent! I will add an option (or object property) for vhost-user-blk server oject which will tell the daemon process to exit when the client disconnects, thus "make check-qtest" will not get held by this daemon process. After that since Kevin's qemu-storage-daemon support "-object" option
(https://patchew.org/QEMU/[email protected]/[email protected]/)
and vhost-user-server is a user-creatable QOM object, it will work out of the box.

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Would it make sense to prefix the object type name with "x-" so we can later retire it from the external user interface without a deprecation period?

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Quoted 4 lines

I'm curious when will be formal version of qemu-storage-daemon finished so I can take advantage of it? Or should I apply the RFC PATCHes to my working branch directly and submit them together with the patches on vhost-user-blk server feature when posting v3?

It's the next thing I'm planning to work on after completing the coroutine-base QMP handlers (which I hope to get finished very soon).

For the time being I would suggest that you put any patches that depend on qemu-storage-daemon (if you do need it) at the end of your series so that we could apply the first part even if the storage daemon isn't in yet.

The latest version of my patches is at:

git://repo.or.cz/qemu/kevin.git storage-daemon

But if you just need something for testing your code, I think it would even make sense if you kept your standalone tool around (even though we'll never merge it) and we'll deal with integration in the storage daemon once both parts are ready.

Kevin

Signature · 2 lines
Best regards,
Coiby
author
#15 lore raw

Re: [PATCH v2 3/5] a standone-alone tool to directly share disk image file via vhost-user protocol

Quoted 11 lines
Quoted 10 lines
Quoted 4 lines

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Does it mean TYPE_USER_CREATABLE interface in QOM will become deprecated in the future? I'm curious what are the reasons for making QOM object no user creatable? Because we may still need to start vhost-user block device backend through HMP or QMP instead of stating it as a standalone-alone daemon.

Quoted 4 lines

Not in general, but if we have something like a block-export-add QMP command, the QOM interface would be redundant. We could still leave it there and have both a low-level and a high-level interface, but whether we would want to is something we still have to decide.

I see. So QOM interface will still be used as a low-level interface. In the draft version, vhost-user-blk-server is started using specific command vhost_user_server_start/vhost_user_server_stop which proivide interfaces easier for implementing block-export-add QMP command. But in later versions, only object_add/object_del syntax is supported to start/stop vhost-user-blk-server. I'll keep an eye on how the storage daemon develops and adapt my code accordingly.

Quoted 38 lines · Kevin Wolf

Am 31.01.2020 um 17:42 hat Coiby Xu geschrieben:

Quoted 10 lines
Quoted 4 lines

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Does it mean TYPE_USER_CREATABLE interface in QOM will become deprecated in the future? I'm curious what are the reasons for making QOM object no user creatable? Because we may still need to start vhost-user block device backend through HMP or QMP instead of stating it as a standalone-alone daemon.

Not in general, but if we have something like a block-export-add QMP command, the QOM interface would be redundant. We could still leave it there and have both a low-level and a high-level interface, but whether we would want to is something we still have to decide.

Quoted 5 lines
Quoted 2 lines

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Accroding to https://patchew.org/QEMU/[email protected]/[email protected]/,

Quoted 10 lines

qemu-vu.c line 70

Quoted 6 lines
qemu-vu.c new file +6 -0
+static bool exit_requested = false;
+
+void qemu_system_killed(int signal, pid_t pid)
+{
+    exit_requested = true;
+}

if exit_requested = true, qemu-storage-daemon will exit the main loop and then quit. So is calling qemu_system_killed by what you means "to send an explicit 'quit' command to the daemon"?

qemu_system_killed() is call in the signal handlers for, amongst others, SIGTERM and SIGINT. This is one way to stop the storage daemon (for manual use, sending SIGINT with Ctrl-C is probably the easiest way).

Quoted 61 lines

What I actually meant is the 'quit' QMP command which will cause qmp_quit() to be run, which contains the same code. But if sending a signal is more convenient, that's just as good.

Kevin

Quoted 54 lines · Kevin Wolf
Quoted 38 lines

Am 17.01.2020 um 09:12 hat Coiby Xu geschrieben:

Quoted 8 lines

Excellent! I will add an option (or object property) for vhost-user-blk server oject which will tell the daemon process to exit when the client disconnects, thus "make check-qtest" will not get held by this daemon process. After that since Kevin's qemu-storage-daemon support "-object" option
(https://patchew.org/QEMU/[email protected]/[email protected]/)
and vhost-user-server is a user-creatable QOM object, it will work out of the box.

Yes, I think at least for the moment it should work fine this way. Eventually, I'd like to integrate it with --export (and associated QMP commands, which are still to be created), too. Maybe at that point we want to make the QOM object not user creatable any more.

Would it make sense to prefix the object type name with "x-" so we can later retire it from the external user interface without a deprecation period?

As for test cases, do you think it would be hard to just modify the tests to send an explicit 'quit' command to the daemon?

Quoted 4 lines

I'm curious when will be formal version of qemu-storage-daemon finished so I can take advantage of it? Or should I apply the RFC PATCHes to my working branch directly and submit them together with the patches on vhost-user-blk server feature when posting v3?

It's the next thing I'm planning to work on after completing the coroutine-base QMP handlers (which I hope to get finished very soon).

For the time being I would suggest that you put any patches that depend on qemu-storage-daemon (if you do need it) at the end of your series so that we could apply the first part even if the storage daemon isn't in yet.

The latest version of my patches is at:

Quoted 9 lines

git://repo.or.cz/qemu/kevin.git storage-daemon

But if you just need something for testing your code, I think it would even make sense if you kept your standalone tool around (even though we'll never merge it) and we'll deal with integration in the storage daemon once both parts are ready.

Kevin

--
Best regards,
Coiby

--
Best regards,
Coiby

author
#16 lore raw

PATCH v2 4/5 new qTest case for the vhost-user-blk device backend

In reply to Coiby Xu

This test case has the same tests as tests/virtio-blk-test.c except for tests have block_resize.

---
 tests/libqos/vhost-user-blk.c | 125 ++++++
 tests/libqos/vhost-user-blk.h |  44 +++
 tests/vhost-user-blk-test.c   | 691 ++++++++++++++++++++++++++++++++++
 3 files changed, 860 insertions(+)
 create mode 100644 tests/libqos/vhost-user-blk.c
 create mode 100644 tests/libqos/vhost-user-blk.h
 create mode 100644 tests/vhost-user-blk-test.c
tests/libqos/vhost-user-blk.c new file +125 -0
new file mode 100644
index 0000000000..1f8e6eec7e
@@ -0,0 +1,125 @@
+/*
+ * libqos driver framework
+ *
+ * Copyright (c) 2018 Emanuele Giuseppe Esposito <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "qemu/module.h"
+#include "standard-headers/linux/virtio_blk.h"
+#include "libqos/qgraph.h"
+#include "libqos/vhost-user-blk.h"
+
+#define PCI_SLOT                0x04
+#define PCI_FN                  0x00
+
+/* virtio-blk-device */
+static void *qvhost_user_blk_get_driver(QVhostUserBlk *v_blk,
+                                    const char *interface)
+{
+    if (!g_strcmp0(interface, "vhost-user-blk")) {
+        return v_blk;
+    }
+    if (!g_strcmp0(interface, "virtio")) {
+        return v_blk->vdev;
+    }
+
+    fprintf(stderr, "%s not present in vhost-user-blk-device\n", interface);
+    g_assert_not_reached();
+}
+
+static void *qvhost_user_blk_device_get_driver(void *object,
+                                           const char *interface)
+{
+    QVhostUserBlkDevice *v_blk = object;
+    return qvhost_user_blk_get_driver(&v_blk->blk, interface);
+}
+
+static void *vhost_user_blk_device_create(void *virtio_dev,
+                                      QGuestAllocator *t_alloc,
+                                      void *addr)
+{
+    QVhostUserBlkDevice *vhost_user_blk = g_new0(QVhostUserBlkDevice, 1);
+    QVhostUserBlk *interface = &vhost_user_blk->blk;
+
+    interface->vdev = virtio_dev;
+
+    vhost_user_blk->obj.get_driver = qvhost_user_blk_device_get_driver;
+
+    return &vhost_user_blk->obj;
+}
+
+/* virtio-blk-pci */
+static void *qvhost_user_blk_pci_get_driver(void *object, const char *interface)
+{
+    QVhostUserBlkPCI *v_blk = object;
+    if (!g_strcmp0(interface, "pci-device")) {
+        return v_blk->pci_vdev.pdev;
+    }
+    return qvhost_user_blk_get_driver(&v_blk->blk, interface);
+}
+
+static void *vhost_user_blk_pci_create(void *pci_bus, QGuestAllocator *t_alloc,
+                                      void *addr)
+{
+    QVhostUserBlkPCI *vhost_user_blk = g_new0(QVhostUserBlkPCI, 1);
+    QVhostUserBlk *interface = &vhost_user_blk->blk;
+    QOSGraphObject *obj = &vhost_user_blk->pci_vdev.obj;
+
+    virtio_pci_init(&vhost_user_blk->pci_vdev, pci_bus, addr);
+    interface->vdev = &vhost_user_blk->pci_vdev.vdev;
+
+    g_assert_cmphex(interface->vdev->device_type, ==, VIRTIO_ID_BLOCK);
+
+    obj->get_driver = qvhost_user_blk_pci_get_driver;
+
+    return obj;
+}
+
+static void vhost_user_blk_register_nodes(void)
+{
+    /* FIXME: every test using these two nodes needs to setup a
+     * -drive,id=drive0 otherwise QEMU is not going to start.
+     * Therefore, we do not include "produces" edge for virtio
+     * and pci-device yet.
+    */
+
+    char *arg = g_strdup_printf("id=drv0,chardev=char1,addr=%x.%x",
+                                PCI_SLOT, PCI_FN);
+
+    QPCIAddress addr = {
+        .devfn = QPCI_DEVFN(PCI_SLOT, PCI_FN),
+    };
+
+    QOSGraphEdgeOptions opts = { };
+
+    /* virtio-blk-device */
+    /** opts.extra_device_opts = "drive=drive0"; */
+    qos_node_create_driver("vhost-user-blk-device", vhost_user_blk_device_create);
+    qos_node_consumes("vhost-user-blk-device", "virtio-bus", &opts);
+    qos_node_produces("vhost-user-blk-device", "vhost-user-blk");
+
+    /* virtio-blk-pci */
+    opts.extra_device_opts = arg;
+    add_qpci_address(&opts, &addr);
+    qos_node_create_driver("vhost-user-blk-pci", vhost_user_blk_pci_create);
+    qos_node_consumes("vhost-user-blk-pci", "pci-bus", &opts);
+    qos_node_produces("vhost-user-blk-pci", "vhost-user-blk");
+
+    g_free(arg);
+}
+
+libqos_init(vhost_user_blk_register_nodes);
tests/libqos/vhost-user-blk.h new file +44 -0

44 lines not on this page · raw message

tests/vhost-user-blk-test.c new file +691 -0

691 lines not on this page · raw message

--
2.24.1

author
#17 lore raw

PATCH v2 5/5 building configuration files changes

In reply to Coiby Xu

libvhost-user depends on sys/poll.h, sys/socket.h and eventfd. Although Windows has the equivalent Winsock, it doesn't have eventfd. So only enable this feature on Linux.

---
 Makefile               | 1 +
 Makefile.objs          | 2 +-
 Makefile.target        | 1 +
 configure              | 2 +-
 tests/Makefile.include | 5 ++++-
 5 files changed, 8 insertions(+), 3 deletions(-)
Makefile +4 -0
index 6b5ad1121b..80c514f96b 100644
@@ -558,6 +558,10 @@ qemu-img.o: qemu-img-cmds.h
 
 qemu-img$(EXESUF): qemu-img.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS)
 qemu-nbd$(EXESUF): qemu-nbd.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS)
+
+ifdef CONFIG_LINUX
+qemu-vu$(EXESUF): qemu-vu.o blockdev-vu.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS) libvhost-user.a
+endif
 qemu-io$(EXESUF): qemu-io.o $(authz-obj-y) $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) $(COMMON_LDADDS)
 
 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o $(COMMON_LDADDS)
Makefile.objs +1 -0
index 7c1e50f9d6..5404daca00 100644
@@ -53,6 +53,7 @@ common-obj-$(CONFIG_WIN32) += os-win32.o
 common-obj-$(CONFIG_POSIX) += os-posix.o
 
 common-obj-$(CONFIG_LINUX) += fsdev/
+common-obj-$(CONFIG_LINUX) += blockdev-vu.o
 
 common-obj-y += accel/
 common-obj-y += migration/
Makefile.target +3 -0
index 6e61f607b1..d5405976ce 100644
@@ -159,6 +159,9 @@ obj-y += monitor/
 obj-y += qapi/
 obj-y += memory.o
 obj-y += memory_mapping.o
+ifdef CONFIG_LINUX
+obj-y += ../contrib/libvhost-user/libvhost-user.o
+endif
 obj-y += migration/ram.o
 LIBS := $(libs_softmmu) $(LIBS)
 
configure +3 -0
index 0ce2c0354a..8ad4da57be 100755
@@ -6169,6 +6169,9 @@ if test "$want_tools" = "yes" ; then
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
     tools="qemu-nbd\$(EXESUF) $tools"
   fi
+  if [ "$linux" = "yes" ] ; then
+    tools="qemu-vu\$(EXESUF) $tools"
+  fi
   if [ "$ivshmem" = "yes" ]; then
     tools="ivshmem-client\$(EXESUF) ivshmem-server\$(EXESUF) $tools"
   fi
tests/Makefile.include +4 -1
index 49e3b0d319..33bd48247b 100644
@@ -746,6 +746,7 @@ qos-test-obj-y += tests/libqos/virtio.o
 qos-test-obj-$(CONFIG_VIRTFS) += tests/libqos/virtio-9p.o
 qos-test-obj-y += tests/libqos/virtio-balloon.o
 qos-test-obj-y += tests/libqos/virtio-blk.o
+qos-test-obj-y += tests/libqos/vhost-user-blk.o
 qos-test-obj-y += tests/libqos/virtio-mmio.o
 qos-test-obj-y += tests/libqos/virtio-net.o
 qos-test-obj-y += tests/libqos/virtio-pci.o
@@ -788,6 +789,7 @@ qos-test-obj-$(CONFIG_VHOST_NET_USER) += tests/vhost-user-test.o $(chardev-obj-y
 qos-test-obj-y += tests/virtio-test.o
 qos-test-obj-$(CONFIG_VIRTFS) += tests/virtio-9p-test.o
 qos-test-obj-y += tests/virtio-blk-test.o
+qos-test-obj-$(CONFIG_LINUX) += tests/vhost-user-blk-test.o
 qos-test-obj-y += tests/virtio-net-test.o
 qos-test-obj-y += tests/virtio-rng-test.o
 qos-test-obj-y += tests/virtio-scsi-test.o
@@ -935,7 +937,8 @@ endef
 $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: %-softmmu/all $(check-qtest-y)
 	$(call do_test_human,$(check-qtest-$*-y) $(check-qtest-generic-y), \
 	  QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
-	  QTEST_QEMU_IMG=qemu-img$(EXESUF))
+	  QTEST_QEMU_IMG=./qemu-img$(EXESUF) \
+	  QTEST_QEMU_VU_BINARY=./qemu-vu$(EXESUF))
 
 check-unit: $(check-unit-y)
 	$(call do_test_human, $^)

--
2.24.1

#18 lore raw

Re: [PATCH v2 5/5] building configuration files changes

Am 14.01.2020 um 15:06 hat Coiby Xu geschrieben:

Quoted 5 lines

libvhost-user depends on sys/poll.h, sys/socket.h and eventfd. Although Windows has the equivalent Winsock, it doesn't have eventfd. So only enable this feature on Linux.

This shouldn't be a separate patch at the end of the series, but each patch should make sure that the files it adds are actually built. This way you can apply individual patches and they still make sense. It also helps for bisecting.

While applying the patches locally for review, I noticed that there is a merge conflict with recent changes that moved qos test cases to a different subdirectory. I resolved the conflict manually, but for v3 you should rebase.

Kevin