GlusterFS support in QEMU - v2

+ char *bounce;

+ struct BDRVGlusterState *s;

+ int cancelled;

block/gluster.c new file +10 -0

+
+typedef struct BDRVGlusterState {
+    struct glfs *glfs;
+    int fds[2];
+    int open_flags;
+    struct glfs_fd *fd;
+    int qemu_aio_count;
+    int event_reader_pos;
+    GlusterCBKData *event_gcbk;
+} BDRVGlusterState;

block/gluster.c new file +17 -0

+/*
+ * file=protocol:server@port:volname:image
+ */
+static int qemu_gluster_parsename(GlusterConf *c, const char *filename)
+{
+    char *file = g_strdup(filename);
+    char *token, *next_token, *saveptr;
+    char *token_s, *next_token_s, *saveptr_s;
+    int ret = -EINVAL;
+
+    /* Discard the protocol */
+    token = strtok_r(file, ":", &saveptr);
+    if (!token) {
+        goto out;
+    }
+
+    /* server@port */

block/gluster.c new file +14 -0

+static struct glfs *qemu_gluster_init(GlusterConf *c, const char *filename)
+{
+    struct glfs *glfs = NULL;
+    int ret;
+
+    ret = qemu_gluster_parsename(c, filename);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    glfs = glfs_new(c->volname);
+    if (!glfs) {
+        goto out;

block/gluster.c new file +16 -0

+     * Need to use defines like gf_loglevel_t:GF_LOG_INFO instead of
+     * hard coded values like 7 here.
+     */
+    ret = glfs_set_logging(glfs, "/tmp/qemu-gluster.log", 7);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = glfs_init(glfs);
+    if (ret < 0) {
+        goto out;
+    }
+    return glfs;
+
+out:
+    if (glfs) {

block/gluster.c new file +14 -0

+static int qemu_gluster_create(const char *filename,
+        QEMUOptionParameter *options)
+{
+    struct glfs *glfs;
+    struct glfs_fd *fd;
+    GlusterConf *c = g_malloc(sizeof(GlusterConf));
+    int ret = 0;
+    int64_t total_size = 0;
+
+    glfs = qemu_gluster_init(c, filename);
+    if (!glfs) {
+        ret = -errno;
+        goto out;
+    }

block/gluster.c line 24

Quoted 3 lines · Bharata B Rao

block/gluster.c new file +3 -0

+typedef struct GlusterAIOCB {
+    BlockDriverAIOCB common;
+    QEMUIOVector *qiov;

The qiov field is unused.

Quoted 1 line

+ char *bounce;

Unused.

Quoted 2 lines

+ struct BDRVGlusterState *s;

You can get this through common.bs->opaque, but if you like having a shortcut, that's fine.

Quoted 1 line

+ int cancelled;

bool

block/gluster.c line 35

Quoted 9 lines

block/gluster.c new file +8 -0

+} GlusterAIOCB;
+
+typedef struct GlusterCBKData {
+    GlusterAIOCB *acb;
+    struct BDRVGlusterState *s;
+    int64_t size;
+    int ret;
+} GlusterCBKData;

I think GlusterCBKData could just be part of GlusterAIOCB. That would simplify the code a little and avoid some malloc/free.

Quoted 15 lines

block/gluster.c new file +14 -0

+
+typedef struct BDRVGlusterState {
+    struct glfs *glfs;
+    int fds[2];
+    int open_flags;
+    struct glfs_fd *fd;
+    int qemu_aio_count;
+    int event_reader_pos;
+    GlusterCBKData *event_gcbk;
+} BDRVGlusterState;
+
+#define GLUSTER_FD_READ 0
+#define GLUSTER_FD_WRITE 1
+

block/gluster.c line 55

Quoted 6 lines

block/gluster.c new file +6 -0

+static void qemu_gluster_complete_aio(GlusterCBKData *gcbk)
+{
+    GlusterAIOCB *acb = gcbk->acb;
+    int ret;
+
+    if (acb->cancelled) {

Where does cancelled get set?

block/gluster.c line 65

Quoted 11 lines

block/gluster.c new file +10 -0

+        qemu_aio_release(acb);
+        goto done;
+    }
+
+    if (gcbk->ret == gcbk->size) {
+        ret = 0; /* Success */
+    } else if (gcbk->ret < 0) {
+        ret = gcbk->ret; /* Read/Write failed */
+    } else {
+        ret = -EINVAL; /* Partial read/write - fail it */

EINVAL is for invalid arguments. EIO would be better.

Quoted 23 lines

block/gluster.c new file +22 -0

+/*
+ * file=protocol:server@port:volname:image
+ */
+static int qemu_gluster_parsename(GlusterConf *c, const char *filename)
+{
+    char *file = g_strdup(filename);
+    char *token, *next_token, *saveptr;
+    char *token_s, *next_token_s, *saveptr_s;
+    int ret = -EINVAL;
+
+    /* Discard the protocol */
+    token = strtok_r(file, ":", &saveptr);
+    if (!token) {
+        goto out;
+    }
+
+    /* server@port */
+    next_token = strtok_r(NULL, ":", &saveptr);
+    if (!next_token) {
+        goto out;
+    }
+    if (strchr(next_token, '@')) {

block/gluster.c line 128

Quoted 5 lines

block/gluster.c new file +5 -0

+        token_s = strtok_r(next_token, "@", &saveptr_s);
+        if (!token_s) {
+            goto out;
+        }
+        strncpy(c->server, token_s, HOST_NAME_MAX);

strncpy(3) will not NUL-terminate when token_s is HOST_NAME_MAX characters long. QEMU has cutils.c:pstrcpy().

When the argument is too long we should probably report an error instead of truncating.

Same below.

block/gluster.c line 133

Quoted 5 lines

block/gluster.c new file +5 -0

+        next_token_s = strtok_r(NULL, "@", &saveptr_s);
+        if (!next_token_s) {
+            goto out;
+        }
+        c->port = atoi(next_token_s);

No error checking. If the input is invalid an error message would help the user here.

Quoted 20 lines

block/gluster.c new file +19 -0

+static struct glfs *qemu_gluster_init(GlusterConf *c, const char *filename)
+{
+    struct glfs *glfs = NULL;
+    int ret;
+
+    ret = qemu_gluster_parsename(c, filename);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    glfs = glfs_new(c->volname);
+    if (!glfs) {
+        goto out;
+    }
+
+    ret = glfs_set_volfile_server(glfs, "socket", c->server, c->port);
+    if (ret < 0) {
+        goto out;

block/gluster.c line 181

Quoted 5 lines

block/gluster.c new file +5 -0

+    }
+
+    /*
+     * TODO: Logging is not necessary but instead nice to have.
+     * Can QEMU optionally log into a standard place ?

QEMU prints to stderr, can you do that here too? The global log file is not okay, especially when multiple QEMU instances are running.

Quoted 21 lines

block/gluster.c new file +20 -0

+     * Need to use defines like gf_loglevel_t:GF_LOG_INFO instead of
+     * hard coded values like 7 here.
+     */
+    ret = glfs_set_logging(glfs, "/tmp/qemu-gluster.log", 7);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = glfs_init(glfs);
+    if (ret < 0) {
+        goto out;
+    }
+    return glfs;
+
+out:
+    if (glfs) {
+        (void)glfs_fini(glfs);
+    }
+    return NULL;
+}

block/gluster.c line 207

Quoted 6 lines

block/gluster.c new file +6 -0

+
+static int qemu_gluster_open(BlockDriverState *bs, const char *filename,
+    int bdrv_flags)
+{
+    BDRVGlusterState *s = bs->opaque;
+    GlusterConf *c = g_malloc(sizeof(GlusterConf));

Can this be allocated on the stack?

block/gluster.c line 216

Quoted 10 lines

block/gluster.c new file +9 -0

+    int ret;
+
+    s->glfs = qemu_gluster_init(c, filename);
+    if (!s->glfs) {
+        ret = -errno;
+        goto out;
+    }
+
+    s->open_flags |=  O_BINARY;

Can open_flags be a local variable?

Quoted 19 lines

block/gluster.c new file +18 -0

+static int qemu_gluster_create(const char *filename,
+        QEMUOptionParameter *options)
+{
+    struct glfs *glfs;
+    struct glfs_fd *fd;
+    GlusterConf *c = g_malloc(sizeof(GlusterConf));
+    int ret = 0;
+    int64_t total_size = 0;
+
+    glfs = qemu_gluster_init(c, filename);
+    if (!glfs) {
+        ret = -errno;
+        goto out;
+    }
+
+    /* Read out options */
+    while (options && options->name) {
+        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {

block/gluster.c line 279

Quoted 6 lines

block/gluster.c new file +6 -0

+            total_size = options->value.n / BDRV_SECTOR_SIZE;
+        }
+        options++;
+    }
+
+    fd = glfs_creat(glfs, c->image, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, S_IRWXU);

Why set the execute permission bit?

block/gluster.c line 450

Quoted 9 lines

block/gluster.c new file +8 -0

+static void qemu_gluster_close(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+
+    if (s->fd) {
+        glfs_close(s->fd);
+        s->fd = NULL;
+    }

Why not call glfs_fini() here?

Quoted 13 lines · Stefan Hajnoczi

block/gluster.c line 35

Quoted 8 lines · Bharata B Rao

block/gluster.c new file +8 -0

+} GlusterAIOCB;
+
+typedef struct GlusterCBKData {
+    GlusterAIOCB *acb;
+    struct BDRVGlusterState *s;
+    int64_t size;
+    int ret;
+} GlusterCBKData;

I think GlusterCBKData could just be part of GlusterAIOCB. That would simplify the code a little and avoid some malloc/free.

Are you suggesting to put a field

GlusterCBKData gcbk;

inside GlusterAIOCB and use gcbk from there or

Are you suggesting that I make the fields of GlusterCBKData part of GlusterAIOCB and get rid of GlusterCBKData altogether ? This means I would have to pass the GlusterAIOCB to gluster async calls and update its fields from gluster callback routine. I can do this, but I am not sure if you can touch the fields of GlusterAIOCB in non-QEMU threads (gluster callback thread).

Quoted 2 lines

When the argument is too long we should probably report an error instead of truncating.

Or should we let gluster APIs to flag an error with truncated server and volume names ?

Quoted 19 lines

Quoted 19 lines

block/gluster.c new file +19 -0

+static struct glfs *qemu_gluster_init(GlusterConf *c, const char *filename)
+{
+    struct glfs *glfs = NULL;
+    int ret;
+
+    ret = qemu_gluster_parsename(c, filename);
+    if (ret < 0) {
+        errno = -ret;
+        goto out;
+    }
+
+    glfs = glfs_new(c->volname);
+    if (!glfs) {
+        goto out;
+    }
+
+    ret = glfs_set_volfile_server(glfs, "socket", c->server, c->port);
+    if (ret < 0) {
+        goto out;

Quoted 8 lines

block/gluster.c line 181

Quoted 5 lines

block/gluster.c new file +5 -0

+    }
+
+    /*
+     * TODO: Logging is not necessary but instead nice to have.
+     * Can QEMU optionally log into a standard place ?

QEMU prints to stderr, can you do that here too? The global log file is not okay, especially when multiple QEMU instances are running.

Ok, I can do glfs_set_logging(glfs, "/dev/stderr", loglevel);

Quoted 20 lines

Quoted 20 lines

block/gluster.c new file +20 -0

+     * Need to use defines like gf_loglevel_t:GF_LOG_INFO instead of
+     * hard coded values like 7 here.
+     */
+    ret = glfs_set_logging(glfs, "/tmp/qemu-gluster.log", 7);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = glfs_init(glfs);
+    if (ret < 0) {
+        goto out;
+    }
+    return glfs;
+
+out:
+    if (glfs) {
+        (void)glfs_fini(glfs);
+    }
+    return NULL;
+}

Quoted 8 lines

block/gluster.c line 207

Quoted 6 lines

block/gluster.c new file +6 -0

+
+static int qemu_gluster_open(BlockDriverState *bs, const char *filename,
+    int bdrv_flags)
+{
+    BDRVGlusterState *s = bs->opaque;
+    GlusterConf *c = g_malloc(sizeof(GlusterConf));

Can this be allocated on the stack?

It consists of PATH_MAX(4096), HOST_NAME_MAX(255) and GLUSTERD_MAX_VOLUME_NAME (1000). A bit heavy to be on stack ?

We now
specify the gluster backed image like this:

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.
- 'volname' is the name of the gluster volume which contains the VM image.
- 'image' is the path to the actual VM image in the gluster volume.

Note that we are no longer using volfiles directly and use volume names instead. For this to work, gluster management daemon (glusterd) needs to be running on the QEMU node.

Scenarios
---------
Base: QEMU boots from directly from image on gluster brick.
Fuse mount: QEMU boots from VM image on gluster FUSE mount.
Fuse bypass: QEMU uses gluster protocol and uses standard client volfile. Fuse bypass custom: QEMU uses gluster protocol and uses a minimal client
volfile that just has client xlator.
RPC bypass: QEMU uses just posix xlator and doesn't depend on gluster server.

Quoted 14 lines · Bharata B Rao

We now
specify the gluster backed image like this:

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.
- 'volname' is the name of the gluster volume which contains the VM image.
- 'image' is the path to the actual VM image in the gluster volume.

Quoted 3 lines

Note that we are no longer using volfiles directly and use volume names instead. For this to work, gluster management daemon (glusterd) needs to be running on the QEMU node.

glusterd needs to be running on the server that is specified in 'server@port' option.

Quoted 12 lines

Scenarios
---------
Base: QEMU boots from directly from image on gluster brick.
Fuse mount: QEMU boots from VM image on gluster FUSE mount.
Fuse bypass: QEMU uses gluster protocol and uses standard client volfile. Fuse bypass custom: QEMU uses gluster protocol and uses a minimal client
volfile that just has client xlator.
RPC bypass: QEMU uses just posix xlator and doesn't depend on gluster server.

For the Fuse mount, Fuse bypass and Fuse bypass custom scenarios, I assume a single local gluster brick is being used. Is this correct?

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.

Note that we are no longer using volfiles directly and use volume names instead. For this to work, gluster management daemon (glusterd) needs to be running on the QEMU node. This limits the QEMU user to access the volumes by the default volfiles that are generated by gluster CLI. This should be fine as long as gluster CLI provides the capability to generate or regenerate volume files for a given volume with the xlator set that QEMU user is interested in. GlusterFS developers tell me that this can be provided with some enhancements to Gluster CLI/glusterd. Note that the custom volume files is typically needed when GlusterFS server is co-located with QEMU in which case it would be beneficial to get rid of client-server overhead and RPC communication overhead.

- As mentioned above, the VM image on gluster volume can be specified like this:
-drive file=gluster:localhost:testvol:/F17,format=gluster

Note that format=gluster is not needed ideally and its a work around I have until libgfapi provides a working connection cleanup routine (glfs_fini()). When the format isn't specified, QEMU figures out the format by doing find_image_format that results in one open and close before opening the image file long term for standard read and write. Gluster connection initialization is done from open and connection termination is done from close. But since glfs_fini() isn't working yet, I am bypassing find_image_format by specifying format=gluster directly which results in just one open and hence I am not limited by glfs_fini().

Quoted 8 lines · Bharata B Rao

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.

'server@port' is weird notation. Normally it is 'server:port' (e.g. URLs). Can you change it?

What about the other transports supported by libgfapi: UNIX domain sockets and RDMA? My reading of glfs.h is that there are 3 connection options:
1. 'transport': 'socket' (default), 'unix', 'rdma'
2. 'host': server hostname for 'socket', path to UNIX domain socket for 'unix', or something else for 'rdma'
3. 'port': TCP port when 'socket' is used. Ignored otherwise.

Unfortunately QEMU block drivers cannot take custom options yet. That would make it possible to cleanly map these connection options and save you from inventing syntax which doesn't expose all options.

In the meantime it would be nice if the syntax exposed all options.

Quoted 12 lines

Note that we are no longer using volfiles directly and use volume names instead. For this to work, gluster management daemon (glusterd) needs to be running on the QEMU node. This limits the QEMU user to access the volumes by the default volfiles that are generated by gluster CLI. This should be fine as long as gluster CLI provides the capability to generate or regenerate volume files for a given volume with the xlator set that QEMU user is interested in. GlusterFS developers tell me that this can be provided with some enhancements to Gluster CLI/glusterd. Note that the custom volume files is typically needed when GlusterFS server is co-located with QEMU in which case it would be beneficial to get rid of client-server overhead and RPC communication overhead.

My knowledge of GlusterFS is limited. Here is what I am thinking:

1. The user cannot specify a local configuration file, you require that there is a glusterd running which provides configuration information.

2. It is currently not possible to bypass RPC because the glusterd managed configuration file doesn't support that.

I'm not sure if these statements are true?

Would you support local volfiles in the future again? Why force users to run glusterd?

Quoted 5 lines

- As mentioned above, the VM image on gluster volume can be specified like this:
-drive file=gluster:localhost:testvol:/F17,format=gluster

Quoted 9 lines

Note that format=gluster is not needed ideally and its a work around I have until libgfapi provides a working connection cleanup routine (glfs_fini()). When the format isn't specified, QEMU figures out the format by doing find_image_format that results in one open and close before opening the image file long term for standard read and write. Gluster connection initialization is done from open and connection termination is done from close. But since glfs_fini() isn't working yet, I am bypassing find_image_format by specifying format=gluster directly which results in just one open and hence I am not limited by glfs_fini().

Has libgfapi been released yet?

Does it have versioning which will
allow the QEMU GlusterFS block driver to build against different versions? I'm just wondering how the pieces will fit together once distros start shipping them.

Quoted 11 lines · Stefan Hajnoczi

Quoted 8 lines · Bharata B Rao

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.

Quoted 2 lines

'server@port' is weird notation. Normally it is 'server:port' (e.g. URLs). Can you change it?

I don't like but, but settled for it since port was optional and : was being used as separator here.

Quoted 9 lines

What about the other transports supported by libgfapi: UNIX domain sockets and RDMA? My reading of glfs.h is that there are 3 connection options:
1. 'transport': 'socket' (default), 'unix', 'rdma'
2. 'host': server hostname for 'socket', path to UNIX domain socket for 'unix', or something else for 'rdma'
3. 'port': TCP port when 'socket' is used. Ignored otherwise.

Quoted 5 lines

Unfortunately QEMU block drivers cannot take custom options yet. That would make it possible to cleanly map these connection options and save you from inventing syntax which doesn't expose all options.

In the meantime it would be nice if the syntax exposed all options.

So without the capability to pass custom options to block drivers, am I forced to keep extending the file= with more and more options ?

file=gluster:transport:server:port:volname:image ?

Looks ugly and not easy to make any particular option optional. If needed I can support this from GlusterFS backend.

Quoted 33 lines · Bharata B Rao

Quoted 11 lines · Stefan Hajnoczi

Quoted 8 lines · Bharata B Rao

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.

Quoted 2 lines

'server@port' is weird notation. Normally it is 'server:port' (e.g. URLs). Can you change it?

I don't like but, but settled for it since port was optional and : was being used as separator here.

Quoted 9 lines

What about the other transports supported by libgfapi: UNIX domain sockets and RDMA? My reading of glfs.h is that there are 3 connection options:
1. 'transport': 'socket' (default), 'unix', 'rdma'
2. 'host': server hostname for 'socket', path to UNIX domain socket for 'unix', or something else for 'rdma'
3. 'port': TCP port when 'socket' is used. Ignored otherwise.

Quoted 5 lines

Unfortunately QEMU block drivers cannot take custom options yet. That would make it possible to cleanly map these connection options and save you from inventing syntax which doesn't expose all options.

In the meantime it would be nice if the syntax exposed all options.

Quoted 7 lines

So without the capability to pass custom options to block drivers, am I forced to keep extending the file= with more and more options ?

file=gluster:transport:server:port:volname:image ?

Looks ugly and not easy to make any particular option optional. If needed I can support this from GlusterFS backend.

Kevin, Markus: Any thoughts on passing options to block drivers? Encoding GlusterFS options into a "filename" string is pretty cumbersome.

in iscsi, i just specify those extra arguments that are required that are not part of the url itself as just command line options :

qemu-system-i386 -iscsi initiator-name=iqn.qemu.test:my-initiator \
    -boot d -drive file=iscsi://127.0.0.1/iqn.qemu.test/1 \
    -cdrom iscsi://127.0.0.1/iqn.qemu.test/2

Here initiator-name is a custom option to the iscsi layer to tell it which name to use when identifying/logging in to the target.

Similar concept could be used by glusterfs ?

Quoted 4 lines

So without the capability to pass custom options to block drivers, am I forced to keep extending the file= with more and more options ?

Quoted 4 lines

file=gluster:transport:server:port:volname:image ?

Looks ugly and not easy to make any particular option optional. If needed I can support this from GlusterFS backend.

Kevin, Markus: Any thoughts on passing options to block drivers? Encoding GlusterFS options into a "filename" string is pretty cumbersome.

-drive file=gluster://server[:port]/volname/image

Quoted 4 lines · Stefan Hajnoczi

Quoted 4 lines

So without the capability to pass custom options to block drivers, am I forced to keep extending the file= with more and more options ?

Quoted 8 lines

Quoted 4 lines

file=gluster:transport:server:port:volname:image ?

Looks ugly and not easy to make any particular option optional. If needed I can support this from GlusterFS backend.

Kevin, Markus: Any thoughts on passing options to block drivers? Encoding GlusterFS options into a "filename" string is pretty cumbersome.

On 07/23/2012 03:28 AM, ronnie sahlberg wrote:> Why not use

Quoted 1 line

Quoted 1 line

-drive file=gluster://server[:port]/volname/image

At which point, options can fit into this URI scheme:

-drive file=gluster://server:port/volname/image?option1=foo&option2=bar

where anything after the ? of the URI can introduce whichever options you need.

Quoted 18 lines · Bharata B Rao

Quoted 8 lines · Stefan Hajnoczi

What about the other transports supported by libgfapi: UNIX domain sockets and RDMA? My reading of glfs.h is that there are 3 connection options:
1. 'transport': 'socket' (default), 'unix', 'rdma'
2. 'host': server hostname for 'socket', path to UNIX domain socket for 'unix', or something else for 'rdma'
3. 'port': TCP port when 'socket' is used. Ignored otherwise.

Quoted 5 lines

Unfortunately QEMU block drivers cannot take custom options yet. That would make it possible to cleanly map these connection options and save you from inventing syntax which doesn't expose all options.

In the meantime it would be nice if the syntax exposed all options.

So without the capability to pass custom options to block drivers, am I forced to keep extending the file= with more and more options ?

Quoted 4 lines

file=gluster:transport:server:port:volname:image ?

Looks ugly and not easy to make any particular option optional. If needed I can support this from GlusterFS backend.

Kevin, Markus: Any thoughts on passing options to block drivers? Encoding GlusterFS options into a "filename" string is pretty cumbersome.

Quoted 13 lines · Bharata B Rao

Quoted 12 lines · Stefan Hajnoczi

Quoted 9 lines · Bharata B Rao

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for
the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.

Quoted 5 lines

Quoted 2 lines

'server@port' is weird notation. Normally it is 'server:port' (e.g. URLs). Can you change it?

I don't like but, but settled for it since port was optional and : was being used as separator here.

Quoted 19 lines

Quoted 8 lines

What about the other transports supported by libgfapi: UNIX domain sockets and RDMA? My reading of glfs.h is that there are 3 connection options:
1. 'transport': 'socket' (default), 'unix', 'rdma'
2. 'host': server hostname for 'socket', path to UNIX domain socket for 'unix', or something else for 'rdma'
3. 'port': TCP port when 'socket' is used. Ignored otherwise.

Quoted 5 lines

Unfortunately QEMU block drivers cannot take custom options yet. That would make it possible to cleanly map these connection options and save you from inventing syntax which doesn't expose all options.

In the meantime it would be nice if the syntax exposed all options.

So without the capability to pass custom options to block drivers, am I forced to keep extending the file= with more and more options ?

file=gluster:transport:server:port:volname:image ?

Quoted 3 lines

Looks ugly and not easy to make any particular option optional. If needed I can
support this from GlusterFS backend.

Kevin, Markus: Any thoughts on passing options to block drivers? Encoding GlusterFS options into a "filename" string is pretty cumbersome.

Quoted 3 lines

2. It is currently not possible to bypass RPC because the glusterd managed configuration file doesn't support that.

It is possible. Gluster already supports custom extensions
to volume names and it is possible to use the required volfile by specifying this custom volname extension.

For eg, if I have a volume named test, by default the volfile used for it will be test-fuse.vol. Currently I can put my own custom volfile into the standard location and get glusterd pick that up. I can specify test.rpcbypass as volname and glusterd will pick test.rpcbypass.vol.

What is currently not supported is the ability to create test.rpcbypass.vol from gluster CLI. I believe that gluster developers are ok with enhancing gluster CLI to support generating/regenerating volfiles for a given volume with custom translator set.

Quoted 6 lines

I'm not sure if these statements are true?

Would you support local volfiles in the future again? Why force users to run glusterd?

I will let gluster folks on CC to answer this and let us know the benefits of always depending on glusterd.

I guess running glusterd would be beneficial when supporting migration. QEMU working from a local volume (with volname=test.rpcbypass) can be easily restarted on a different node by just changing volname to test. glusterd will take care of fetching the right volfile automatically for us.

Quoted 4 lines

Has libgfapi been released yet?

Its part of gluster mainline now.

Quoted 4 lines

Does it have versioning which will
allow the QEMU GlusterFS block driver to build against different versions? I'm just wondering how the pieces will fit together once distros start shipping them.

I request gluster folks on CC to comment about version and shipping information.

Hi,

Here is the v2 patchset for supporting GlusterFS protocol from QEMU.

This set of patches enables QEMU to boot VM images from gluster volumes. This is achieved by adding gluster as a new block backend driver in QEMU. Its already possible to boot from VM images on gluster volumes, but this patchset provides the ability to boot VM images from gluster volumes by by-passing the FUSE layer in gluster. In case the image is present on the local system, it is possible to even bypass client and server translator and hence the RPC overhead.

The major change in this version is to not implement libglusterfs based gluster backend within QEMU but instead use libgfapi. libgfapi library from GlusterFS project provides APIs to access gluster volumes directly. With the use of libgfapi, the specification of gluster backend from QEMU matches more closely with the GlusterFS's way of specifying volumes. We now specify the gluster backed image like this:

-drive file=gluster:server@port:volname:image

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.
- 'volname' is the name of the gluster volume which contains the VM image.
- 'image' is the path to the actual VM image in the gluster volume.

Quoted 21 lines · Bharata B Rao

Hi,

Here is the v2 patchset for supporting GlusterFS protocol from QEMU.

This set of patches enables QEMU to boot VM images from gluster volumes. This is achieved by adding gluster as a new block backend driver in QEMU. Its already possible to boot from VM images on gluster volumes, but this patchset provides the ability to boot VM images from gluster volumes by by-passing the FUSE layer in gluster. In case the image is present on the local system, it is possible to even bypass client and server translator and hence the RPC overhead.

The major change in this version is to not implement libglusterfs based gluster backend within QEMU but instead use libgfapi. libgfapi library from GlusterFS project provides APIs to access gluster volumes directly. With the use of libgfapi, the specification of gluster backend from QEMU matches more closely with the GlusterFS's way of specifying volumes. We now specify the gluster backed image like this:

-drive file=gluster:server@port:volname:image

Quoted 8 lines

- Here 'gluster' is the protocol.
- 'server@port' specifies the server where the volume file specification for the given volume resides. 'port' is the port number on which gluster management daemon (glusterd) is listening. This is optional and if not specified, QEMU will send 0 which will make libgfapi to use the default port.
- 'volname' is the name of the gluster volume which contains the VM image.
- 'image' is the path to the actual VM image in the gluster volume.

I don't think we should be using '@' as a field separator here, when ":" can do that job just fine. In addition we already have a precendent set with the sheepdog driver for using ':' for separating all fields:

-drive file=sheepdog:example.org:6000:imagename

If you want to allow for port number to be omitted, this can be handled thus:

-drive file=sheepdog:example.org::imagename

which is how -chardev deals with omitted port numbers

Regards,
Daniel
--

|: http://berrange.com      -o-    http://www.flickr.com/photos/dberrange/ :|
|: http://libvirt.org              -o-             http://virt-manager.org :|
|: http://autobuild.org       -o-         http://search.cpan.org/~danberr/ :|
|: http://entangle-photo.org       -o-       http://live.gnome.org/gtk-vnc :|