Skip to contentSkip to conversation

rdma: Remove RDMA subsystem and pvrdma device

PATCH In review Philippe Mathieu-Daudé 16 participants 81 messages Mar 28, 2024 – Jun 7, 2024

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

v2 cover rdma: Remove RDMA subsystem and pvrdma device

Since v1:
- split in 3 (Thomas)
- justify gluster removal

Philippe Mathieu-Daudé (3):
hw/rdma: Remove pvrdma device and rdmacm-mux helper
migration: Remove RDMA protocol handling block/gluster: Remove RDMA protocol handling

 MAINTAINERS                                   |   17 -
 docs/about/deprecated.rst                     |    9 -
 docs/about/removed-features.rst               |    4 +
 docs/devel/migration/main.rst                 |    6 -
 docs/pvrdma.txt                               |  345 --
 docs/rdma.txt                                 |  420 --
 docs/system/device-url-syntax.rst.inc         |    4 +-
 docs/system/loongarch/virt.rst                |    2 +-
 docs/system/qemu-block-drivers.rst.inc        |    1 -
 meson.build                                   |   59 -
 qapi/machine.json                             |   17 -
 qapi/migration.json                           |   31 +-
 qapi/qapi-schema.json                         |    1 -
 qapi/rdma.json                                |   38 -
 contrib/rdmacm-mux/rdmacm-mux.h               |   61 -
 hw/rdma/rdma_backend.h                        |  129 -
 hw/rdma/rdma_backend_defs.h                   |   76 -
 hw/rdma/rdma_rm.h                             |   97 -
 hw/rdma/rdma_rm_defs.h                        |  146 -
 hw/rdma/rdma_utils.h                          |   63 -
 hw/rdma/trace.h                               |    1 -
 hw/rdma/vmw/pvrdma.h                          |  144 -
 hw/rdma/vmw/pvrdma_dev_ring.h                 |   46 -
 hw/rdma/vmw/pvrdma_qp_ops.h                   |   28 -
 hw/rdma/vmw/trace.h                           |    1 -
 include/hw/rdma/rdma.h                        |   37 -
 include/monitor/hmp.h                         |    1 -
 .../infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h |  685 ---
 .../infiniband/hw/vmw_pvrdma/pvrdma_verbs.h   |  348 --
 .../standard-headers/rdma/vmw_pvrdma-abi.h    |  310 --
 migration/migration-stats.h                   |    6 +-
 migration/migration.h                         |    9 -
 migration/options.h                           |    2 -
 migration/rdma.h                              |   69 -
 block/gluster.c                               |   39 -
 contrib/rdmacm-mux/main.c                     |  831 ----
 hw/core/machine-qmp-cmds.c                    |   32 -
 hw/rdma/rdma.c                                |   30 -
 hw/rdma/rdma_backend.c                        | 1401 ------
 hw/rdma/rdma_rm.c                             |  812 ----
 hw/rdma/rdma_utils.c                          |  126 -
 hw/rdma/vmw/pvrdma_cmd.c                      |  815 ----
 hw/rdma/vmw/pvrdma_dev_ring.c                 |  141 -
 hw/rdma/vmw/pvrdma_main.c                     |  735 ---
 hw/rdma/vmw/pvrdma_qp_ops.c                   |  298 --
 migration/migration-stats.c                   |    5 +-
 migration/migration.c                         |   31 -
 migration/options.c                           |   16 -
 migration/qemu-file.c                         |    1 -
 migration/ram.c                               |   86 +-
 migration/rdma.c                              | 4184 -----------------
 migration/savevm.c                            |    2 +-
 monitor/qmp-cmds.c                            |    1 -
 Kconfig.host                                  |    3 -
 contrib/rdmacm-mux/meson.build                |    7 -
 hmp-commands-info.hx                          |   13 -
 hw/Kconfig                                    |    1 -
 hw/meson.build                                |    1 -
 hw/rdma/Kconfig                               |    3 -
 hw/rdma/meson.build                           |   12 -
 hw/rdma/trace-events                          |   31 -
 hw/rdma/vmw/trace-events                      |   17 -
 meson_options.txt                             |    4 -
 migration/meson.build                         |    1 -
 migration/trace-events                        |   68 +-
 qapi/meson.build                              |    1 -
 qemu-options.hx                               |    6 -
 .../org.centos/stream/8/build-environment.yml |    1 -
 .../ci/org.centos/stream/8/x86_64/configure   |    3 -
 scripts/ci/setup/build-environment.yml        |    4 -
 scripts/coverity-scan/run-coverity-scan       |    2 +-
 scripts/meson-buildoptions.sh                 |    6 -
 scripts/update-linux-headers.sh               |   27 -
 tests/lcitool/projects/qemu.yml               |    3 -
 tests/migration/guestperf/engine.py           |    4 +-
 75 files changed, 20 insertions(+), 12997 deletions(-)
 delete mode 100644 docs/pvrdma.txt
 delete mode 100644 docs/rdma.txt
 delete mode 100644 qapi/rdma.json
 delete mode 100644 contrib/rdmacm-mux/rdmacm-mux.h
 delete mode 100644 hw/rdma/rdma_backend.h
 delete mode 100644 hw/rdma/rdma_backend_defs.h
 delete mode 100644 hw/rdma/rdma_rm.h
 delete mode 100644 hw/rdma/rdma_rm_defs.h
 delete mode 100644 hw/rdma/rdma_utils.h
 delete mode 100644 hw/rdma/trace.h
 delete mode 100644 hw/rdma/vmw/pvrdma.h
 delete mode 100644 hw/rdma/vmw/pvrdma_dev_ring.h
 delete mode 100644 hw/rdma/vmw/pvrdma_qp_ops.h
 delete mode 100644 hw/rdma/vmw/trace.h
 delete mode 100644 include/hw/rdma/rdma.h
 delete mode 100644 include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
 delete mode 100644 include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
 delete mode 100644 include/standard-headers/rdma/vmw_pvrdma-abi.h
 delete mode 100644 migration/rdma.h
 delete mode 100644 contrib/rdmacm-mux/main.c
 delete mode 100644 hw/rdma/rdma.c
 delete mode 100644 hw/rdma/rdma_backend.c
 delete mode 100644 hw/rdma/rdma_rm.c
 delete mode 100644 hw/rdma/rdma_utils.c
 delete mode 100644 hw/rdma/vmw/pvrdma_cmd.c
 delete mode 100644 hw/rdma/vmw/pvrdma_dev_ring.c
 delete mode 100644 hw/rdma/vmw/pvrdma_main.c
 delete mode 100644 hw/rdma/vmw/pvrdma_qp_ops.c
 delete mode 100644 migration/rdma.c
 delete mode 100644 contrib/rdmacm-mux/meson.build
 delete mode 100644 hw/rdma/Kconfig
 delete mode 100644 hw/rdma/meson.build
 delete mode 100644 hw/rdma/trace-events
 delete mode 100644 hw/rdma/vmw/trace-events
Signature · 1 line
2.41.0
#1 lore raw

PATCH v2 1/3 hw/rdma: Remove pvrdma device and rdmacm-mux helper

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- PVRDMA device
- generated vmw_pvrdma/ directory from linux-headers
- rdmacm-mux tool from contrib/

---
 MAINTAINERS                                   |   10 -
 docs/about/deprecated.rst                     |    9 -
 docs/about/removed-features.rst               |    4 +
 docs/pvrdma.txt                               |  345 ----
 docs/system/loongarch/virt.rst                |    2 +-
 meson.build                                   |   36 -
 qapi/machine.json                             |   17 -
 qapi/qapi-schema.json                         |    1 -
 qapi/rdma.json                                |   38 -
 contrib/rdmacm-mux/rdmacm-mux.h               |   61 -
 hw/rdma/rdma_backend.h                        |  129 --
 hw/rdma/rdma_backend_defs.h                   |   76 -
 hw/rdma/rdma_rm.h                             |   97 --
 hw/rdma/rdma_rm_defs.h                        |  146 --
 hw/rdma/rdma_utils.h                          |   63 -
 hw/rdma/trace.h                               |    1 -
 hw/rdma/vmw/pvrdma.h                          |  144 --
 hw/rdma/vmw/pvrdma_dev_ring.h                 |   46 -
 hw/rdma/vmw/pvrdma_qp_ops.h                   |   28 -
 hw/rdma/vmw/trace.h                           |    1 -
 include/hw/rdma/rdma.h                        |   37 -
 include/monitor/hmp.h                         |    1 -
 .../infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h |  685 --------
 .../infiniband/hw/vmw_pvrdma/pvrdma_verbs.h   |  348 ----
 .../standard-headers/rdma/vmw_pvrdma-abi.h    |  310 ----
 contrib/rdmacm-mux/main.c                     |  831 ----------
 hw/core/machine-qmp-cmds.c                    |   32 -
 hw/rdma/rdma.c                                |   30 -
 hw/rdma/rdma_backend.c                        | 1401 -----------------
 hw/rdma/rdma_rm.c                             |  812 ----------
 hw/rdma/rdma_utils.c                          |  126 --
 hw/rdma/vmw/pvrdma_cmd.c                      |  815 ----------
 hw/rdma/vmw/pvrdma_dev_ring.c                 |  141 --
 hw/rdma/vmw/pvrdma_main.c                     |  735 ---------
 hw/rdma/vmw/pvrdma_qp_ops.c                   |  298 ----
 monitor/qmp-cmds.c                            |    1 -
 Kconfig.host                                  |    3 -
 contrib/rdmacm-mux/meson.build                |    7 -
 hmp-commands-info.hx                          |   13 -
 hw/Kconfig                                    |    1 -
 hw/meson.build                                |    1 -
 hw/rdma/Kconfig                               |    3 -
 hw/rdma/meson.build                           |   12 -
 hw/rdma/trace-events                          |   31 -
 hw/rdma/vmw/trace-events                      |   17 -
 meson_options.txt                             |    2 -
 qapi/meson.build                              |    1 -
 qemu-options.hx                               |    3 -
 .../ci/org.centos/stream/8/x86_64/configure   |    1 -
 scripts/meson-buildoptions.sh                 |    3 -
 scripts/update-linux-headers.sh               |   27 -
 51 files changed, 5 insertions(+), 7977 deletions(-)
 delete mode 100644 docs/pvrdma.txt
 delete mode 100644 qapi/rdma.json
 delete mode 100644 contrib/rdmacm-mux/rdmacm-mux.h
 delete mode 100644 hw/rdma/rdma_backend.h
 delete mode 100644 hw/rdma/rdma_backend_defs.h
 delete mode 100644 hw/rdma/rdma_rm.h
 delete mode 100644 hw/rdma/rdma_rm_defs.h
 delete mode 100644 hw/rdma/rdma_utils.h
 delete mode 100644 hw/rdma/trace.h
 delete mode 100644 hw/rdma/vmw/pvrdma.h
 delete mode 100644 hw/rdma/vmw/pvrdma_dev_ring.h
 delete mode 100644 hw/rdma/vmw/pvrdma_qp_ops.h
 delete mode 100644 hw/rdma/vmw/trace.h
 delete mode 100644 include/hw/rdma/rdma.h
 delete mode 100644 include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
 delete mode 100644 include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
 delete mode 100644 include/standard-headers/rdma/vmw_pvrdma-abi.h
 delete mode 100644 contrib/rdmacm-mux/main.c
 delete mode 100644 hw/rdma/rdma.c
 delete mode 100644 hw/rdma/rdma_backend.c
 delete mode 100644 hw/rdma/rdma_rm.c
 delete mode 100644 hw/rdma/rdma_utils.c
 delete mode 100644 hw/rdma/vmw/pvrdma_cmd.c
 delete mode 100644 hw/rdma/vmw/pvrdma_dev_ring.c
 delete mode 100644 hw/rdma/vmw/pvrdma_main.c
 delete mode 100644 hw/rdma/vmw/pvrdma_qp_ops.c
 delete mode 100644 contrib/rdmacm-mux/meson.build
 delete mode 100644 hw/rdma/Kconfig
 delete mode 100644 hw/rdma/meson.build
 delete mode 100644 hw/rdma/trace-events
 delete mode 100644 hw/rdma/vmw/trace-events
MAINTAINERS +0 -10
index a07af6b9d4..91ab5235b8 100644
@@ -4060,16 +4060,6 @@ F: block/replication.c
 F: tests/unit/test-replication.c
 F: docs/block-replication.txt
 
-PVRDMA
-M: Yuval Shaia <[email protected]>
-M: Marcel Apfelbaum <[email protected]>
-S: Odd Fixes
-F: hw/rdma/*
-F: hw/rdma/vmw/*
-F: docs/pvrdma.txt
-F: contrib/rdmacm-mux/*
-F: qapi/rdma.json
-
 Semihosting
 M: Alex Bennée <[email protected]>
 S: Maintained
docs/about/deprecated.rst +0 -9
index 7b548519b5..29eae69e50 100644
@@ -376,15 +376,6 @@ recommending to switch to their stable counterparts:
 - "Zve64f" should be replaced with "zve64f"
 - "Zve64d" should be replaced with "zve64d"
 
-``-device pvrdma`` and the rdma subsystem (since 8.2)
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The pvrdma device and the whole rdma subsystem are in a bad shape and
-without active maintenance. The QEMU project intends to remove this
-device and subsystem from the code base in a future release without
-replacement unless somebody steps up and improves the situation.
-
-
 Block device options
 ''''''''''''''''''''
 
docs/about/removed-features.rst +4 -0
index f9cf874f7b..4d5bdc43b4 100644
@@ -909,6 +909,10 @@ contains native support for this feature and thus use of the option
 ROM approach was obsolete. The native SeaBIOS support can be activated
 by using ``-machine graphics=off``.
 
+``pvrdma`` and the RDMA subsystem (removed in 9.1)
+''''''''''''''''''''''''''''''''''''''''''''''''''
+
+The 'pvrdma' device and the whole RDMA subsystem have been removed.
 
 Related binaries
 ----------------
docs/pvrdma.txt deleted +0 -345

345 lines not on this page · raw message

docs/system/loongarch/virt.rst +1 -1
index c37268b404..06d034b8ef 100644
@@ -39,7 +39,7 @@ can be accessed by following steps.
 
 .. code-block:: bash
 
-  ./configure --disable-rdma --disable-pvrdma --prefix=/usr \
+  ./configure --disable-rdma --prefix=/usr \
               --target-list="loongarch64-softmmu" \
               --disable-libiscsi --disable-libnfs --disable-libpmem \
               --disable-glusterfs --enable-libusb --enable-usb-redir \
meson.build +0 -36
index c9c3217ba4..d6af3cd53a 100644
@@ -2829,37 +2829,6 @@ config_host_data.set('CONFIG_ARM_AES_BUILTIN', cc.compiles('''
     void foo(uint8x16_t *p) { *p = vaesmcq_u8(*p); }
   '''))
 
-have_pvrdma = get_option('pvrdma') \
-  .require(rdma.found(), error_message: 'PVRDMA requires OpenFabrics libraries') \
-  .require(cc.compiles(gnu_source_prefix + '''
-    #include <sys/mman.h>
-    int main(void)
-    {
-      char buf = 0;
-      void *addr = &buf;
-      addr = mremap(addr, 0, 1, MREMAP_MAYMOVE | MREMAP_FIXED);
-
-      return 0;
-    }'''), error_message: 'PVRDMA requires mremap').allowed()
-
-if have_pvrdma
-  config_host_data.set('LEGACY_RDMA_REG_MR', not cc.links('''
-    #include <infiniband/verbs.h>
-    int main(void)
-    {
-      struct ibv_mr *mr;
-      struct ibv_pd *pd = NULL;
-      size_t length = 10;
-      uint64_t iova = 0;
-      int access = 0;
-      void *addr = NULL;
-
-      mr = ibv_reg_mr_iova(pd, addr, length, iova, access);
-      ibv_dereg_mr(mr);
-      return 0;
-    }'''))
-endif
-
 if get_option('membarrier').disabled()
   have_membarrier = false
 elif host_os == 'windows'
@@ -2993,7 +2962,6 @@ host_kconfig = \
   (have_vhost_kernel ? ['CONFIG_VHOST_KERNEL=y'] : []) + \
   (have_virtfs ? ['CONFIG_VIRTFS=y'] : []) + \
   (host_os == 'linux' ? ['CONFIG_LINUX=y'] : []) + \
-  (have_pvrdma ? ['CONFIG_PVRDMA=y'] : []) + \
   (multiprocess_allowed ? ['CONFIG_MULTIPROCESS_ALLOWED=y'] : []) + \
   (vfio_user_server_allowed ? ['CONFIG_VFIO_USER_SERVER_ALLOWED=y'] : []) + \
   (hv_balloon ? ['CONFIG_HV_BALLOON_POSSIBLE=y'] : [])
@@ -3357,8 +3325,6 @@ if have_system
     'hw/pci',
     'hw/pci-host',
     'hw/ppc',
-    'hw/rdma',
-    'hw/rdma/vmw',
     'hw/rtc',
     'hw/s390x',
     'hw/scsi',
@@ -4028,7 +3994,6 @@ if have_tools
     }]
   endforeach
 
-  subdir('contrib/rdmacm-mux')
   subdir('contrib/elf2dmp')
 
   executable('qemu-edid', files('qemu-edid.c', 'hw/display/edid-generate.c'),
@@ -4434,7 +4399,6 @@ summary_info += {'Linux AIO support': libaio}
 summary_info += {'Linux io_uring support': linux_io_uring}
 summary_info += {'ATTR/XATTR support': libattr}
 summary_info += {'RDMA support':      rdma}
-summary_info += {'PVRDMA support':    have_pvrdma}
 summary_info += {'fdt support':       fdt_opt == 'disabled' ? false : fdt_opt}
 summary_info += {'libcap-ng support': libcap_ng}
 summary_info += {'bpf support':       libbpf}
qapi/machine.json +0 -17
index e8b60641f2..e9f0f0c49a 100644
@@ -1737,23 +1737,6 @@
   'returns': 'HumanReadableText',
   'features': [ 'unstable' ] }
 
-##
-# @x-query-rdma:
-#
-# Query RDMA state
-#
-# Features:
-#
-# @unstable: This command is meant for debugging.
-#
-# Returns: RDMA state
-#
-# Since: 6.2
-##
-{ 'command': 'x-query-rdma',
-  'returns': 'HumanReadableText',
-  'features': [ 'unstable' ] }
-
 ##
 # @x-query-roms:
 #
qapi/qapi-schema.json +0 -1
index 8304d45625..5e33da7228 100644
@@ -54,7 +54,6 @@
 { 'include': 'dump.json' }
 { 'include': 'net.json' }
 { 'include': 'ebpf.json' }
-{ 'include': 'rdma.json' }
 { 'include': 'rocker.json' }
 { 'include': 'tpm.json' }
 { 'include': 'ui.json' }
qapi/rdma.json deleted +0 -38
deleted file mode 100644
index 195c001850..0000000000
@@ -1,38 +0,0 @@
-# -*- Mode: Python -*-
-# vim: filetype=python
-#
-
-##
-# = RDMA device
-##
-
-##
-# @RDMA_GID_STATUS_CHANGED:
-#
-# Emitted when guest driver adds/deletes GID to/from device
-#
-# @netdev: RoCE Network Device name
-#
-# @gid-status: Add or delete indication
-#
-# @subnet-prefix: Subnet Prefix
-#
-# @interface-id: Interface ID
-#
-# Since: 4.0
-#
-# Example:
-#
-#     <- {"timestamp": {"seconds": 1541579657, "microseconds": 986760},
-#         "event": "RDMA_GID_STATUS_CHANGED",
-#         "data":
-#             {"netdev": "bridge0",
-#             "interface-id": 15880512517475447892,
-#             "gid-status": true,
-#             "subnet-prefix": 33022}}
-##
-{ 'event': 'RDMA_GID_STATUS_CHANGED',
-  'data': { 'netdev'        : 'str',
-            'gid-status'    : 'bool',
-            'subnet-prefix' : 'uint64',
-            'interface-id'  : 'uint64' } }
contrib/rdmacm-mux/rdmacm-mux.h deleted +0 -61

61 lines not on this page · raw message

hw/rdma/rdma_backend.h deleted +0 -129

129 lines not on this page · raw message

hw/rdma/rdma_backend_defs.h deleted +0 -76

76 lines not on this page · raw message

hw/rdma/rdma_rm.h deleted +0 -97

97 lines not on this page · raw message

hw/rdma/rdma_rm_defs.h deleted +0 -146

146 lines not on this page · raw message

hw/rdma/rdma_utils.h deleted +0 -63

63 lines not on this page · raw message

hw/rdma/trace.h deleted +0 -1
deleted file mode 100644
index b3fa8ebc51..0000000000
@@ -1 +0,0 @@
-#include "trace/trace-hw_rdma.h"
hw/rdma/vmw/pvrdma.h deleted +0 -144

144 lines not on this page · raw message

hw/rdma/vmw/pvrdma_dev_ring.h deleted +0 -46

46 lines not on this page · raw message

hw/rdma/vmw/pvrdma_qp_ops.h deleted +0 -28
deleted file mode 100644
index bf2b15c5ce..0000000000
@@ -1,28 +0,0 @@
-/*
- * QEMU VMWARE paravirtual RDMA QP Operations
- *
- * Copyright (C) 2018 Oracle
- * Copyright (C) 2018 Red Hat Inc
- *
- * Authors:
- *     Yuval Shaia <[email protected]>
- *     Marcel Apfelbaum <[email protected]>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#ifndef PVRDMA_QP_OPS_H
-#define PVRDMA_QP_OPS_H
-
-#include "pvrdma.h"
-
-int pvrdma_qp_ops_init(void);
-void pvrdma_qp_ops_fini(void);
-void pvrdma_qp_send(PVRDMADev *dev, uint32_t qp_handle);
-void pvrdma_qp_recv(PVRDMADev *dev, uint32_t qp_handle);
-void pvrdma_srq_recv(PVRDMADev *dev, uint32_t srq_handle);
-void pvrdma_cq_poll(RdmaDeviceResources *dev_res, uint32_t cq_handle);
-
-#endif
hw/rdma/vmw/trace.h deleted +0 -1
deleted file mode 100644
index 3ebc9fb7ad..0000000000
@@ -1 +0,0 @@
-#include "trace/trace-hw_rdma_vmw.h"
include/hw/rdma/rdma.h deleted +0 -37
deleted file mode 100644
index 80b2e531c4..0000000000
@@ -1,37 +0,0 @@
-/*
- * RDMA device interface
- *
- * Copyright (C) 2019 Oracle
- * Copyright (C) 2019 Red Hat Inc
- *
- * Authors:
- *     Yuval Shaia <[email protected]>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#ifndef RDMA_H
-#define RDMA_H
-
-#include "qom/object.h"
-
-#define INTERFACE_RDMA_PROVIDER "rdma"
-
-typedef struct RdmaProviderClass RdmaProviderClass;
-DECLARE_CLASS_CHECKERS(RdmaProviderClass, RDMA_PROVIDER,
-                       INTERFACE_RDMA_PROVIDER)
-#define RDMA_PROVIDER(obj) \
-    INTERFACE_CHECK(RdmaProvider, (obj), \
-                    INTERFACE_RDMA_PROVIDER)
-
-typedef struct RdmaProvider RdmaProvider;
-
-struct RdmaProviderClass {
-    InterfaceClass parent;
-
-    void (*format_statistics)(RdmaProvider *obj, GString *buf);
-};
-
-#endif
include/monitor/hmp.h +0 -1
index 13f9a2dedb..f4cf8f6717 100644
@@ -37,7 +37,6 @@ void hmp_info_spice(Monitor *mon, const QDict *qdict);
 void hmp_info_balloon(Monitor *mon, const QDict *qdict);
 void hmp_info_irq(Monitor *mon, const QDict *qdict);
 void hmp_info_pic(Monitor *mon, const QDict *qdict);
-void hmp_info_rdma(Monitor *mon, const QDict *qdict);
 void hmp_info_pci(Monitor *mon, const QDict *qdict);
 void hmp_info_tpm(Monitor *mon, const QDict *qdict);
 void hmp_info_iothreads(Monitor *mon, const QDict *qdict);
include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h deleted +0 -685

685 lines not on this page · raw message

include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h deleted +0 -348

348 lines not on this page · raw message

include/standard-headers/rdma/vmw_pvrdma-abi.h deleted +0 -310

310 lines not on this page · raw message

contrib/rdmacm-mux/main.c deleted +0 -831

293 lines not on this page · raw message

hw/core/machine-qmp-cmds.c +0 -32
index 4b72009cd3..c20829b9ae 100644

Lines of this file are not shown. View the raw message.

hw/rdma/rdma.c deleted +0 -30
deleted file mode 100644
index 7bec0d0d2c..0000000000

Lines of this file are not shown. View the raw message.

hw/rdma/rdma_backend.c deleted +0 -1401

1,401 lines not on this page · raw message

hw/rdma/rdma_rm.c deleted +0 -812

812 lines not on this page · raw message

hw/rdma/rdma_utils.c deleted +0 -126

126 lines not on this page · raw message

hw/rdma/vmw/pvrdma_cmd.c deleted +0 -815

815 lines not on this page · raw message

hw/rdma/vmw/pvrdma_dev_ring.c deleted +0 -141

141 lines not on this page · raw message

hw/rdma/vmw/pvrdma_main.c deleted +0 -735

735 lines not on this page · raw message

hw/rdma/vmw/pvrdma_qp_ops.c deleted +0 -298

298 lines not on this page · raw message

monitor/qmp-cmds.c +0 -1
index b0f948d337..f84a0dc523 100644

Lines of this file are not shown. View the raw message.

Kconfig.host +0 -3
index f496475f8e..f6a2a131e6 100644

Lines of this file are not shown. View the raw message.

contrib/rdmacm-mux/meson.build deleted +0 -7
deleted file mode 100644
index 36c9c89630..0000000000

Lines of this file are not shown. View the raw message.

hmp-commands-info.hx +0 -13
index ad1b1306e3..20a9835ea8 100644

Lines of this file are not shown. View the raw message.

hw/Kconfig +0 -1
index 2c00936c28..32f876deb0 100644

Lines of this file are not shown. View the raw message.

hw/meson.build +0 -1
index 463d702683..3049a6fab0 100644

Lines of this file are not shown. View the raw message.

hw/rdma/Kconfig deleted +0 -3
deleted file mode 100644
index 840320bdc0..0000000000

Lines of this file are not shown. View the raw message.

hw/rdma/meson.build deleted +0 -12
deleted file mode 100644
index 363c9b8c83..0000000000

Lines of this file are not shown. View the raw message.

hw/rdma/trace-events deleted +0 -31
deleted file mode 100644
index c23175120e..0000000000

Lines of this file are not shown. View the raw message.

hw/rdma/vmw/trace-events deleted +0 -17
deleted file mode 100644
index a6c77e1e10..0000000000

Lines of this file are not shown. View the raw message.

meson_options.txt +0 -2
index 0a99a059ec..b5c0bad9e7 100644

Lines of this file are not shown. View the raw message.

qapi/meson.build +0 -1
index 375d564277..c92af6e063 100644

Lines of this file are not shown. View the raw message.

qemu-options.hx +0 -3
index 7fd1713fa8..f7ef9b4e41 100644

Lines of this file are not shown. View the raw message.

scripts/ci/org.centos/stream/8/x86_64/configure +0 -1
index 76781f17f4..868db665f6 100755

Lines of this file are not shown. View the raw message.

scripts/meson-buildoptions.sh +0 -3
index 680fa3f581..5ace33f167 100644

Lines of this file are not shown. View the raw message.

scripts/update-linux-headers.sh +0 -27
index a0006eec6f..73c292bbac 100755

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.41.0

Part of the diff is not shown here. View the raw message.

Reviewed-by on v2 1/3
#2 lore raw

Re: [PATCH-for-9.1 v2 1/3] hw/rdma: Remove pvrdma device and rdmacm-mux helper

Quoted 9 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- PVRDMA device
- generated vmw_pvrdma/ directory from linux-headers
- rdmacm-mux tool from contrib/

Quoted 4 lines

---

#3 lore raw

PATCH v2 2/3 migration: Remove RDMA protocol handling

In reply to Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

---
 MAINTAINERS                                   |    7 -
 docs/devel/migration/main.rst                 |    6 -
 docs/rdma.txt                                 |  420 --
 docs/system/loongarch/virt.rst                |    2 +-
 meson.build                                   |   23 -
 qapi/migration.json                           |   31 +-
 migration/migration-stats.h                   |    6 +-
 migration/migration.h                         |    9 -
 migration/options.h                           |    2 -
 migration/rdma.h                              |   69 -
 migration/migration-stats.c                   |    5 +-
 migration/migration.c                         |   31 -
 migration/options.c                           |   16 -
 migration/qemu-file.c                         |    1 -
 migration/ram.c                               |   86 +-
 migration/rdma.c                              | 4184 -----------------
 migration/savevm.c                            |    2 +-
 meson_options.txt                             |    2 -
 migration/meson.build                         |    1 -
 migration/trace-events                        |   68 +-
 qemu-options.hx                               |    3 -
 .../org.centos/stream/8/build-environment.yml |    1 -
 .../ci/org.centos/stream/8/x86_64/configure   |    2 -
 scripts/ci/setup/build-environment.yml        |    4 -
 scripts/coverity-scan/run-coverity-scan       |    2 +-
 scripts/meson-buildoptions.sh                 |    3 -
 tests/lcitool/projects/qemu.yml               |    3 -
 tests/migration/guestperf/engine.py           |    4 +-
 28 files changed, 14 insertions(+), 4979 deletions(-)
 delete mode 100644 docs/rdma.txt
 delete mode 100644 migration/rdma.h
 delete mode 100644 migration/rdma.c
MAINTAINERS +0 -7
index 91ab5235b8..05226cea0a 100644
@@ -3426,13 +3426,6 @@ F: docs/devel/migration.rst
 F: qapi/migration.json
 F: tests/migration/
 F: util/userfaultfd.c
-X: migration/rdma*
-
-RDMA Migration
-R: Li Zhijian <[email protected]>
-R: Peter Xu <[email protected]>
-S: Odd Fixes
-F: migration/rdma*
 
 Migration dirty limit and dirty page rate
 M: Hyman Huang <[email protected]>
docs/devel/migration/main.rst +0 -6
index 54385a23e5..70278ce1e3 100644
@@ -47,12 +47,6 @@ over any transport.
   QEMU interference. Note that QEMU does not flush cached file
   data/metadata at the end of migration.
 
-In addition, support is included for migration using RDMA, which
-transports the page data using ``RDMA``, where the hardware takes care of
-transporting the pages, and the load on the CPU is much lower.  While the
-internals of RDMA migration are a bit different, this isn't really visible
-outside the RAM migration code.
-
 All these migration protocols use the same infrastructure to
 save/restore state devices.  This infrastructure is shared with the
 savevm/loadvm functionality.
docs/rdma.txt deleted +0 -420

420 lines not on this page · raw message

docs/system/loongarch/virt.rst +1 -1
index 06d034b8ef..0a8e0766e4 100644
@@ -39,7 +39,7 @@ can be accessed by following steps.
 
 .. code-block:: bash
 
-  ./configure --disable-rdma --prefix=/usr \
+  ./configure --prefix=/usr \
               --target-list="loongarch64-softmmu" \
               --disable-libiscsi --disable-libnfs --disable-libpmem \
               --disable-glusterfs --enable-libusb --enable-usb-redir \
meson.build +0 -23
index d6af3cd53a..bd65abad13 100644
@@ -1854,21 +1854,6 @@ if numa.found() and not cc.links('''
   endif
 endif
 
-rdma = not_found
-if not get_option('rdma').auto() or have_system
-  libumad = cc.find_library('ibumad', required: get_option('rdma'))
-  rdma_libs = [cc.find_library('rdmacm', has_headers: ['rdma/rdma_cma.h'],
-                               required: get_option('rdma')),
-               cc.find_library('ibverbs', required: get_option('rdma')),
-               libumad]
-  rdma = declare_dependency(dependencies: rdma_libs)
-  foreach lib: rdma_libs
-    if not lib.found()
-      rdma = not_found
-    endif
-  endforeach
-endif
-
 cacard = not_found
 if not get_option('smartcard').auto() or have_system
   cacard = dependency('libcacard', required: get_option('smartcard'),
@@ -2246,7 +2231,6 @@ endif
 config_host_data.set('CONFIG_OPENGL', opengl.found())
 config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
 config_host_data.set('CONFIG_RBD', rbd.found())
-config_host_data.set('CONFIG_RDMA', rdma.found())
 config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
 config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
 config_host_data.set('CONFIG_SDL', sdl.found())
@@ -2399,12 +2383,6 @@ if rbd.found()
                                        dependencies: rbd,
                                        prefix: '#include <rbd/librbd.h>'))
 endif
-if rdma.found()
-  config_host_data.set('HAVE_IBV_ADVISE_MR',
-                       cc.has_function('ibv_advise_mr',
-                                       dependencies: rdma,
-                                       prefix: '#include <infiniband/verbs.h>'))
-endif
 
 have_asan_fiber = false
 if get_option('sanitizers') and \
@@ -4398,7 +4376,6 @@ summary_info += {'Multipath support': mpathpersist}
 summary_info += {'Linux AIO support': libaio}
 summary_info += {'Linux io_uring support': linux_io_uring}
 summary_info += {'ATTR/XATTR support': libattr}
-summary_info += {'RDMA support':      rdma}
 summary_info += {'fdt support':       fdt_opt == 'disabled' ? false : fdt_opt}
 summary_info += {'libcap-ng support': libcap_ng}
 summary_info += {'bpf support':       libbpf}
qapi/migration.json +4 -27
index 8c65b90328..9a56d403be 100644
@@ -221,8 +221,8 @@
 #
 # @setup-time: amount of setup time in milliseconds *before* the
 #     iterations begin but *after* the QMP command is issued.  This is
-#     designed to provide an accounting of any activities (such as
-#     RDMA pinning) which may be expensive, but do not actually occur
+#     designed to provide an accounting of any activities which may be
+#     expensive, but do not actually occur
 #     during the iterative migration rounds themselves.  (since 1.6)
 #
 # @cpu-throttle-percentage: percentage of time guest cpus are being
@@ -430,10 +430,6 @@
 #     for certain work loads, by sending compressed difference of the
 #     pages
 #
-# @rdma-pin-all: Controls whether or not the entire VM memory
-#     footprint is mlock()'d on demand or all at once.  Refer to
-#     docs/rdma.txt for usage.  Disabled by default.  (since 2.0)
-#
 # @zero-blocks: During storage migration encode blocks of zeroes
 #     efficiently.  This essentially saves 1MB of zeroes per block on
 #     the wire.  Enabling requires source and target VM to support
@@ -547,7 +543,7 @@
 # Since: 1.2
 ##
 { 'enum': 'MigrationCapability',
-  'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks',
+  'data': ['xbzrle', 'auto-converge', 'zero-blocks',
            { 'name': 'compress', 'features': [ 'deprecated' ] },
            'events', 'postcopy-ram',
            { 'name': 'x-colo', 'features': [ 'unstable' ] },
@@ -606,7 +602,6 @@
 #     -> { "execute": "query-migrate-capabilities" }
 #     <- { "return": [
 #           {"state": false, "capability": "xbzrle"},
-#           {"state": false, "capability": "rdma-pin-all"},
 #           {"state": false, "capability": "auto-converge"},
 #           {"state": false, "capability": "zero-blocks"},
 #           {"state": false, "capability": "compress"},
@@ -1654,14 +1649,12 @@
 #
 # @exec: Direct the migration stream to another process.
 #
-# @rdma: Migrate via RDMA.
-#
 # @file: Direct the migration stream to a file.
 #
 # Since: 8.2
 ##
 { 'enum': 'MigrationAddressType',
-  'data': [ 'socket', 'exec', 'rdma', 'file' ] }
+  'data': [ 'socket', 'exec', 'file' ] }
 
 ##
 # @FileMigrationArgs:
@@ -1701,7 +1694,6 @@
   'data': {
     'socket': 'SocketAddress',
     'exec': 'MigrationExecCommand',
-    'rdma': 'InetSocketAddress',
     'file': 'FileMigrationArgs' } }
 
 ##
@@ -1804,14 +1796,6 @@
 #     -> { "execute": "migrate",
 #          "arguments": {
 #              "channels": [ { "channel-type": "main",
-#                              "addr": { "transport": "rdma",
-#                                        "host": "10.12.34.9",
-#                                        "port": "1050" } } ] } }
-#     <- { "return": {} }
-#
-#     -> { "execute": "migrate",
-#          "arguments": {
-#              "channels": [ { "channel-type": "main",
 #                              "addr": { "transport": "file",
 #                                        "filename": "/tmp/migfile",
 #                                        "offset": "0x1000" } } ] } }
@@ -1879,13 +1863,6 @@
 #                                                  "/some/sock" ] } } ] } }
 #     <- { "return": {} }
 #
-#     -> { "execute": "migrate-incoming",
-#          "arguments": {
-#              "channels": [ { "channel-type": "main",
-#                              "addr": { "transport": "rdma",
-#                                        "host": "10.12.34.9",
-#                                        "port": "1050" } } ] } }
-#     <- { "return": {} }
 ##
 { 'command': 'migrate-incoming',
              'data': {'*uri': 'str',
migration/migration-stats.h +1 -5
index 05290ade76..817c53559a 100644
@@ -93,10 +93,6 @@ typedef struct {
      * Maximum amount of data we can send in a cycle.
      */
     Stat64 rate_limit_max;
-    /*
-     * Number of bytes sent through RDMA.
-     */
-    Stat64 rdma_bytes;
     /*
      * Number of pages transferred that were full of zeros.
      */
@@ -133,7 +129,7 @@ void migration_rate_set(uint64_t new_rate);
  *
  * Returns how many bytes have we transferred since the beginning of
  * the migration.  It accounts for bytes sent through any migration
- * channel, multifd, qemu_file, rdma, ....
+ * channel, multifd, qemu_file, ....
  */
 uint64_t migration_transferred_bytes(void);
 #endif
migration/migration.h +0 -9
index 8045e39c26..d097828580 100644
@@ -162,13 +162,6 @@ struct MigrationIncomingState {
 
     int state;
 
-    /*
-     * The incoming migration coroutine, non-NULL during qemu_loadvm_state().
-     * Used to wake the migration incoming coroutine from rdma code. How much is
-     * it safe - it's a question.
-     */
-    Coroutine *loadvm_co;
-
     /* The coroutine we should enter (back) after failover */
     Coroutine *colo_incoming_co;
     QemuSemaphore colo_incoming_sem;
@@ -463,8 +456,6 @@ struct MigrationState {
      * switchover has been received.
      */
     bool switchover_acked;
-    /* Is this a rdma migration */
-    bool rdma_migration;
 };
 
 void migrate_set_state(int *state, int old_state, int new_state);
migration/options.h +0 -2
index ab8199e207..c00213973e 100644
@@ -37,7 +37,6 @@ bool migrate_multifd(void);
 bool migrate_pause_before_switchover(void);
 bool migrate_postcopy_blocktime(void);
 bool migrate_postcopy_preempt(void);
-bool migrate_rdma_pin_all(void);
 bool migrate_release_ram(void);
 bool migrate_return_path(void);
 bool migrate_validate_uuid(void);
@@ -54,7 +53,6 @@ bool migrate_zero_copy_send(void);
 
 bool migrate_multifd_flush_after_each_section(void);
 bool migrate_postcopy(void);
-bool migrate_rdma(void);
 bool migrate_tls(void);
 
 /* capabilities helpers */
migration/rdma.h deleted +0 -69

69 lines not on this page · raw message

migration/migration-stats.c +2 -3
index f690b98a03..9bc8d7018f 100644
@@ -62,9 +62,8 @@ void migration_rate_reset(void)
 uint64_t migration_transferred_bytes(void)
 {
     uint64_t multifd = stat64_get(&mig_stats.multifd_bytes);
-    uint64_t rdma = stat64_get(&mig_stats.rdma_bytes);
     uint64_t qemu_file = stat64_get(&mig_stats.qemu_file_transferred);
 
-    trace_migration_transferred_bytes(qemu_file, multifd, rdma);
-    return qemu_file + multifd + rdma;
+    trace_migration_transferred_bytes(qemu_file, multifd);
+    return qemu_file + multifd;
 }
migration/migration.c +0 -31
index 9fe8fd2afd..8e17914c8b 100644
@@ -25,7 +25,6 @@
 #include "sysemu/runstate.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/cpu-throttle.h"
-#include "rdma.h"
 #include "ram.h"
 #include "ram-compress.h"
 #include "migration/global_state.h"
@@ -545,7 +544,6 @@ bool migrate_uri_parse(const char *uri, MigrationChannel **channel,
 {
     g_autoptr(MigrationChannel) val = g_new0(MigrationChannel, 1);
     g_autoptr(MigrationAddress) addr = g_new0(MigrationAddress, 1);
-    InetSocketAddress *isock = &addr->u.rdma;
     strList **tail = &addr->u.exec.args;
 
     if (strstart(uri, "exec:", NULL)) {
@@ -558,12 +556,6 @@ bool migrate_uri_parse(const char *uri, MigrationChannel **channel,
         QAPI_LIST_APPEND(tail, g_strdup("-c"));
 #endif
         QAPI_LIST_APPEND(tail, g_strdup(uri + strlen("exec:")));
-    } else if (strstart(uri, "rdma:", NULL)) {
-        if (inet_parse(isock, uri + strlen("rdma:"), errp)) {
-            qapi_free_InetSocketAddress(isock);
-            return false;
-        }
-        addr->transport = MIGRATION_ADDRESS_TYPE_RDMA;
     } else if (strstart(uri, "tcp:", NULL) ||
                 strstart(uri, "unix:", NULL) ||
                 strstart(uri, "vsock:", NULL) ||
@@ -645,22 +637,6 @@ static void qemu_start_incoming_migration(const char *uri, bool has_channels,
         } else if (saddr->type == SOCKET_ADDRESS_TYPE_FD) {
             fd_start_incoming_migration(saddr->u.fd.str, errp);
         }
-#ifdef CONFIG_RDMA
-    } else if (addr->transport == MIGRATION_ADDRESS_TYPE_RDMA) {
-        if (migrate_compress()) {
-            error_setg(errp, "RDMA and compression can't be used together");
-            return;
-        }
-        if (migrate_xbzrle()) {
-            error_setg(errp, "RDMA and XBZRLE can't be used together");
-            return;
-        }
-        if (migrate_multifd()) {
-            error_setg(errp, "RDMA and multifd can't be used together");
-            return;
-        }
-        rdma_start_incoming_migration(&addr->u.rdma, errp);
-#endif
     } else if (addr->transport == MIGRATION_ADDRESS_TYPE_EXEC) {
         exec_start_incoming_migration(addr->u.exec.args, errp);
     } else if (addr->transport == MIGRATION_ADDRESS_TYPE_FILE) {
@@ -751,9 +727,7 @@ process_incoming_migration_co(void *opaque)
     migrate_set_state(&mis->state, MIGRATION_STATUS_SETUP,
                       MIGRATION_STATUS_ACTIVE);
 
-    mis->loadvm_co = qemu_coroutine_self();
     ret = qemu_loadvm_state(mis->from_src_file);
-    mis->loadvm_co = NULL;
 
     trace_vmstate_downtime_checkpoint("dst-precopy-loadvm-completed");
 
@@ -1679,7 +1653,6 @@ int migrate_init(MigrationState *s, Error **errp)
     s->iteration_initial_bytes = 0;
     s->threshold_size = 0;
     s->switchover_acked = false;
-    s->rdma_migration = false;
     /*
      * set mig_stats memory to zero for a new migration
      */
@@ -2100,10 +2073,6 @@ void qmp_migrate(const char *uri, bool has_channels,
         } else if (saddr->type == SOCKET_ADDRESS_TYPE_FD) {
             fd_start_outgoing_migration(s, saddr->u.fd.str, &local_err);
         }
-#ifdef CONFIG_RDMA
-    } else if (addr->transport == MIGRATION_ADDRESS_TYPE_RDMA) {
-        rdma_start_outgoing_migration(s, &addr->u.rdma, &local_err);
-#endif
     } else if (addr->transport == MIGRATION_ADDRESS_TYPE_EXEC) {
         exec_start_outgoing_migration(s, addr->u.exec.args, &local_err);
     } else if (addr->transport == MIGRATION_ADDRESS_TYPE_FILE) {
migration/options.c +0 -16
index bfd7753b69..02fc0b9ae8 100644
@@ -185,7 +185,6 @@ Property migration_properties[] = {
 
     /* Migration capabilities */
     DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE),
-    DEFINE_PROP_MIG_CAP("x-rdma-pin-all", MIGRATION_CAPABILITY_RDMA_PIN_ALL),
     DEFINE_PROP_MIG_CAP("x-auto-converge", MIGRATION_CAPABILITY_AUTO_CONVERGE),
     DEFINE_PROP_MIG_CAP("x-zero-blocks", MIGRATION_CAPABILITY_ZERO_BLOCKS),
     DEFINE_PROP_MIG_CAP("x-compress", MIGRATION_CAPABILITY_COMPRESS),
@@ -323,13 +322,6 @@ bool migrate_postcopy_ram(void)
     return s->capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
 }
 
-bool migrate_rdma_pin_all(void)
-{
-    MigrationState *s = migrate_get_current();
-
-    return s->capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL];
-}
-
 bool migrate_release_ram(void)
 {
     MigrationState *s = migrate_get_current();
@@ -393,13 +385,6 @@ bool migrate_postcopy(void)
     return migrate_postcopy_ram() || migrate_dirty_bitmaps();
 }
 
-bool migrate_rdma(void)
-{
-    MigrationState *s = migrate_get_current();
-
-    return s->rdma_migration;
-}
-
 bool migrate_tls(void)
 {
     MigrationState *s = migrate_get_current();
@@ -458,7 +443,6 @@ INITIALIZE_MIGRATE_CAPS_SET(check_caps_background_snapshot,
     MIGRATION_CAPABILITY_PAUSE_BEFORE_SWITCHOVER,
     MIGRATION_CAPABILITY_AUTO_CONVERGE,
     MIGRATION_CAPABILITY_RELEASE_RAM,
-    MIGRATION_CAPABILITY_RDMA_PIN_ALL,
     MIGRATION_CAPABILITY_COMPRESS,
     MIGRATION_CAPABILITY_XBZRLE,
     MIGRATION_CAPABILITY_X_COLO,
migration/qemu-file.c +0 -1
index a10882d47f..ad2efb332e 100644
@@ -32,7 +32,6 @@
 #include "trace.h"
 #include "options.h"
 #include "qapi/error.h"
-#include "rdma.h"
 #include "io/channel-file.h"
 
 #define IO_BUF_SIZE 32768
migration/ram.c +2 -84

158 lines not on this page · raw message

migration/rdma.c deleted +0 -4184

2,004 lines not on this page · raw message

migration/savevm.c +1 -1
index 388d7af7cd..939d35d69e 100644

Lines of this file are not shown. View the raw message.

meson_options.txt +0 -2
index b5c0bad9e7..79b69d4286 100644

Lines of this file are not shown. View the raw message.

migration/meson.build +0 -1
index 1eeb915ff6..e2cd92c01f 100644

Lines of this file are not shown. View the raw message.

migration/trace-events +1 -67

68 lines not on this page · raw message

qemu-options.hx +0 -3
index f7ef9b4e41..4f390c33ef 100644

Lines of this file are not shown. View the raw message.

scripts/ci/org.centos/stream/8/build-environment.yml +0 -1
index 1ead77e2cb..a366bb185b 100644

Lines of this file are not shown. View the raw message.

scripts/ci/org.centos/stream/8/x86_64/configure +0 -2
index 868db665f6..5dead834fb 100755

Lines of this file are not shown. View the raw message.

scripts/ci/setup/build-environment.yml +0 -4
index f344d1a850..0359b1c023 100644

Lines of this file are not shown. View the raw message.

scripts/coverity-scan/run-coverity-scan +1 -1
index 43cf770f5e..3dd14c3cc4 100755

Lines of this file are not shown. View the raw message.

scripts/meson-buildoptions.sh +0 -3
index 5ace33f167..52c34598ba 100644

Lines of this file are not shown. View the raw message.

tests/lcitool/projects/qemu.yml +0 -3
index 149b15de57..511e48a5ec 100644

Lines of this file are not shown. View the raw message.

tests/migration/guestperf/engine.py +1 -3
index 608d7270f6..a704419082 100644

Lines of this file are not shown. View the raw message.

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.41.0

Part of the diff is not shown here. View the raw message.

#4 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 8 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Quoted 7 lines

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

Just to be clear, because people raised the point in the last version, the first link in the deprecation commit links to a thread comprising entirely of rdma migration patches. I don't see any ambiguity on whether the deprecation was intended to include migration. There's even an ack from Juan.

So on the basis of not reverting the previous maintainer's decision, my Ack stands here.

We also had pretty obvious bugs ([1], [2]) in the past that would have been caught if we had any kind of testing for the feature, so I can't even say this thing works currently.

@Peter Xu, @Li Zhijian, what are your thoughts on this?

1- https://lore.kernel.org/r/[email protected]
2- https://lore.kernel.org/r/CAHEcVy7HXSwn4Ow_Kog+Q+TN6f_kMeiCHevz1qGM-fbxBPp1hQ@mail.gmail.com

#5 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 18 lines · Fabiano Rosas
Quoted 8 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Quoted 7 lines

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

Quoted 5 lines

Just to be clear, because people raised the point in the last version, the first link in the deprecation commit links to a thread comprising entirely of rdma migration patches. I don't see any ambiguity on whether the deprecation was intended to include migration. There's even an ack from Juan.

Yes I remember that's the plan.

Quoted 9 lines

So on the basis of not reverting the previous maintainer's decision, my Ack stands here.

We also had pretty obvious bugs ([1], [2]) in the past that would have been caught if we had any kind of testing for the feature, so I can't even say this thing works currently.

@Peter Xu, @Li Zhijian, what are your thoughts on this?

Generally I definitely agree with such a removal sooner or later, as that's how deprecation works, and even after Juan's left I'm not aware of any other new RDMA users. Personally, I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance, however I assume that's not a blocker either, as I think we can also manage it.

IMHO it's more important to know whether there are still users and whether they would still like to see it around. That's also one thing I notice that e9a54265f533f didn't yet get acks from RDMA users that we are aware, even if they're rare. According to [2] it could be that such user may only rely on the release versions of QEMU when it broke things.

So I'm copying Yu too (while Zhijian is already in the loop), just in case someone would like to stand up and speak.

Thanks,

Quoted 4 lines

1- https://lore.kernel.org/r/[email protected]
2- https://lore.kernel.org/r/CAHEcVy7HXSwn4Ow_Kog+Q+TN6f_kMeiCHevz1qGM-fbxBPp1hQ@mail.gmail.com

Signature · 1 line
Peter Xu
#6 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 37 lines · Peter Xu
Quoted 18 lines · Fabiano Rosas
Quoted 8 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Quoted 7 lines

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

Quoted 5 lines

Just to be clear, because people raised the point in the last version, the first link in the deprecation commit links to a thread comprising entirely of rdma migration patches. I don't see any ambiguity on whether the deprecation was intended to include migration. There's even an ack from Juan.

Yes I remember that's the plan.

Quoted 9 lines

So on the basis of not reverting the previous maintainer's decision, my Ack stands here.

We also had pretty obvious bugs ([1], [2]) in the past that would have been caught if we had any kind of testing for the feature, so I can't even say this thing works currently.

@Peter Xu, @Li Zhijian, what are your thoughts on this?

Quoted 8 lines

Generally I definitely agree with such a removal sooner or later, as that's how deprecation works, and even after Juan's left I'm not aware of any other new RDMA users. Personally, I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance, however I assume that's not a blocker either, as I think we can also manage it.

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

Since e9a54265f5 was not very clear about rdma migration code, should we maybe rather add a separate deprecation note for the migration part, and add a proper warning message to the migration code in case someone tries to use it there, and then only remove the rdma migration code after two more releases?

Thomas

#7 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 5 lines · Thomas Huth

Since e9a54265f5 was not very clear about rdma migration code, should we maybe rather add a separate deprecation note for the migration part, and add a proper warning message to the migration code in case someone tries to use it there, and then only remove the rdma migration code after two more releases?

Definitely a valid option to me.

So far RDMA isn't covered in tests (actually same to COLO, and I wonder our position of COLO too in this case..), so unfortunately we don't even know when it'll break just like before.

From other activities that I can see when new code comes, maintaining RDMA code should be fairly manageable so far (and whoever will write new rdma codes in those two releases will also need to take the maintainer's role). We did it for those years, and we can keep that for two more releases. Hopefully that can ring a louder alarm to the current users with such warnings, so that people can either stick with old binaries, or invest developer/test resources to the community.

Thanks,

Signature · 1 line
Peter Xu
#8 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu
Quoted 43 lines · Peter Xu
Quoted 18 lines · Fabiano Rosas
Quoted 8 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Quoted 7 lines

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

Quoted 5 lines

Just to be clear, because people raised the point in the last version, the first link in the deprecation commit links to a thread comprising entirely of rdma migration patches. I don't see any ambiguity on whether the deprecation was intended to include migration. There's even an ack from Juan.

Yes I remember that's the plan.

Quoted 9 lines

So on the basis of not reverting the previous maintainer's decision, my Ack stands here.

We also had pretty obvious bugs ([1], [2]) in the past that would have been caught if we had any kind of testing for the feature, so I can't even say this thing works currently.

@Peter Xu, @Li Zhijian, what are your thoughts on this?

Generally I definitely agree with such a removal sooner or later, as that's how deprecation works, and even after Juan's left I'm not aware of any other new RDMA users. Personally, I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance, however I assume that's not a blocker either, as I think we can also manage it.

Quoted 8 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around. That's also one thing I notice that e9a54265f533f didn't yet get acks from RDMA users that we are aware, even if they're rare. According to [2] it could be that such user may only rely on the release versions of QEMU when it broke things.

So I'm copying Yu too (while Zhijian is already in the loop), just in case someone would like to stand up and speak.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.
However I was a bit surprised when I saw the removal of the RDMA migration. I wasn't aware that this feature has not been marked as deprecated(at least there is no prompt to end-user).

Quoted 2 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

Agree.
I didn't immediately express my opinion in V1 because I'm also consulting our customers for this feature in the future.

Personally, I agree with Perter's idea that "I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance"

Thanks
Zhijian

Quoted 8 lines

Thanks,

Quoted 4 lines

1- https://lore.kernel.org/r/[email protected]
2- https://lore.kernel.org/r/CAHEcVy7HXSwn4Ow_Kog+Q+TN6f_kMeiCHevz1qGM-fbxBPp1hQ@mail.gmail.com

#9 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Zhijian,

Quoted 66 lines · Zhijian Li (Fujitsu)
Quoted 43 lines · Peter Xu
Quoted 18 lines · Fabiano Rosas
Quoted 8 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Quoted 7 lines

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

Quoted 5 lines

Just to be clear, because people raised the point in the last version, the first link in the deprecation commit links to a thread comprising entirely of rdma migration patches. I don't see any ambiguity on whether the deprecation was intended to include migration. There's even an ack from Juan.

Yes I remember that's the plan.

Quoted 9 lines

So on the basis of not reverting the previous maintainer's decision, my Ack stands here.

We also had pretty obvious bugs ([1], [2]) in the past that would have been caught if we had any kind of testing for the feature, so I can't even say this thing works currently.

@Peter Xu, @Li Zhijian, what are your thoughts on this?

Generally I definitely agree with such a removal sooner or later, as that's how deprecation works, and even after Juan's left I'm not aware of any other new RDMA users. Personally, I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance, however I assume that's not a blocker either, as I think we can also manage it.

Quoted 8 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around. That's also one thing I notice that e9a54265f533f didn't yet get acks from RDMA users that we are aware, even if they're rare. According to [2] it could be that such user may only rely on the release versions of QEMU when it broke things.

So I'm copying Yu too (while Zhijian is already in the loop), just in case someone would like to stand up and speak.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.
However I was a bit surprised when I saw the removal of the RDMA migration. I wasn't aware that this feature has not been marked as deprecated(at least there is no prompt to end-user).

Quoted 2 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

Quoted 6 lines

Agree.
I didn't immediately express my opinion in V1 because I'm also consulting our customers for this feature in the future.

Personally, I agree with Perter's idea that "I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance"

Do you mind posting a deprecation patch to clarify the situation?

Thanks,

Phil.

Quoted 11 lines

Thanks
Zhijian

Quoted 7 lines

Thanks,

Quoted 4 lines

1- https://lore.kernel.org/r/[email protected]
2- https://lore.kernel.org/r/CAHEcVy7HXSwn4Ow_Kog+Q+TN6f_kMeiCHevz1qGM-fbxBPp1hQ@mail.gmail.com

#10 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 69 lines · Philippe Mathieu-Daudé

Hi Zhijian,

Quoted 66 lines · Zhijian Li (Fujitsu)
Quoted 43 lines · Peter Xu
Quoted 18 lines · Fabiano Rosas
Quoted 8 lines · Philippe Mathieu-Daudé

The whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Remove:
- RDMA handling from migration
- dependencies on libibumad, libibverbs and librdmacm

Quoted 7 lines

Keep the RAM_SAVE_FLAG_HOOK definition since it might appears in old migration streams.

Quoted 5 lines

Just to be clear, because people raised the point in the last version, the first link in the deprecation commit links to a thread comprising entirely of rdma migration patches. I don't see any ambiguity on whether the deprecation was intended to include migration. There's even an ack from Juan.

Yes I remember that's the plan.

Quoted 9 lines

So on the basis of not reverting the previous maintainer's decision, my Ack stands here.

We also had pretty obvious bugs ([1], [2]) in the past that would have been caught if we had any kind of testing for the feature, so I can't even say this thing works currently.

@Peter Xu, @Li Zhijian, what are your thoughts on this?

Generally I definitely agree with such a removal sooner or later, as that's how deprecation works, and even after Juan's left I'm not aware of any other new RDMA users. Personally, I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance, however I assume that's not a blocker either, as I think we can also manage it.

Quoted 8 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around. That's also one thing I notice that e9a54265f533f didn't yet get acks from RDMA users that we are aware, even if they're rare. According to [2] it could be that such user may only rely on the release versions of QEMU when it broke things.

So I'm copying Yu too (while Zhijian is already in the loop), just in case someone would like to stand up and speak.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.
However I was a bit surprised when I saw the removal of the RDMA migration. I wasn't aware that this feature has not been marked as deprecated(at least there is no prompt to end-user).

Quoted 2 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

Quoted 8 lines
Quoted 6 lines

Agree.
I didn't immediately express my opinion in V1 because I'm also consulting our customers for this feature in the future.

Personally, I agree with Perter's idea that "I'd slightly prefer postponing it one more release which might help a bit of our downstream maintenance"

Do you mind posting a deprecation patch to clarify the situation?

The key thing the first deprecation patch missed was that it failed to issue a warning message when RDMA migration was actually used.

With regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#11 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Philippe Mathieu-Daudé

Phil,

Quoted 11 lines · Philippe Mathieu-Daudé
Quoted 11 lines
Quoted 5 lines

IMHO it's more important to know whether there are still users and whether
they would still like to see it around.

Agree.
I didn't immediately express my opinion in V1 because I'm also consulting our
customers for this feature in the future.

Quoted 6 lines
Quoted 3 lines

Personally, I agree with Perter's idea that "I'd slightly prefer postponing it one
more release which might help a bit of our downstream maintenance"

Do you mind posting a deprecation patch to clarify the situation?

No problem, i just posted a deprecation patch, please take a look.
https://lore.kernel.org/qemu-devel/[email protected]/T/#u

Thanks
Zhijian

#12 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

Quoted 5 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

The maintenance of this part could be too costly or difficult from your point of view.

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

Best regards,
Yu Zhang

Quoted 27 lines · Zhijian Li (Fujitsu)

Phil,

Quoted 11 lines · Philippe Mathieu-Daudé
Quoted 11 lines
Quoted 5 lines

IMHO it's more important to know whether there are still users and whether
they would still like to see it around.

Agree.
I didn't immediately express my opinion in V1 because I'm also consulting our
customers for this feature in the future.

Quoted 6 lines
Quoted 3 lines

Personally, I agree with Perter's idea that "I'd slightly prefer postponing it one
more release which might help a bit of our downstream maintenance"

Do you mind posting a deprecation patch to clarify the situation?

No problem, i just posted a deprecation patch, please take a look.
https://lore.kernel.org/qemu-devel/[email protected]/T/#u

Thanks
Zhijian

#13 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 4 lines · Yu Zhang

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

Quoted 14 lines
Quoted 6 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

Quoted 2 lines

The maintenance of this part could be too costly or difficult from your point of view.

It may or may not be too costly, it's just that we need real users of RDMA taking some care of it. Having it broken easily for >1 releases definitely is a sign of lack of users. It is an implication to the community that we should consider dropping some features so that we can get the best use of the community resources for the things that may have a broader audience.

One thing majorly missing is a RDMA tester to guard all the merges to not break RDMA paths, hopefully in CI. That should not rely on RDMA hardwares but just to sanity check the migration+rdma code running all fine. RDMA taught us the lesson so we're requesting CI coverage for all other new features that will be merged at least for migration subsystem, so that we plan to not merge anything that is not covered by CI unless extremely necessary in the future.

For sure CI is not the only missing part, but I'd say we should start with it, then someone should also take care of the code even if only in maintenance mode (no new feature to add on top).

Quoted 6 lines

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

RDMA doesn't get new features anyway, if there's specific use case for RDMA migrations, would it work if such scenario uses the old binary? Is it possible to switch to the TCP protocol with some good NICs?

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

Thanks,

Signature · 1 line
Peter Xu
#14 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter,

Quoted 53 lines · Peter Xu
Quoted 4 lines · Yu Zhang

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

Quoted 14 lines
Quoted 6 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

Quoted 2 lines

The maintenance of this part could be too costly or difficult from your point of view.

It may or may not be too costly, it's just that we need real users of RDMA taking some care of it. Having it broken easily for >1 releases definitely is a sign of lack of users. It is an implication to the community that we should consider dropping some features so that we can get the best use of the community resources for the things that may have a broader audience.

One thing majorly missing is a RDMA tester to guard all the merges to not break RDMA paths, hopefully in CI. That should not rely on RDMA hardwares but just to sanity check the migration+rdma code running all fine. RDMA taught us the lesson so we're requesting CI coverage for all other new features that will be merged at least for migration subsystem, so that we plan to not merge anything that is not covered by CI unless extremely necessary in the future.

For sure CI is not the only missing part, but I'd say we should start with it, then someone should also take care of the code even if only in maintenance mode (no new feature to add on top).

Quoted 6 lines

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

Quoted 3 lines

RDMA doesn't get new features anyway, if there's specific use case for RDMA migrations, would it work if such a scenario uses the old binary? Is it possible to switch to the TCP protocol with some good NICs?

We have used rdma migration with HCA from Nvidia for years, our experience is RDMA migration works better than tcp (over ipoib).

Switching back to TCP will lead us to the old problems which was solved by RDMA migration.

Quoted 9 lines

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

+cc some guys from Huawei.

I'm surprised RDMA users are rare, I guess maybe many are just working with different code base.

Quoted 5 lines

Thanks,

--
Peter Xu

Thx!
Jinpu Wang

Quoted 1 line
#15 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Jinpu Wang

Hi Peter,

Jinpu,

Thanks for joining the discussion.

Quoted 55 lines · Peter Xu
Quoted 53 lines · Yu Zhang
Quoted 4 lines

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

Quoted 14 lines
Quoted 6 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

Quoted 2 lines

The maintenance of this part could be too costly or difficult from your point of view.

It may or may not be too costly, it's just that we need real users of RDMA taking some care of it. Having it broken easily for >1 releases definitely is a sign of lack of users. It is an implication to the community that we should consider dropping some features so that we can get the best use of the community resources for the things that may have a broader audience.

One thing majorly missing is a RDMA tester to guard all the merges to not break RDMA paths, hopefully in CI. That should not rely on RDMA hardwares but just to sanity check the migration+rdma code running all fine. RDMA taught us the lesson so we're requesting CI coverage for all other new features that will be merged at least for migration subsystem, so that we plan to not merge anything that is not covered by CI unless extremely necessary in the future.

For sure CI is not the only missing part, but I'd say we should start with it, then someone should also take care of the code even if only in maintenance mode (no new feature to add on top).

Quoted 6 lines

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

Quoted 5 lines
Quoted 3 lines

RDMA doesn't get new features anyway, if there's specific use case for RDMA migrations, would it work if such a scenario uses the old binary? Is it possible to switch to the TCP protocol with some good NICs?

We have used rdma migration with HCA from Nvidia for years, our experience is RDMA migration works better than tcp (over ipoib).

Please bare with me, as I know little on rdma stuff.

I'm actually pretty confused (and since a long time ago..) on why we need to operation with rdma contexts when ipoib seems to provide all the tcp layers. I meant, can it work with the current "tcp:" protocol with ipoib even if there's rdma/ib hardwares underneath? Is it because of performance improvements so that we must use a separate path comparing to generic "tcp:" protocol here?

Quoted 3 lines

Switching back to TCP will lead us to the old problems which was solved by RDMA migration.

Can you elaborate the problems, and why tcp won't work in this case? They may not be directly relevant to the issue we're discussing, but I'm happy to learn more.

What is the NICs you were testing before? Did the test carry out with things like modern ones (50Gbps-200Gbps NICs), or the test was done when these hardwares are not common?

Per my recent knowledge on the new Intel hardwares, at least the ones that support QPL, it's easy to achieve single core 50Gbps+.

https://lore.kernel.org/r/PH7PR11MB5941A91AC1E514BCC32896A6A3342@PH7PR11MB5941.namprd11.prod.outlook.com

Quote from Yuan:

Yes, I use iperf3 to check the bandwidth for one core, the bandwith is 60Gbps.

  [ ID] Interval           Transfer     Bitrate         Retr  Cwnd
  [  5]   0.00-1.00   sec  7.00 GBytes  60.1 Gbits/sec    0   2.87 MBytes
  [  5]   1.00-2.00   sec  7.05 GBytes  60.6 Gbits/sec    0   2.87 Mbytes

And in the live migration test, a multifd thread's CPU utilization is almost 100%

It boils down to what old problems were there with tcp first, though.

Quoted 10 lines
Quoted 10 lines

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

Quoted 4 lines

+cc some guys from Huawei.

I'm surprised RDMA users are rare, I guess maybe many are just working with different code base.

Yes, please cc whoever might be interested (or surprised.. :) to know this, and let's be open to all possibilities.

I don't think it makes sense if there're a lot of users of a feature then we deprecate that without a good reason. However there's always the resource limitation issue we're facing, so it could still have the possibility that this gets deprecated if nobody is working on our upstream branch. Say, if people use private branches anyway to support rdma without collaborating upstream, keeping such feature upstream then may not make much sense either, unless there's some way to collaborate. We'll see.

It seems there can still be people joining this discussion. I'll hold off a bit on merging this patch to provide enough window for anyone to chim in.

Thanks,

Quoted 10 lines
Quoted 5 lines

Thanks,

--
Peter Xu

Thx!
Jinpu Wang

Quoted 1 line
Signature · 1 line
Peter Xu
#16 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter,

Quoted 69 lines · Peter Xu
Quoted 1 line · Jinpu Wang

Hi Peter,

Jinpu,

Thanks for joining the discussion.

Quoted 55 lines · Peter Xu
Quoted 53 lines · Yu Zhang
Quoted 4 lines

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

Quoted 14 lines
Quoted 6 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

Quoted 2 lines

The maintenance of this part could be too costly or difficult from your point of view.

It may or may not be too costly, it's just that we need real users of RDMA taking some care of it. Having it broken easily for >1 releases definitely is a sign of lack of users. It is an implication to the community that we should consider dropping some features so that we can get the best use of the community resources for the things that may have a broader audience.

One thing majorly missing is a RDMA tester to guard all the merges to not break RDMA paths, hopefully in CI. That should not rely on RDMA hardwares but just to sanity check the migration+rdma code running all fine. RDMA taught us the lesson so we're requesting CI coverage for all other new features that will be merged at least for migration subsystem, so that we plan to not merge anything that is not covered by CI unless extremely necessary in the future.

For sure CI is not the only missing part, but I'd say we should start with it, then someone should also take care of the code even if only in maintenance mode (no new feature to add on top).

Quoted 6 lines

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

Quoted 5 lines
Quoted 3 lines

RDMA doesn't get new features anyway, if there's specific use case for RDMA migrations, would it work if such a scenario uses the old binary? Is it possible to switch to the TCP protocol with some good NICs?

We have used rdma migration with HCA from Nvidia for years, our experience is RDMA migration works better than tcp (over ipoib).

Quoted 8 lines

Please bare with me, as I know little on rdma stuff.

I'm actually pretty confused (and since a long time ago..) on why we need to operation with rdma contexts when ipoib seems to provide all the tcp layers. I meant, can it work with the current "tcp:" protocol with ipoib even if there's rdma/ib hardwares underneath? Is it because of performance improvements so that we must use a separate path comparing to generic "tcp:" protocol here?

using rdma protocol with ib verbs , we can leverage the full benefit of RDMA by talking directly to NIC which bypasses the kernel overhead, less cpu utilization and better performance.

While IPoIB is more for compatibility to applications using tcp, but can't get full benefit of RDMA. When you have mix generation of IB devices, there are performance issue on IPoIB, we've seen 40G HCA can only reach 2 Gb/s on IPoIB, but with raw RDMA can reach full line speed.

I just run a simple iperf3 test via ipoib and ib_send_bw on same hosts:

iperf 3.9
Linux ps404a-3 5.15.137-pserver #5.15.137-6~deb11 SMP Thu Jan 4 07:19:34 UTC 2024 x86_64
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
Time: Tue, 09 Apr 2024 06:55:02 GMT
Accepted connection from 2a02:247f:401:4:2:0:b:3, port 41130
Cookie: cer2hexlldrowclq6izh7gbg5toviffqbcwt
TCP MSS: 0 (default)
[ 5] local 2a02:247f:401:4:2:0:a:3 port 5201 connected to 2a02:247f:401:4:2:0:b:3 port 41136
Starting Test: protocol: TCP, 1 streams, 131072 byte blocks, omitting 0 seconds, 10 second test, tos 0

[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec  1.80 GBytes  15.5 Gbits/sec
[  5]   1.00-2.00   sec  1.85 GBytes  15.9 Gbits/sec
[  5]   2.00-3.00   sec  1.88 GBytes  16.2 Gbits/sec
[  5]   3.00-4.00   sec  1.87 GBytes  16.1 Gbits/sec
[  5]   4.00-5.00   sec  1.88 GBytes  16.2 Gbits/sec
[  5]   5.00-6.00   sec  1.93 GBytes  16.6 Gbits/sec
[  5]   6.00-7.00   sec  2.00 GBytes  17.2 Gbits/sec
[  5]   7.00-8.00   sec  1.93 GBytes  16.6 Gbits/sec
[  5]   8.00-9.00   sec  1.86 GBytes  16.0 Gbits/sec
[  5]   9.00-10.00  sec  1.95 GBytes  16.8 Gbits/sec
[  5]  10.00-10.04  sec  85.2 MBytes  17.3 Gbits/sec

- - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results:
[ ID] Interval Transfer Bitrate
[ 5] (sender statistics not available)
[ 5] 0.00-10.04 sec 19.0 GBytes 16.3 Gbits/sec receiver
rcv_tcp_congestion cubic
iperf 3.9
Linux ps404a-3 5.15.137-pserver #5.15.137-6~deb11 SMP Thu Jan 4 07:19:34 UTC 2024 x86_64
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
^Ciperf3: interrupt - the server has terminated
1 [email protected]:~$ sudo ib_send_bw -F -a

************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
Send BW Test
Dual-port : OFF Device : mlx5_0
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
PCIe relax order: ON

 ibv_wr* API     : ON
 RX depth        : 512
 CQ Moderation   : 100
 Mtu             : 4096[B]
 Link type       : IB

Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0x24 QPN 0x0174 PSN 0x300138
remote address: LID 0x17 QPN 0x004a PSN 0xc54d6f
---------------------------------------------------------------------------------------

 #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]
 2          1000             0.00               6.46       3.385977
 4          1000             0.00               10.38     2.721894
 8          1000             0.00               25.69     3.367830
 16         1000             0.00               41.46     2.716859
 32         1000             0.00               102.98    3.374577
 64         1000             0.00               206.12    3.377053
 128        1000             0.00               405.03    3.318007
 256        1000             0.00               821.52    3.364939
 512        1000             0.00               2150.78    4.404803
 1024       1000             0.00               4288.13    4.391044
 2048       1000             0.00               8518.25    4.361346
 4096       1000             0.00               11440.77    2.928836
 8192       1000             0.00               11526.45    1.475385
 16384      1000             0.00               11526.06    0.737668
 32768      1000             0.00               11524.86    0.368795
 65536      1000             0.00               11331.84    0.181309
 131072     1000             0.00               11524.75    0.092198
 262144     1000             0.00               11525.82    0.046103
 524288     1000             0.00               11524.70    0.023049
 1048576    1000             0.00               11510.84    0.011511
 2097152    1000             0.00               11524.58    0.005762
 4194304    1000             0.00               11514.26    0.002879
 8388608    1000             0.00               11511.01    0.001439

---------------------------------------------------------------------------------------

you can see with ipoib, it reaches 16 Gb/s using TCP, 1 streams, 131072 byte blocks
with RDMA at 4k+ message size it reaches 100 Gb/s

Quoted 5 lines
Quoted 4 lines

Switching back to TCP will lead us to the old problems which was solved by RDMA migration.

Quoted 7 lines

Can you elaborate the problems, and why tcp won't work in this case? They may not be directly relevant to the issue we're discussing, but I'm happy to learn more.

What is the NICs you were testing before? Did the test carry out with things like modern ones (50Gbps-200Gbps NICs), or the test was done when these hardwares are not common?

We use Mellanox/NVidia IB HCA from 40 Gb/s to 200 Gb/s mixed generation across globe.

Quoted 3 lines

Per my recent knowledge on the new Intel hardwares, at least the ones that support QPL, it's easy to achieve single core 50Gbps+.

In good case, I've also seen 50 Gbps + on Mellanox HCA.

Quoted 5 lines

https://lore.kernel.org/r/PH7PR11MB5941A91AC1E514BCC32896A6A3342@PH7PR11MB5941.namprd11.prod.outlook.com

Quote from Yuan:

Quoted 8 lines

Yes, I use iperf3 to check the bandwidth for one core, the bandwith is 60Gbps.

  [ ID] Interval           Transfer     Bitrate         Retr  Cwnd
  [  5]   0.00-1.00   sec  7.00 GBytes  60.1 Gbits/sec    0   2.87 MBytes
  [  5]   1.00-2.00   sec  7.05 GBytes  60.6 Gbits/sec    0   2.87 Mbytes

And in the live migration test, a multifd thread's CPU utilization is almost 100%

It boils down to what old problems were there with tcp first, though.

Yeah, this is the key reason we use RDMA. (low cpu ulitization and better performance)

Quoted 19 lines
Quoted 11 lines
Quoted 11 lines

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

Quoted 4 lines

+cc some guys from Huawei.

I'm surprised RDMA users are rare, I guess maybe many are just working with different code base.

Yes, please cc whoever might be interested (or surprised.. :) to know this, and let's be open to all possibilities.

Quoted 7 lines

I don't think it makes sense if there're a lot of users of a feature then we deprecate that without a good reason. However there's always the resource limitation issue we're facing, so it could still have the possibility that this gets deprecated if nobody is working on our upstream branch. Say, if people use private branches anyway to support rdma without collaborating upstream, keeping such feature upstream then may not make much sense either, unless there's some way to collaborate. We'll see.

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

Quoted 3 lines

It seems there can still be people joining this discussion. I'll hold off a bit on merging this patch to provide enough window for anyone to chim in.

Thx for discussion and understanding.

Jinpu Wang

Quoted 17 lines

Thanks,

Quoted 10 lines
Quoted 5 lines

Thanks,

--
Peter Xu

Thx!
Jinpu Wang

Quoted 1 line

--
Peter Xu

#17 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 178 lines · Jinpu Wang

Hi Peter,

Quoted 69 lines · Peter Xu
Quoted 1 line · Jinpu Wang

Hi Peter,

Jinpu,

Thanks for joining the discussion.

Quoted 55 lines · Peter Xu
Quoted 53 lines · Yu Zhang
Quoted 4 lines

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

Quoted 14 lines
Quoted 6 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

Quoted 2 lines

The maintenance of this part could be too costly or difficult from your point of view.

It may or may not be too costly, it's just that we need real users of RDMA taking some care of it. Having it broken easily for >1 releases definitely is a sign of lack of users. It is an implication to the community that we should consider dropping some features so that we can get the best use of the community resources for the things that may have a broader audience.

One thing majorly missing is a RDMA tester to guard all the merges to not break RDMA paths, hopefully in CI. That should not rely on RDMA hardwares but just to sanity check the migration+rdma code running all fine. RDMA taught us the lesson so we're requesting CI coverage for all other new features that will be merged at least for migration subsystem, so that we plan to not merge anything that is not covered by CI unless extremely necessary in the future.

For sure CI is not the only missing part, but I'd say we should start with it, then someone should also take care of the code even if only in maintenance mode (no new feature to add on top).

Quoted 6 lines

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

Quoted 5 lines
Quoted 3 lines

RDMA doesn't get new features anyway, if there's specific use case for RDMA migrations, would it work if such a scenario uses the old binary? Is it possible to switch to the TCP protocol with some good NICs?

We have used rdma migration with HCA from Nvidia for years, our experience is RDMA migration works better than tcp (over ipoib).

Quoted 8 lines

Please bare with me, as I know little on rdma stuff.

I'm actually pretty confused (and since a long time ago..) on why we need to operation with rdma contexts when ipoib seems to provide all the tcp layers. I meant, can it work with the current "tcp:" protocol with ipoib even if there's rdma/ib hardwares underneath? Is it because of performance improvements so that we must use a separate path comparing to generic "tcp:" protocol here?

using rdma protocol with ib verbs , we can leverage the full benefit of RDMA by talking directly to NIC which bypasses the kernel overhead, less cpu utilization and better performance.

While IPoIB is more for compatibility to applications using tcp, but can't get full benefit of RDMA. When you have mix generation of IB devices, there are performance issue on IPoIB, we've seen 40G HCA can only reach 2 Gb/s on IPoIB, but with raw RDMA can reach full line speed.

I just run a simple iperf3 test via ipoib and ib_send_bw on same hosts:

iperf 3.9
Linux ps404a-3 5.15.137-pserver #5.15.137-6~deb11 SMP Thu Jan 4 07:19:34 UTC 2024 x86_64
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
Time: Tue, 09 Apr 2024 06:55:02 GMT
Accepted connection from 2a02:247f:401:4:2:0:b:3, port 41130
Cookie: cer2hexlldrowclq6izh7gbg5toviffqbcwt
TCP MSS: 0 (default)
[ 5] local 2a02:247f:401:4:2:0:a:3 port 5201 connected to 2a02:247f:401:4:2:0:b:3 port 41136
Starting Test: protocol: TCP, 1 streams, 131072 byte blocks, omitting 0 seconds, 10 second test, tos 0

[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec  1.80 GBytes  15.5 Gbits/sec
[  5]   1.00-2.00   sec  1.85 GBytes  15.9 Gbits/sec
[  5]   2.00-3.00   sec  1.88 GBytes  16.2 Gbits/sec
[  5]   3.00-4.00   sec  1.87 GBytes  16.1 Gbits/sec
[  5]   4.00-5.00   sec  1.88 GBytes  16.2 Gbits/sec
[  5]   5.00-6.00   sec  1.93 GBytes  16.6 Gbits/sec
[  5]   6.00-7.00   sec  2.00 GBytes  17.2 Gbits/sec
[  5]   7.00-8.00   sec  1.93 GBytes  16.6 Gbits/sec
[  5]   8.00-9.00   sec  1.86 GBytes  16.0 Gbits/sec
[  5]   9.00-10.00  sec  1.95 GBytes  16.8 Gbits/sec
[  5]  10.00-10.04  sec  85.2 MBytes  17.3 Gbits/sec

- - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results:
[ ID] Interval Transfer Bitrate
[ 5] (sender statistics not available)
[ 5] 0.00-10.04 sec 19.0 GBytes 16.3 Gbits/sec receiver
rcv_tcp_congestion cubic
iperf 3.9
Linux ps404a-3 5.15.137-pserver #5.15.137-6~deb11 SMP Thu Jan 4 07:19:34 UTC 2024 x86_64
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
^Ciperf3: interrupt - the server has terminated
1 [email protected]:~$ sudo ib_send_bw -F -a

************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
Send BW Test
Dual-port : OFF Device : mlx5_0
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
PCIe relax order: ON

 ibv_wr* API     : ON
 RX depth        : 512
 CQ Moderation   : 100
 Mtu             : 4096[B]
 Link type       : IB

Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0x24 QPN 0x0174 PSN 0x300138
remote address: LID 0x17 QPN 0x004a PSN 0xc54d6f
---------------------------------------------------------------------------------------

 #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]
 2          1000             0.00               6.46       3.385977
 4          1000             0.00               10.38     2.721894
 8          1000             0.00               25.69     3.367830
 16         1000             0.00               41.46     2.716859
 32         1000             0.00               102.98    3.374577
 64         1000             0.00               206.12    3.377053
 128        1000             0.00               405.03    3.318007
 256        1000             0.00               821.52    3.364939
 512        1000             0.00               2150.78    4.404803
 1024       1000             0.00               4288.13    4.391044
 2048       1000             0.00               8518.25    4.361346
 4096       1000             0.00               11440.77    2.928836
 8192       1000             0.00               11526.45    1.475385
 16384      1000             0.00               11526.06    0.737668
 32768      1000             0.00               11524.86    0.368795
 65536      1000             0.00               11331.84    0.181309
 131072     1000             0.00               11524.75    0.092198
 262144     1000             0.00               11525.82    0.046103
 524288     1000             0.00               11524.70    0.023049
 1048576    1000             0.00               11510.84    0.011511
 2097152    1000             0.00               11524.58    0.005762
 4194304    1000             0.00               11514.26    0.002879
 8388608    1000             0.00               11511.01    0.001439
Quoted 5 lines

---------------------------------------------------------------------------------------

you can see with ipoib, it reaches 16 Gb/s using TCP, 1 streams, 131072 byte blocks
with RDMA at 4k+ message size it reaches 100 Gb/s

I get it now, thank you!

Quoted 14 lines
Quoted 7 lines
Quoted 6 lines

Switching back to TCP will lead us to the old problems which was solved by RDMA migration.

Quoted 7 lines

Can you elaborate the problems, and why tcp won't work in this case? They may not be directly relevant to the issue we're discussing, but I'm happy to learn more.

What is the NICs you were testing before? Did the test carry out with things like modern ones (50Gbps-200Gbps NICs), or the test was done when these hardwares are not common?

Quoted 6 lines

We use Mellanox/NVidia IB HCA from 40 Gb/s to 200 Gb/s mixed generation across globe.

Quoted 3 lines

Per my recent knowledge on the new Intel hardwares, at least the ones that support QPL, it's easy to achieve single core 50Gbps+.

In good case, I've also seen 50 Gbps + on Mellanox HCA.

I see. Have you compared the HCAs v.s. the modern NICs? Now NICs can achieve similar performance from their spec as I said; I am not sure how they perform in real life, but maybe worth trying. I only tried 100G nic and I rem I can hit 70+Gbps with multifd migrations at peak bandwidth. Have you tried that before?

Note that here I didn't want to compare the performance between the two and find a winner. The issue we're facing now is we have the RDMA migration now mostly having its own path all over the place, while the rest protocols (socket, fd, file, etc.) all share the rest.

Then, _if_ modern NICs can work similarly v.s. rdma, I don't yet see a good reason to keep it. It could be that technology just improved so we can use less code to do as good. It's a good news to help QEMU evolve by dropping unused code.

For some details there on the rdma complications for migration:

(1) RDMA is the only protocol that doesn't yet support QIOChannel, while migration uses QIOChannels mostly everywhere now.. e.g. in multifd, it means it won't easily support any new things using QIOChannels.

(2) RDMA is the only protocol that mostly hard-coded everywhere in the RAM migrations, polluting the core logic with much more code internally to support this protocol.

For (1), see migrate_fd_connect() from rdma_start_outgoing_migration(). While the rest protocols all go via migration_channel_connect().

For (2), see all the "rdma_*" functions in migration/ram.c, where I don't think it's common to a protocol - most of the rest protocols don't need those hard-coded stuff. migration/rdma.c has 4000+ LOC for these stuff, while to do a not-so-fair comparison, migration/fd.c only has <100 LOC.

Then, we found we don't even know who's using it.

I hope I explained why people started this idea, and also why I think that makes sense at least to me.

Quoted 42 lines
Quoted 5 lines

https://lore.kernel.org/r/PH7PR11MB5941A91AC1E514BCC32896A6A3342@PH7PR11MB5941.namprd11.prod.outlook.com

Quote from Yuan:

Quoted 8 lines

Yes, I use iperf3 to check the bandwidth for one core, the bandwith is 60Gbps.

  [ ID] Interval           Transfer     Bitrate         Retr  Cwnd
  [  5]   0.00-1.00   sec  7.00 GBytes  60.1 Gbits/sec    0   2.87 MBytes
  [  5]   1.00-2.00   sec  7.05 GBytes  60.6 Gbits/sec    0   2.87 Mbytes

And in the live migration test, a multifd thread's CPU utilization is almost 100%

It boils down to what old problems were there with tcp first, though.

Yeah, this is the key reason we use RDMA. (low cpu ulitization and better performance)

Quoted 19 lines
Quoted 11 lines
Quoted 11 lines

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

Quoted 4 lines

+cc some guys from Huawei.

I'm surprised RDMA users are rare, I guess maybe many are just working with different code base.

Yes, please cc whoever might be interested (or surprised.. :) to know this, and let's be open to all possibilities.

Quoted 7 lines

I don't think it makes sense if there're a lot of users of a feature then we deprecate that without a good reason. However there's always the resource limitation issue we're facing, so it could still have the possibility that this gets deprecated if nobody is working on our upstream branch. Say, if people use private branches anyway to support rdma without collaborating upstream, keeping such feature upstream then may not make much sense either, unless there's some way to collaborate. We'll see.

Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Quoted 5 lines
Quoted 3 lines

It seems there can still be people joining this discussion. I'll hold off a bit on merging this patch to provide enough window for anyone to chim in.

Thx for discussion and understanding.

Thanks for all these inputs so far. These can help us make a wiser and clearer step no matter which way we choose.

Signature · 1 line
Peter Xu
#18 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah,  RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Thanks
Zhijian

Quoted 6 lines
Quoted 4 lines
Quoted 3 lines

It seems there can still be people joining this discussion. I'll hold off a bit on merging this patch to provide enough window for anyone to chim in.

Thx for discussion and understanding.

Thanks for all these inputs so far. These can help us make a wiser and clearer step no matter which way we choose.

#19 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 19 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah,  RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Signature · 1 line
Peter Xu
#20 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 26 lines · Peter Xu
Quoted 19 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah,  RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Quoted 4 lines

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Some people got dropped on the cc list for unknown reason, I'm adding them back (Fabiano, Peter Maydell, Phil). Let's make sure nobody is dropped by accident.

I'll try to summarize what is still missing, and I think these will be greatly helpful if we don't want to deprecate rdma migration:

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

2) Some performance tests between modern RDMA and NIC devices are welcomed. The current knowledge is modern NIC can work similarly to RDMA in performance, then it's debatable why we still maintain so much rdma specific code.

3) No need to be soild patchsets for this one, but some plan to improve RDMA migration code so that it is not almost isolated from the rest protocols.

4) Someone to look after this code for real.

For 2) and 3) more info is here:

https://lore.kernel.org/r/ZhWa0YeAb9ySVKD1@x1n

Here 4) can be the most important as Markus pointed out. We just didn't get there yet on the discussions, but maybe Markus is right that we should talk that first.

Thanks,

Signature · 1 line
Peter Xu
#21 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 2 lines

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

We use a batch of regression test cases for the stack, which covers the test for QEMU. I did such test for most of the QEMU releases planned as candidates for rollout.

The migration test needs a pair of (either physical or virtual) servers with InfiniBand network, which makes it difficult to do on a single server. The nested VM could be a possible approach, for which we may need virtual InfiniBand network. Is SoftRoCE [1] a choice? I will try it and let you know.

[1] https://enterprise-support.nvidia.com/s/article/howto-configure-soft-roce

Thanks and best regards!

Quoted 67 lines · Peter Xu
Quoted 26 lines · Peter Xu
Quoted 19 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah, RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Quoted 4 lines

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Some people got dropped on the cc list for unknown reason, I'm adding them back (Fabiano, Peter Maydell, Phil). Let's make sure nobody is dropped by accident.

I'll try to summarize what is still missing, and I think these will be greatly helpful if we don't want to deprecate rdma migration:

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

2) Some performance tests between modern RDMA and NIC devices are welcomed. The current knowledge is modern NIC can work similarly to RDMA in performance, then it's debatable why we still maintain so much rdma specific code.

3) No need to be soild patchsets for this one, but some plan to improve RDMA migration code so that it is not almost isolated from the rest protocols.

4) Someone to look after this code for real.

For 2) and 3) more info is here:

https://lore.kernel.org/r/ZhWa0YeAb9ySVKD1@x1n

Here 4) can be the most important as Markus pointed out. We just didn't get there yet on the discussions, but maybe Markus is right that we should talk that first.

Thanks,

--
Peter Xu

#22 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Yu,

Quoted 5 lines · Yu Zhang
Quoted 2 lines

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

We use a batch of regression test cases for the stack, which covers the test for QEMU. I did such test for most of the QEMU releases planned as candidates for rollout.

The least I can think of is a few tests in one release. Definitely too less if one release can already break..

Quoted 7 lines

The migration test needs a pair of (either physical or virtual) servers with InfiniBand network, which makes it difficult to do on a single server. The nested VM could be a possible approach, for which we may need virtual InfiniBand network. Is SoftRoCE [1] a choice? I will try it and let you know.

[1] https://enterprise-support.nvidia.com/s/article/howto-configure-soft-roce

Does it require a kernel driver? The less host kernel / hardware / .. dependencies the better.

I am wondering whether there can be a library doing everything in userspace, translating RDMA into e.g. socket messages (so maybe ultimately that's something like IP->rdma->IP.. just to cover the "rdma" procedures), then that'll work for CI reliably.

Please also see my full list, though, especially entry 4). Thanks already for looking for solutions on the tests, but I don't want to waste your time then found that tests are not enough even if ready. I think we need people that understand these stuff well enough, have dedicated time and look after it.

Thanks,

Signature · 1 line
Peter Xu
#23 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Yu Zhang

Hi All (and Peter),

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name: https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

If you don't mind, I'd like to let him send over his (very detailed) proposal,

- Michael

Quoted 78 lines · Yu Zhang
Quoted 2 lines

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

We use a batch of regression test cases for the stack, which covers the test for QEMU. I did such test for most of the QEMU releases planned as candidates for rollout.

The migration test needs a pair of (either physical or virtual) servers with InfiniBand network, which makes it difficult to do on a single server. The nested VM could be a possible approach, for which we may need virtual InfiniBand network. Is SoftRoCE [1] a choice? I will try it and let you know.

[1] https://urldefense.com/v3/__https://enterprise-support.nvidia.com/s/article/howto-configure-soft-roce__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWvGHE3ig$

Thanks and best regards!

Quoted 62 lines · Peter Xu
Quoted 23 lines · Peter Xu
Quoted 17 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah, RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Quoted 4 lines

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Some people got dropped on the cc list for unknown reason, I'm adding them back (Fabiano, Peter Maydell, Phil). Let's make sure nobody is dropped by accident.

I'll try to summarize what is still missing, and I think these will be greatly helpful if we don't want to deprecate rdma migration:

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

2) Some performance tests between modern RDMA and NIC devices are welcomed. The current knowledge is modern NIC can work similarly to RDMA in performance, then it's debatable why we still maintain so much rdma specific code.

3) No need to be soild patchsets for this one, but some plan to improve RDMA migration code so that it is not almost isolated from the rest protocols.

4) Someone to look after this code for real.

For 2) and 3) more info is here:

https://urldefense.com/v3/__https://lore.kernel.org/r/ZhWa0YeAb9ySVKD1@x1n__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWpIWYBhQ$

Here 4) can be the most important as Markus pointed out. We just didn't get there yet on the discussions, but maybe Markus is right that we should talk that first.

Thanks,

--
Peter Xu

#24 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 9 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:
https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

Quoted 8 lines

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

Quoted 3 lines

If you don't mind, I'd like to let him send over his (very detailed) proposal,

Yes please, it's exactly the time to share the plan. The hope is we try to reach a consensus before or around the middle of this release (9.1). Normally QEMU has a 3~4 months window for each release and 9.1 schedule is not yet out, but I think it means we make a decision before or around middle of June.

Thanks,

Quoted 83 lines

- Michael

Quoted 78 lines · Yu Zhang
Quoted 2 lines

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

We use a batch of regression test cases for the stack, which covers the test for QEMU. I did such test for most of the QEMU releases planned as candidates for rollout.

The migration test needs a pair of (either physical or virtual) servers with InfiniBand network, which makes it difficult to do on a single server. The nested VM could be a possible approach, for which we may need virtual InfiniBand network. Is SoftRoCE [1] a choice? I will try it and let you know.

[1] https://urldefense.com/v3/__https://enterprise-support.nvidia.com/s/article/howto-configure-soft-roce__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWvGHE3ig$

Thanks and best regards!

Quoted 62 lines · Peter Xu
Quoted 23 lines · Peter Xu
Quoted 17 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah, RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Quoted 4 lines

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Some people got dropped on the cc list for unknown reason, I'm adding them back (Fabiano, Peter Maydell, Phil). Let's make sure nobody is dropped by accident.

I'll try to summarize what is still missing, and I think these will be greatly helpful if we don't want to deprecate rdma migration:

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

2) Some performance tests between modern RDMA and NIC devices are welcomed. The current knowledge is modern NIC can work similarly to RDMA in performance, then it's debatable why we still maintain so much rdma specific code.

3) No need to be soild patchsets for this one, but some plan to improve RDMA migration code so that it is not almost isolated from the rest protocols.

4) Someone to look after this code for real.

For 2) and 3) more info is here:

https://urldefense.com/v3/__https://lore.kernel.org/r/ZhWa0YeAb9ySVKD1@x1n__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWpIWYBhQ$

Here 4) can be the most important as Markus pointed out. We just didn't get there yet on the discussions, but maybe Markus is right that we should talk that first.

Thanks,

--
Peter Xu

Signature · 1 line
Peter Xu
#25 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hello Michael and Peter,

We are very glad at your quick and kind reply about our plan to take over the maintenance of your code. The message is for presenting our plan and working together.
If we were able to obtain the maintainer's role, our plan is:

1. Create the necessary unit-test cases and get them integrated into the current QEMU GitLab-CI pipeline
2. Review and test the code changes by other developers to ensure that nothing is broken in the changed code before being merged by the community
3. Based on our current practice and application scenario, look for possible improvements when necessary

Besides that, a patch is attached to announce this change in the community.

With your generous support, we hope that the development community will make a positive decision for us.

Kind regards,
Yu Zhang@ IONOS Cloud

Quoted 122 lines · Peter Xu
Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 9 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:
https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

Quoted 8 lines

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

Quoted 3 lines

If you don't mind, I'd like to let him send over his (very detailed) proposal,

Yes please, it's exactly the time to share the plan. The hope is we try to reach a consensus before or around the middle of this release (9.1). Normally QEMU has a 3~4 months window for each release and 9.1 schedule is not yet out, but I think it means we make a decision before or around middle of June.

Thanks,

Quoted 79 lines

- Michael

Quoted 75 lines · Yu Zhang
Quoted 2 lines

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

We use a batch of regression test cases for the stack, which covers the test for QEMU. I did such test for most of the QEMU releases planned as candidates for rollout.

The migration test needs a pair of (either physical or virtual) servers with InfiniBand network, which makes it difficult to do on a single server. The nested VM could be a possible approach, for which we may need virtual InfiniBand network. Is SoftRoCE [1] a choice? I will try it and let you know.

[1] https://urldefense.com/v3/__https://enterprise-support.nvidia.com/s/article/howto-configure-soft-roce__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWvGHE3ig$

Thanks and best regards!

Quoted 59 lines · Peter Xu
Quoted 23 lines · Peter Xu
Quoted 17 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah, RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Quoted 4 lines

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Some people got dropped on the cc list for unknown reason, I'm adding them back (Fabiano, Peter Maydell, Phil). Let's make sure nobody is dropped by accident.

I'll try to summarize what is still missing, and I think these will be greatly helpful if we don't want to deprecate rdma migration:

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

2) Some performance tests between modern RDMA and NIC devices are welcomed. The current knowledge is modern NIC can work similarly to RDMA in performance, then it's debatable why we still maintain so much rdma specific code.

3) No need to be soild patchsets for this one, but some plan to improve RDMA migration code so that it is not almost isolated from the rest protocols.

4) Someone to look after this code for real.

For 2) and 3) more info is here:

https://urldefense.com/v3/__https://lore.kernel.org/r/ZhWa0YeAb9ySVKD1@x1n__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWpIWYBhQ$

Here 4) can be the most important as Markus pointed out. We just didn't get there yet on the discussions, but maybe Markus is right that we should talk that first.

Thanks,

Quoted 8 lines
Quoted 4 lines
Quoted 3 lines
Quoted 3 lines

--
Peter Xu

--
Peter Xu

From 40dea392f9ca606c2a0c53999d662670eb08b2d8 Mon Sep 17 00:00:00 2001
From: Yu Zhang <[email protected]>
Date: Mon, 29 Apr 2024 15:31:53 +0200
Subject: [PATCH] MAINTAINERS: Update the maintainers and reviewers for RDMA
migration

As the links [1][2] below stated, QEMU development community is currently having some difficulties in maintaining the RDMA migration subsystem due to the lack of resources (maintainers, test cases, test environment etc.) and considering to deprecate it.

According to our user experience in the recent two years, we observed that RDMA is capable of providing higher migration speed and lower performance impact to a running VM, which can significantly improve the end-user's experience during the VM live migration. We believe that RDMA still plays a key role for the QoS and can't yet be replaced by TCP/IP for VM migration at the moment.

With the consent and supports from Michael Galaxy, who has developed this feature for QEMU, we would like to take over the maintainer's role and create the necessary resources to maintain it further for the community.

Jinpu Wang is the upstream maintainer of RNBD/RTRS. He is experienced in RDMA programming, and Yu Zhang maintains the downstream QEMU for IONOS cloud in production.

[1] https://mail.gnu.org/archive/html/qemu-devel/2024-04/msg00001.html
[2] https://mail.gnu.org/archive/html/qemu-devel/2024-04/msg00228.html

---
 MAINTAINERS | 3 +++
 1 file changed, 3 insertions(+)
MAINTAINERS +3 -0
index 302b6fd00c..54d32dff94 100644
@@ -3413,7 +3413,10 @@ F: util/userfaultfd.c
 X: migration/rdma*
 
 RDMA Migration
+M: Yu Zhang <[email protected]>
+M: Jack Wang <[email protected]>
 R: Li Zhijian <[email protected]>
+R: Michael Galaxy <[email protected]>
 R: Peter Xu <[email protected]>
 S: Odd Fixes
 F: migration/rdma*
Signature · 1 line
2.25.1
Reviewed-by
#26 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Thanks Yu Zhang and Peter.

- Michael

Quoted 144 lines · Yu Zhang

Hello Michael and Peter,

We are very glad at your quick and kind reply about our plan to take over the maintenance of your code. The message is for presenting our plan and working together.
If we were able to obtain the maintainer's role, our plan is:

1. Create the necessary unit-test cases and get them integrated into the current QEMU GitLab-CI pipeline
2. Review and test the code changes by other developers to ensure that nothing is broken in the changed code before being merged by the community
3. Based on our current practice and application scenario, look for possible improvements when necessary

Besides that, a patch is attached to announce this change in the community.

With your generous support, we hope that the development community will make a positive decision for us.

Kind regards,
Yu Zhang@ IONOS Cloud

Quoted 120 lines · Peter Xu
Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 8 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:

https://urldefense.com/v3/__https://www.linkedin.com/in/mrgalaxy/__;!!GjvTz_vk!TZmnCE90EK692dSjZGr-2cpOEZBQTBsTO2bW5z3rSbpZgNVCexZkxwDXhmIOWG2GAKZAUovQ5xe5coQ$ )

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

Quoted 8 lines

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

Quoted 2 lines

If you don't mind, I'd like to let him send over his (very detailed) proposal,

Yes please, it's exactly the time to share the plan. The hope is we try to reach a consensus before or around the middle of this release (9.1). Normally QEMU has a 3~4 months window for each release and 9.1 schedule is not yet out, but I think it means we make a decision before or around middle of June.

Thanks,

Quoted 80 lines

- Michael

Quoted 77 lines · Yu Zhang
Quoted 2 lines

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

We use a batch of regression test cases for the stack, which covers the test for QEMU. I did such test for most of the QEMU releases planned as candidates for rollout.

The migration test needs a pair of (either physical or virtual) servers with InfiniBand network, which makes it difficult to do on a single server. The nested VM could be a possible approach, for which we may need virtual InfiniBand network. Is SoftRoCE [1] a choice? I will try it and let you know.

[1] https://urldefense.com/v3/__https://enterprise-support.nvidia.com/s/article/howto-configure-soft-roce__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWvGHE3ig$

Thanks and best regards!

Quoted 61 lines · Peter Xu
Quoted 22 lines · Peter Xu
Quoted 16 lines · Zhijian Li (Fujitsu) via
Quoted 9 lines · Peter Xu
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

Yeah, RXE aka. SOFT-RoCE is able to emulate the RDMA, for example

$ sudo rdma link add rxe_eth0 type rxe netdev eth0  # on host
then we can get a new RDMA interface "rxe_eth0".
This new RDMA interface is able to do the QEMU RDMA migration.
Quoted 5 lines

Also, the loopback(lo) device is able to emulate the RDMA interface "rxe_lo", however when
I tried(years ago) to do RDMA migration over this interface(rdma:127.0.0.1:3333) , it got something wrong. So i gave up enabling the RDMA migration qtest at that time.

Quoted 4 lines

Thanks, Zhijian.

I'm not sure adding an emu-link for rdma is doable for CI systems, though. Maybe someone more familiar with how CI works can chim in.

Some people got dropped on the cc list for unknown reason, I'm adding them back (Fabiano, Peter Maydell, Phil). Let's make sure nobody is dropped by accident.

I'll try to summarize what is still missing, and I think these will be greatly helpful if we don't want to deprecate rdma migration:

1) Either a CI test covering at least the major RDMA paths, or at least periodically tests for each QEMU release will be needed.

2) Some performance tests between modern RDMA and NIC devices are welcomed. The current knowledge is modern NIC can work similarly to RDMA in performance, then it's debatable why we still maintain so much rdma specific code.

3) No need to be soild patchsets for this one, but some plan to improve RDMA migration code so that it is not almost isolated from the rest protocols.

4) Someone to look after this code for real.

For 2) and 3) more info is here:

https://urldefense.com/v3/__https://lore.kernel.org/r/ZhWa0YeAb9ySVKD1@x1n__;!!GjvTz_vk!VEqNfg3Kdf58Oh1FkGL6ErDLfvUXZXPwMTaXizuIQeIgJiywPzuwbqx8wM0KUsyopw_EYQxWpIWYBhQ$

Here 4) can be the most important as Markus pointed out. We just didn't get there yet on the discussions, but maybe Markus is right that we should talk that first.

Thanks,

--
Peter Xu

--
Peter Xu

#27 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu
Quoted 23 lines · Peter Xu
Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 9 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:
https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

Quoted 8 lines

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

Quoted 6 lines

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

How much experience we expect of maintainers depends on the subsystem and other circumstances. The hard requirement isn't experience, it's trust. See the recent attack on xz.

I do not mean to express any doubts whatsoever on Yu Zhang's integrity! I'm merely reminding y'all what's at stake.

[...]

#28 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 32 lines · Markus Armbruster
Quoted 23 lines · Peter Xu
Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 9 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:
https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

Quoted 8 lines

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

Quoted 6 lines

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

Quoted 6 lines

How much experience we expect of maintainers depends on the subsystem and other circumstances. The hard requirement isn't experience, it's trust. See the recent attack on xz.

I do not mean to express any doubts whatsoever on Yu Zhang's integrity! I'm merely reminding y'all what's at stake.

I think we shouldn't overly obsess[1] about 'xz', because the overwhealmingly common scenario is that volunteer maintainers are honest people. QEMU is in a massively better peer review situation. With xz there was basically no oversight of the new maintainer. With QEMU, we have oversight from 1000's of people on the list, a huge pool of general maintainers, the specific migration maintainers, and the release manager merging code.

With a lack of historical experiance with QEMU maintainership, I'd suggest that new RDMA volunteers would start by adding themselves to the "MAINTAINERS" file with only the 'Reviewer' classification. The main migration maintainers would still handle pull requests, but wait for a R-b from one of the RMDA volunteers. After some period of time the RDMA folks could graduate to full maintainer status if the migration maintainers needed to reduce their load. I suspect that might prove unneccesary though, given RDMA isn't an area of code with a high turnover of patches.

With regards,
Daniel

[1] If we do want to obsess about something bad though, we should look at our handling of binary blobs in the repo and tarballs. ie the firmware binaries that all get built in an arbitrary environment of their respective maintainer. If we need firmware blobs in tree, we should strive to come up with a reprodicble build environment that gives us byte-for-byte identical results, so the blobs can be verified. This is rather a tangent from this thread though :)

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#29 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 47 lines · Daniel P. Berrangé
Quoted 32 lines · Markus Armbruster
Quoted 23 lines · Peter Xu
Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 9 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:
https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

Quoted 8 lines

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run tests) while they learn about it.

Quoted 6 lines

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

Quoted 6 lines

How much experience we expect of maintainers depends on the subsystem and other circumstances. The hard requirement isn't experience, it's trust. See the recent attack on xz.

I do not mean to express any doubts whatsoever on Yu Zhang's integrity! I'm merely reminding y'all what's at stake.

I think we shouldn't overly obsess[1] about 'xz', because the overwhealmingly common scenario is that volunteer maintainers are honest people. QEMU is in a massively better peer review situation. With xz there was basically no oversight of the new maintainer. With QEMU, we have oversight from 1000's of people on the list, a huge pool of general maintainers, the specific migration maintainers, and the release manager merging code.

Quoted 8 lines

With a lack of historical experiance with QEMU maintainership, I'd suggest that new RDMA volunteers would start by adding themselves to the "MAINTAINERS" file with only the 'Reviewer' classification. The main migration maintainers would still handle pull requests, but wait for a R-b from one of the RMDA volunteers. After some period of time the RDMA folks could graduate to full maintainer status if the migration maintainers needed to reduce their load. I suspect that might prove unneccesary though, given RDMA isn't an area of code with a high turnover of patches.

Right, and we can do that as a start, it also follows our normal rules of starting from Reviewers to maintain something. I even considered Zhijian to be the previous rdma goto guy / maintainer no matter what role he used to have in the MAINTAINERS file.

Here IMHO it's more about whether any company would like to stand up and provide help, without yet binding that to be able to send pull requests in the near future or even longer term.

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

So even if we want to keep RDMA around I hope with this chance we can at least have clear picture on when we should still suggest any new user to use RDMA (with the reasons behind). Or we simply shouldn't suggest any new user to use RDMA at all (because at least it'll lose many new migration features).

Thanks,

Signature · 1 line
Peter Xu
#30 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 12 lines · Peter Xu

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

With regards,
Daniel

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#31 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 35 lines · Daniel P. Berrangé
Quoted 12 lines · Peter Xu

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

Quoted 4 lines

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

My understanding so far is RDMA is sololy for performance but nothing else, then it's a question on whether rdma existing users would like to do so if it will run slower.

Jinpu mentioned on the explicit usages of ib verbs but I am just mostly quotting that word as I don't really know such details:

https://lore.kernel.org/qemu-devel/CAMGffEm2TWJxOPcNQTQ1Sjytf5395dBzTCMYiKRqfxDzJwSN6A@mail.gmail.com/

So not sure whether that applies here too, in that having qiochannel wrapper may not allow direct access to those ib verbs.

Thanks,

Quoted 13 lines

With regards,
Daniel

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
Signature · 1 line
Peter Xu
#32 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Yu Zhang / Jinpu,

Any possibility (at your lesiure, and within the disclosure rules of your company, IONOS) if you could share any of your performance information to educate the group?

NICs have indeed changed, but not everybody has 100ge mellanox cards at their disposal. Some people don't.

- Michael

Quoted 65 lines · Peter Xu
Quoted 34 lines · Daniel P. Berrangé
Quoted 12 lines · Peter Xu

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

Quoted 4 lines

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

My understanding so far is RDMA is sololy for performance but nothing else, then it's a question on whether rdma existing users would like to do so if it will run slower.

Jinpu mentioned on the explicit usages of ib verbs but I am just mostly quotting that word as I don't really know such details:

https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/CAMGffEm2TWJxOPcNQTQ1Sjytf5395dBzTCMYiKRqfxDzJwSN6A@mail.gmail.com/__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOew9oW_kg$

So not sure whether that applies here too, in that having qiochannel wrapper may not allow direct access to those ib verbs.

Thanks,

Quoted 12 lines

With regards,
Daniel

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.

Signature · 3 lines
|: https://urldefense.com/v3/__https://berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfyTmFFUQ$       -o-    https://urldefense.com/v3/__https://www.flickr.com/photos/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf8A5OC0Q$  :|
|: https://urldefense.com/v3/__https://libvirt.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf3gffAdg$          -o-            https://urldefense.com/v3/__https://fstop138.berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfPMofYqw$  :|
|: https://urldefense.com/v3/__https://entangle-photo.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOeQ5jjAeQ$     -o-    https://urldefense.com/v3/__https://www.instagram.com/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfhaDF9WA$  :|
#33 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Michael, Hi Peter,

Quoted 9 lines · Michael Galaxy

Yu Zhang / Jinpu,

Any possibility (at your lesiure, and within the disclosure rules of your company, IONOS) if you could share any of your performance information to educate the group?

NICs have indeed changed, but not everybody has 100ge mellanox cards at their disposal. Some people don't.

Our staging env is with 100 Gb/s IB environment.
We will have a new setup in the coming months with Ethernet (RoCE), we will run some performance
comparison when we have the environment ready.

Quoted 2 lines

- Michael

Thx!
Jinpu

Quoted 67 lines · Peter Xu
Quoted 65 lines · Daniel P. Berrangé
Quoted 34 lines · Peter Xu
Quoted 12 lines

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

Quoted 4 lines

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

My understanding so far is RDMA is sololy for performance but nothing else, then it's a question on whether rdma existing users would like to do so if it will run slower.

Jinpu mentioned on the explicit usages of ib verbs but I am just mostly quotting that word as I don't really know such details:

https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/CAMGffEm2TWJxOPcNQTQ1Sjytf5395dBzTCMYiKRqfxDzJwSN6A@mail.gmail.com/__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOew9oW_kg$

So not sure whether that applies here too, in that having qiochannel wrapper may not allow direct access to those ib verbs.

Thanks,

Quoted 12 lines

With regards,
Daniel

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.
--

|: https://urldefense.com/v3/__https://berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfyTmFFUQ$       -o-    https://urldefense.com/v3/__https://www.flickr.com/photos/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf8A5OC0Q$  :|
|: https://urldefense.com/v3/__https://libvirt.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf3gffAdg$          -o-            https://urldefense.com/v3/__https://fstop138.berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfPMofYqw$  :|
|: https://urldefense.com/v3/__https://entangle-photo.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOeQ5jjAeQ$     -o-    https://urldefense.com/v3/__https://www.instagram.com/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfhaDF9WA$  :|
#34 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 11 lines · Jinpu Wang

Hi Michael, Hi Peter,

Quoted 7 lines · Michael Galaxy

Yu Zhang / Jinpu,

Any possibility (at your lesiure, and within the disclosure rules of your company, IONOS) if you could share any of your performance information to educate the group?

Quoted 6 lines
Quoted 2 lines

NICs have indeed changed, but not everybody has 100ge mellanox cards at their disposal. Some people don't.

Our staging env is with 100 Gb/s IB environment.
We will have a new setup in the coming months with Ethernet (RoCE), we will run some performance
comparison when we have the environment ready.

Thanks both. Please keep us posted.

Just to double check, we're comparing "tcp:" v.s. "rdma:", RoCE is not involved, am I right?

The other note is that the comparison needs to be with multifd enabled for the "tcp:" case. I'd suggest we start with 8 threads if it's 100Gbps.

I think I can still fetch some 100Gbps or even 200Gbps nics around our labs without even waiting for months. If you want I can try to see how we can test together. And btw I don't think we need a cluster, IIUC we simply need two hosts, 100G nic on both sides? IOW, it seems to me we only need two cards just for experiments, systems that can drive the cards, and a wire supporting 100G?

Quoted 74 lines
Quoted 3 lines

- Michael

Thx!
Jinpu

Quoted 67 lines · Peter Xu
Quoted 65 lines · Daniel P. Berrangé
Quoted 34 lines · Peter Xu
Quoted 12 lines

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

Quoted 4 lines

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

My understanding so far is RDMA is sololy for performance but nothing else, then it's a question on whether rdma existing users would like to do so if it will run slower.

Jinpu mentioned on the explicit usages of ib verbs but I am just mostly quotting that word as I don't really know such details:

https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/CAMGffEm2TWJxOPcNQTQ1Sjytf5395dBzTCMYiKRqfxDzJwSN6A@mail.gmail.com/__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOew9oW_kg$

So not sure whether that applies here too, in that having qiochannel wrapper may not allow direct access to those ib verbs.

Thanks,

Quoted 12 lines

With regards,
Daniel

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.
--

|: https://urldefense.com/v3/__https://berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfyTmFFUQ$       -o-    https://urldefense.com/v3/__https://www.flickr.com/photos/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf8A5OC0Q$  :|
|: https://urldefense.com/v3/__https://libvirt.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf3gffAdg$          -o-            https://urldefense.com/v3/__https://fstop138.berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfPMofYqw$  :|
|: https://urldefense.com/v3/__https://entangle-photo.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOeQ5jjAeQ$     -o-    https://urldefense.com/v3/__https://www.instagram.com/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfhaDF9WA$  :|
Signature · 1 line
Peter Xu
#35 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter

Quoted 20 lines · Peter Xu
Quoted 11 lines · Jinpu Wang

Hi Michael, Hi Peter,

Quoted 7 lines · Michael Galaxy

Yu Zhang / Jinpu,

Any possibility (at your lesiure, and within the disclosure rules of your company, IONOS) if you could share any of your performance information to educate the group?

Quoted 6 lines
Quoted 2 lines

NICs have indeed changed, but not everybody has 100ge mellanox cards at their disposal. Some people don't.

Our staging env is with 100 Gb/s IB environment.
We will have a new setup in the coming months with Ethernet (RoCE), we will run some performance
comparison when we have the environment ready.

Quoted 4 lines

Thanks both. Please keep us posted.

Just to double check, we're comparing "tcp:" v.s. "rdma:", RoCE is not involved, am I right?

kinds of. Our new hardware is RDMA capable, we can configure it to run in "rdma" transport or "tcp"
it is more straight comparison,
When run "rdma" transport, RoCE is involved, eg the rdma-core/ibverbs/rdmacm/vendor verbs driver are used.

Quoted 10 lines

The other note is that the comparison needs to be with multifd enabled for the "tcp:" case. I'd suggest we start with 8 threads if it's 100Gbps.

I think I can still fetch some 100Gbps or even 200Gbps nics around our labs without even waiting for months. If you want I can try to see how we can test together. And btw I don't think we need a cluster, IIUC we simply need two hosts, 100G nic on both sides? IOW, it seems to me we only need two cards just for experiments, systems that can drive the cards, and a wire supporting 100G?

Yes, the simple setup can be just two hosts directly connected. This remind me, I may also able to find a test setup with 100 G nic in lab, will keep you posted.

Regards!

Quoted 79 lines
Quoted 75 lines
Quoted 4 lines

- Michael

Thx!
Jinpu

Quoted 67 lines · Peter Xu
Quoted 65 lines · Daniel P. Berrangé
Quoted 34 lines · Peter Xu
Quoted 12 lines

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

Quoted 4 lines

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

My understanding so far is RDMA is sololy for performance but nothing else, then it's a question on whether rdma existing users would like to do so if it will run slower.

Jinpu mentioned on the explicit usages of ib verbs but I am just mostly quotting that word as I don't really know such details:

https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/CAMGffEm2TWJxOPcNQTQ1Sjytf5395dBzTCMYiKRqfxDzJwSN6A@mail.gmail.com/__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOew9oW_kg$

So not sure whether that applies here too, in that having qiochannel wrapper may not allow direct access to those ib verbs.

Thanks,

Quoted 12 lines

With regards,
Daniel

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.
--

|: https://urldefense.com/v3/__https://berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfyTmFFUQ$       -o-    https://urldefense.com/v3/__https://www.flickr.com/photos/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf8A5OC0Q$  :|
|: https://urldefense.com/v3/__https://libvirt.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOf3gffAdg$          -o-            https://urldefense.com/v3/__https://fstop138.berrange.com__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfPMofYqw$  :|
|: https://urldefense.com/v3/__https://entangle-photo.org__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOeQ5jjAeQ$     -o-    https://urldefense.com/v3/__https://www.instagram.com/dberrange__;!!GjvTz_vk!W6-HGWM-XkF_52am249DrLIDQeZctVOHg72LvOHGUcwxqQM5mY0GNYYl-yNJslN7A5GfLOfhaDF9WA$  :|

--
Peter Xu

#36 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Daniel P. Berrangé

Hi Daniel,

Quoted 28 lines · Daniel P. Berrangé
Quoted 12 lines · Peter Xu

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

Quoted 8 lines

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

I don't know much about the originals of RDMA support in QEMU and why this particular design was taken. It is indeed a huge maint burden to have a completely different code flow for RDMA with 4000+ lines of custom protocol signalling which is barely understandable.

Quoted 7 lines

I would note that /usr/include/rdma/rsocket.h provides a higher level API that is a 1-1 match of the normal kernel 'sockets' API. If we had leveraged that, then QIOChannelSocket class and the QAPI SocketAddress type could almost[1] trivially have supported RDMA. There would have been almost no RDMA code required in the migration subsystem, and all the modern features like compression, multifd, post-copy, etc would "just work".

I guess at the time rsocket is less mature, and less performant compared to using uverbs directly.

Quoted 5 lines

I guess the 'rsocket.h' shim may well limit some of the possible performance gains, but it might still have been a better tradeoff to have not quite so good peak performance, but with massively less maint burden.

I had a brief check in the rsocket changelog, there seems some improvement over time,
might be worth revisiting this. due to socket abstraction, we can't
use some feature like
ODP, it won't be a small and easy task.

Quoted 2 lines

With regards,
Daniel

Thanks for the suggestion.

Quoted 10 lines

[1] "almost" trivially, because the poll() integration for rsockets requires a bit more magic sauce since rsockets FDs are not really FDs from the kernel's POV. Still, QIOCHannel likely can abstract that probme.
--

|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#37 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 5 lines · Jinpu Wang

I had a brief check in the rsocket changelog, there seems some improvement over time,
might be worth revisiting this. due to socket abstraction, we can't
use some feature like
ODP, it won't be a small and easy task.

It'll be good to know whether Dan's suggestion would work first, without rewritting everything yet so far. Not sure whether some perf test could help with the rsocket APIs even without QEMU's involvements (or looking for test data supporting / invalidate such conversions).

Thanks,

Signature · 1 line
Peter Xu
#38 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter, hi Daniel,

Quoted 8 lines · Peter Xu
Quoted 5 lines · Jinpu Wang

I had a brief check in the rsocket changelog, there seems some improvement over time,
might be worth revisiting this. due to socket abstraction, we can't
use some feature like
ODP, it won't be a small and easy task.

Quoted 5 lines

It'll be good to know whether Dan's suggestion would work first, without rewritting everything yet so far. Not sure whether some perf test could help with the rsocket APIs even without QEMU's involvements (or looking for test data supporting / invalidate such conversions).

I did a quick test with iperf on 100 G environment and 40 G environment, in summary rsocket works pretty well.

iperf tests between 2 hosts with 40 G (IB),
first a few test with different num. of threads on top of ipoib interface, later with preload rsocket on top of same ipoib interface.

[email protected]:~$ iperf -p 52000 -c 10.43.3.145
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 55602 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0001 sec  2.85 GBytes  2.44 Gbits/sec
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 2
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.3.146 port 39640 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 39626 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0012 sec  2.85 GBytes  2.45 Gbits/sec
[  4] 0.0000-10.0026 sec  2.86 GBytes  2.45 Gbits/sec
[SUM] 0.0000-10.0026 sec  5.71 GBytes  4.90 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.281/0.300/0.318/0.318 ms (tot/err) = 2/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 4
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.3.146 port 46956 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 46978 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 46944 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 46962 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0017 sec  2.85 GBytes  2.45 Gbits/sec
[  4] 0.0000-10.0015 sec  2.85 GBytes  2.45 Gbits/sec
[  5] 0.0000-10.0026 sec  2.85 GBytes  2.45 Gbits/sec
[  6] 0.0000-10.0005 sec  2.85 GBytes  2.45 Gbits/sec
[SUM] 0.0000-10.0005 sec  11.4 GBytes  9.80 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.274/0.312/0.360/0.212 ms (tot/err) = 4/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 8
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  7] local 10.43.3.146 port 35062 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 35058 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 35066 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 35074 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 35038 connected with 10.43.3.145 port 52000
[ 12] local 10.43.3.146 port 35088 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 35048 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 35050 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  4] 0.0000-10.0005 sec  2.85 GBytes  2.44 Gbits/sec
[  8] 0.0000-10.0011 sec  2.85 GBytes  2.45 Gbits/sec
[  5] 0.0000-10.0000 sec  2.85 GBytes  2.45 Gbits/sec
[ 12] 0.0000-10.0021 sec  2.85 GBytes  2.44 Gbits/sec
[  3] 0.0000-10.0003 sec  2.85 GBytes  2.44 Gbits/sec
[  7] 0.0000-10.0065 sec  2.50 GBytes  2.14 Gbits/sec
[  9] 0.0000-10.0077 sec  2.52 GBytes  2.16 Gbits/sec
[  6] 0.0000-10.0003 sec  2.85 GBytes  2.44 Gbits/sec
[SUM] 0.0000-10.0003 sec  22.1 GBytes  19.0 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.096/0.226/0.339/0.109 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 16
[  3] local 10.43.3.146 port 49540 connected with 10.43.3.145 port 52000
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  6] local 10.43.3.146 port 49554 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 49584 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 49552 connected with 10.43.3.145 port 52000
[ 20] local 10.43.3.146 port 49626 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 49606 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 49596 connected with 10.43.3.145 port 52000
[ 10] local 10.43.3.146 port 49604 connected with 10.43.3.145 port 52000
[ 26] local 10.43.3.146 port 49678 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 49556 connected with 10.43.3.145 port 52000
[ 25] local 10.43.3.146 port 49662 connected with 10.43.3.145 port 52000
[ 22] local 10.43.3.146 port 49636 connected with 10.43.3.145 port 52000
[ 11] local 10.43.3.146 port 49612 connected with 10.43.3.145 port 52000
[ 13] local 10.43.3.146 port 49618 connected with 10.43.3.145 port 52000
[ 23] local 10.43.3.146 port 49646 connected with 10.43.3.145 port 52000
[ 15] local 10.43.3.146 port 49688 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[ 11] 0.0000-10.0024 sec  2.28 GBytes  1.95 Gbits/sec
[ 23] 0.0000-10.0022 sec  2.28 GBytes  1.95 Gbits/sec
[ 20] 0.0000-10.0010 sec  2.28 GBytes  1.95 Gbits/sec
[  8] 0.0000-10.0032 sec  2.28 GBytes  1.95 Gbits/sec
[ 26] 0.0000-10.0038 sec  2.28 GBytes  1.95 Gbits/sec
[ 10] 0.0000-10.0002 sec  2.28 GBytes  1.95 Gbits/sec
[  7] 0.0000-10.0033 sec  2.28 GBytes  1.95 Gbits/sec
[ 15] 0.0000-10.0015 sec  2.27 GBytes  1.95 Gbits/sec
[  4] 0.0000-10.0028 sec  2.28 GBytes  1.95 Gbits/sec
[  6] 0.0000-10.0012 sec  2.28 GBytes  1.96 Gbits/sec
[ 13] 0.0000-10.0030 sec  2.28 GBytes  1.95 Gbits/sec
[ 25] 0.0000-10.0051 sec  2.28 GBytes  1.95 Gbits/sec
[  5] 0.0000-10.0001 sec  2.28 GBytes  1.96 Gbits/sec
[  9] 0.0000-10.0017 sec  2.28 GBytes  1.95 Gbits/sec
[ 22] 0.0000-10.0008 sec  2.27 GBytes  1.95 Gbits/sec
[  3] 0.0000-10.0033 sec  2.28 GBytes  1.95 Gbits/sec
[SUM] 0.0000-10.0034 sec  36.4 GBytes  31.3 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.105/0.217/0.401/0.093 ms (tot/err) = 16/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 16
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 48902 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 52777 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 42911 connected with 10.43.3.145 port 52000
[ 11] local 10.43.3.146 port 56354 connected with 10.43.3.145 port 52000
[ 15] local 10.43.3.146 port 43325 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 37041 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 58828 connected with 10.43.3.145 port 52000
[ 17] local 10.43.3.146 port 48858 connected with 10.43.3.145 port 52000
[ 13] local 10.43.3.146 port 49256 connected with 10.43.3.145 port 52000
[ 16] local 10.43.3.146 port 35652 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 48567 connected with 10.43.3.145 port 52000
[ 18] local 10.43.3.146 port 47394 connected with 10.43.3.145 port 52000
[ 19] local 10.43.3.146 port 48065 connected with 10.43.3.145 port 52000
[ 10] local 10.43.3.146 port 39788 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 46818 connected with 10.43.3.145 port 52000
[ 14] local 10.43.3.146 port 57174 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[ 14] 0.0000-10.0002 sec  2.30 GBytes  1.98 Gbits/sec
[  6] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[  5] 0.0000-10.0002 sec  2.31 GBytes  1.98 Gbits/sec
[  8] 0.0000-10.0001 sec  2.31 GBytes  1.98 Gbits/sec
[ 11] 0.0000-10.0003 sec  2.31 GBytes  1.98 Gbits/sec
[ 18] 0.0000-10.0003 sec  2.31 GBytes  1.98 Gbits/sec
[  3] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[  4] 0.0000-10.0005 sec  2.30 GBytes  1.98 Gbits/sec
[ 17] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[ 15] 0.0000-10.0005 sec  2.31 GBytes  1.98 Gbits/sec
[ 19] 0.0000-10.0001 sec  2.30 GBytes  1.98 Gbits/sec
[  7] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[ 13] 0.0000-10.0005 sec  2.31 GBytes  1.98 Gbits/sec
[ 10] 0.0000-10.0003 sec  2.30 GBytes  1.98 Gbits/sec
[  9] 0.0000-10.0000 sec  2.31 GBytes  1.98 Gbits/sec
[ 16] 0.0000-10.0002 sec  2.31 GBytes  1.98 Gbits/sec
[SUM] 0.0000-10.0003 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
88.398/101.706/114.726/24.755 ms (tot/err) = 16/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 1
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 49168 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  34.3 GBytes  29.5 Gbits/sec
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 2
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 42096 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 58667 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  4] 0.0000-10.0001 sec  18.4 GBytes  15.8 Gbits/sec
[  3] 0.0000-10.0000 sec  18.5 GBytes  15.9 Gbits/sec
[SUM] 0.0000-10.0001 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
38.155/38.997/39.839/39.839 ms (tot/err) = 2/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 4
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 36100 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 55108 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 41039 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 34868 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  7] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  5] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  3] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  6] 0.0000-10.0001 sec  9.22 GBytes  7.92 Gbits/sec
[SUM] 0.0000-10.0001 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
51.401/53.644/56.015/30.487 ms (tot/err) = 4/0

You can see with rsocket it reaches ~ 30 Gb/s with single stream, while ipoib only 2.5 Gb/s (12 X), ipoib scales with more threads until
~ 32 Gb/s, which is the link limit.

With 100 G env, rsocket also out perform ipoib, see below:

[email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 10.43.48.59 port 40588 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth
[ 3] 0.0000-10.0000 sec 80.7 GBytes 69.4 Gbits/sec [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 2
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 10.43.48.59 port 41813 connected with 10.43.48.58 port 52000 [ 5] local 10.43.48.59 port 60638 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth

[  5] 0.0000-10.0000 sec  48.9 GBytes  42.0 Gbits/sec
[  3] 0.0000-10.0000 sec  49.8 GBytes  42.8 Gbits/sec
[SUM] 0.0000-10.0000 sec  98.7 GBytes  84.8 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 6.962/7.764/8.567/8.567 ms (tot/err) = 2/0 [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 4
[ 6] local 10.43.48.59 port 58086 connected with 10.43.48.58 port 52000 [ 3] local 10.43.48.59 port 49335 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 4] local 10.43.48.59 port 44593 connected with 10.43.48.58 port 52000 [ 5] local 10.43.48.59 port 60464 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth

[  5] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[  4] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[  3] 0.0000-10.0000 sec  28.0 GBytes  24.1 Gbits/sec
[  6] 0.0000-10.0000 sec  28.0 GBytes  24.1 Gbits/sec
[SUM] 0.0000-10.0001 sec   112 GBytes  96.3 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 7.344/9.619/12.199/5.271 ms (tot/err) = 4/0 [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 8

[  3] local 10.43.48.59 port 43020 connected with 10.43.48.58 port 52000
[  7] local 10.43.48.59 port 59720 connected with 10.43.48.58 port 52000
[  4] local 10.43.48.59 port 52547 connected with 10.43.48.58 port 52000
[  8] local 10.43.48.59 port 41712 connected with 10.43.48.58 port 52000

[ 10] local 10.43.48.59 port 53126 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------

[  6] local 10.43.48.59 port 60311 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 44103 connected with 10.43.48.58 port 52000
[  9] local 10.43.48.59 port 49007 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  9] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  8] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  4] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  6] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[ 10] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  7] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  5] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  3] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[SUM] 0.0000-10.0001 sec   112 GBytes  96.3 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
6.942/12.361/18.109/4.872 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 8
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 58176 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 58180 connected with 10.43.48.58 port 52000
[  3] local 10.43.48.59 port 58178 connected with 10.43.48.58 port 52000
[ 10] local 10.43.48.59 port 58226 connected with 10.43.48.58 port 52000
[ 11] local 10.43.48.59 port 58228 connected with 10.43.48.58 port 52000
[  9] local 10.43.48.59 port 58212 connected with 10.43.48.58 port 52000
[  7] local 10.43.48.59 port 58194 connected with 10.43.48.58 port 52000
[  8] local 10.43.48.59 port 58198 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  9] 0.0000-10.0005 sec  15.8 GBytes  13.5 Gbits/sec
[  4] 0.0000-10.0002 sec  15.8 GBytes  13.6 Gbits/sec
[  3] 0.0000-10.0000 sec  15.8 GBytes  13.6 Gbits/sec
[  5] 0.0000-10.0002 sec  15.8 GBytes  13.6 Gbits/sec
[  8] 0.0000-10.0005 sec  7.89 GBytes  6.78 Gbits/sec
[ 10] 0.0000-10.0000 sec  15.8 GBytes  13.6 Gbits/sec
[ 11] 0.0000-10.0014 sec  7.94 GBytes  6.82 Gbits/sec
[  7] 0.0000-10.0009 sec  15.8 GBytes  13.6 Gbits/sec
[SUM] 0.0000-10.0009 sec   111 GBytes  95.1 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.234/0.325/0.406/0.155 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 4
[  3] local 10.43.48.59 port 42548 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 42558 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 42560 connected with 10.43.48.58 port 52000
[  6] local 10.43.48.59 port 42562 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  6] 0.0000-10.0000 sec  27.8 GBytes  23.9 Gbits/sec
[  5] 0.0000-10.0001 sec  27.3 GBytes  23.5 Gbits/sec
[  3] 0.0000-10.0001 sec  27.8 GBytes  23.9 Gbits/sec
[  4] 0.0000-10.0001 sec  27.8 GBytes  23.9 Gbits/sec
[SUM] 0.0000-10.0001 sec   111 GBytes  95.1 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.295/0.340/0.390/0.201 ms (tot/err) = 4/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 2
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 44194 connected with 10.43.48.58 port 52000
[  3] local 10.43.48.59 port 44186 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  48.3 GBytes  41.5 Gbits/sec
[  4] 0.0000-10.0000 sec  41.3 GBytes  35.5 Gbits/sec
[SUM] 0.0000-10.0000 sec  89.7 GBytes  77.0 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.227/0.233/0.240/0.240 ms (tot/err) = 2/0
[email protected]:~$ pbkvm list
 VM  State  PID  Cores  Mem  VNC  Migration
--------------------------------------------

Total: 0 VMs, Running: 0

[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 1
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  3] local 10.43.48.59 port 40364 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  51.2 GBytes  44.0 Gbits/sec

Thanks!

Quoted 5 lines

Thanks,

--
Peter Xu

#39 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Jinpu Wang

Hi Peter, hi Daniel,

Hi, Jinpu,

Thanks for sharing this test results. Sounds like a great news.

What's your plan next? Would it then be worthwhile / possible moving QEMU into that direction? Would that greatly simplify rdma code as Dan mentioned?

Thanks,

Quoted 370 lines · Peter Xu
Quoted 8 lines · Jinpu Wang
Quoted 5 lines

I had a brief check in the rsocket changelog, there seems some improvement over time,
might be worth revisiting this. due to socket abstraction, we can't
use some feature like
ODP, it won't be a small and easy task.

Quoted 5 lines

It'll be good to know whether Dan's suggestion would work first, without rewritting everything yet so far. Not sure whether some perf test could help with the rsocket APIs even without QEMU's involvements (or looking for test data supporting / invalidate such conversions).

I did a quick test with iperf on 100 G environment and 40 G environment, in summary rsocket works pretty well.

iperf tests between 2 hosts with 40 G (IB),
first a few test with different num. of threads on top of ipoib interface, later with preload rsocket on top of same ipoib interface.

[email protected]:~$ iperf -p 52000 -c 10.43.3.145
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 55602 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0001 sec  2.85 GBytes  2.44 Gbits/sec
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 2
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.3.146 port 39640 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 39626 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0012 sec  2.85 GBytes  2.45 Gbits/sec
[  4] 0.0000-10.0026 sec  2.86 GBytes  2.45 Gbits/sec
[SUM] 0.0000-10.0026 sec  5.71 GBytes  4.90 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.281/0.300/0.318/0.318 ms (tot/err) = 2/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 4
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.3.146 port 46956 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 46978 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 46944 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 46962 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0017 sec  2.85 GBytes  2.45 Gbits/sec
[  4] 0.0000-10.0015 sec  2.85 GBytes  2.45 Gbits/sec
[  5] 0.0000-10.0026 sec  2.85 GBytes  2.45 Gbits/sec
[  6] 0.0000-10.0005 sec  2.85 GBytes  2.45 Gbits/sec
[SUM] 0.0000-10.0005 sec  11.4 GBytes  9.80 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.274/0.312/0.360/0.212 ms (tot/err) = 4/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 8
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  7] local 10.43.3.146 port 35062 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 35058 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 35066 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 35074 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 35038 connected with 10.43.3.145 port 52000
[ 12] local 10.43.3.146 port 35088 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 35048 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 35050 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  4] 0.0000-10.0005 sec  2.85 GBytes  2.44 Gbits/sec
[  8] 0.0000-10.0011 sec  2.85 GBytes  2.45 Gbits/sec
[  5] 0.0000-10.0000 sec  2.85 GBytes  2.45 Gbits/sec
[ 12] 0.0000-10.0021 sec  2.85 GBytes  2.44 Gbits/sec
[  3] 0.0000-10.0003 sec  2.85 GBytes  2.44 Gbits/sec
[  7] 0.0000-10.0065 sec  2.50 GBytes  2.14 Gbits/sec
[  9] 0.0000-10.0077 sec  2.52 GBytes  2.16 Gbits/sec
[  6] 0.0000-10.0003 sec  2.85 GBytes  2.44 Gbits/sec
[SUM] 0.0000-10.0003 sec  22.1 GBytes  19.0 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.096/0.226/0.339/0.109 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 16
[  3] local 10.43.3.146 port 49540 connected with 10.43.3.145 port 52000
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  6] local 10.43.3.146 port 49554 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 49584 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 49552 connected with 10.43.3.145 port 52000
[ 20] local 10.43.3.146 port 49626 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 49606 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 49596 connected with 10.43.3.145 port 52000
[ 10] local 10.43.3.146 port 49604 connected with 10.43.3.145 port 52000
[ 26] local 10.43.3.146 port 49678 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 49556 connected with 10.43.3.145 port 52000
[ 25] local 10.43.3.146 port 49662 connected with 10.43.3.145 port 52000
[ 22] local 10.43.3.146 port 49636 connected with 10.43.3.145 port 52000
[ 11] local 10.43.3.146 port 49612 connected with 10.43.3.145 port 52000
[ 13] local 10.43.3.146 port 49618 connected with 10.43.3.145 port 52000
[ 23] local 10.43.3.146 port 49646 connected with 10.43.3.145 port 52000
[ 15] local 10.43.3.146 port 49688 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[ 11] 0.0000-10.0024 sec  2.28 GBytes  1.95 Gbits/sec
[ 23] 0.0000-10.0022 sec  2.28 GBytes  1.95 Gbits/sec
[ 20] 0.0000-10.0010 sec  2.28 GBytes  1.95 Gbits/sec
[  8] 0.0000-10.0032 sec  2.28 GBytes  1.95 Gbits/sec
[ 26] 0.0000-10.0038 sec  2.28 GBytes  1.95 Gbits/sec
[ 10] 0.0000-10.0002 sec  2.28 GBytes  1.95 Gbits/sec
[  7] 0.0000-10.0033 sec  2.28 GBytes  1.95 Gbits/sec
[ 15] 0.0000-10.0015 sec  2.27 GBytes  1.95 Gbits/sec
[  4] 0.0000-10.0028 sec  2.28 GBytes  1.95 Gbits/sec
[  6] 0.0000-10.0012 sec  2.28 GBytes  1.96 Gbits/sec
[ 13] 0.0000-10.0030 sec  2.28 GBytes  1.95 Gbits/sec
[ 25] 0.0000-10.0051 sec  2.28 GBytes  1.95 Gbits/sec
[  5] 0.0000-10.0001 sec  2.28 GBytes  1.96 Gbits/sec
[  9] 0.0000-10.0017 sec  2.28 GBytes  1.95 Gbits/sec
[ 22] 0.0000-10.0008 sec  2.27 GBytes  1.95 Gbits/sec
[  3] 0.0000-10.0033 sec  2.28 GBytes  1.95 Gbits/sec
[SUM] 0.0000-10.0034 sec  36.4 GBytes  31.3 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.105/0.217/0.401/0.093 ms (tot/err) = 16/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 16
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 48902 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 52777 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 42911 connected with 10.43.3.145 port 52000
[ 11] local 10.43.3.146 port 56354 connected with 10.43.3.145 port 52000
[ 15] local 10.43.3.146 port 43325 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 37041 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 58828 connected with 10.43.3.145 port 52000
[ 17] local 10.43.3.146 port 48858 connected with 10.43.3.145 port 52000
[ 13] local 10.43.3.146 port 49256 connected with 10.43.3.145 port 52000
[ 16] local 10.43.3.146 port 35652 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 48567 connected with 10.43.3.145 port 52000
[ 18] local 10.43.3.146 port 47394 connected with 10.43.3.145 port 52000
[ 19] local 10.43.3.146 port 48065 connected with 10.43.3.145 port 52000
[ 10] local 10.43.3.146 port 39788 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 46818 connected with 10.43.3.145 port 52000
[ 14] local 10.43.3.146 port 57174 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[ 14] 0.0000-10.0002 sec  2.30 GBytes  1.98 Gbits/sec
[  6] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[  5] 0.0000-10.0002 sec  2.31 GBytes  1.98 Gbits/sec
[  8] 0.0000-10.0001 sec  2.31 GBytes  1.98 Gbits/sec
[ 11] 0.0000-10.0003 sec  2.31 GBytes  1.98 Gbits/sec
[ 18] 0.0000-10.0003 sec  2.31 GBytes  1.98 Gbits/sec
[  3] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[  4] 0.0000-10.0005 sec  2.30 GBytes  1.98 Gbits/sec
[ 17] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[ 15] 0.0000-10.0005 sec  2.31 GBytes  1.98 Gbits/sec
[ 19] 0.0000-10.0001 sec  2.30 GBytes  1.98 Gbits/sec
[  7] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[ 13] 0.0000-10.0005 sec  2.31 GBytes  1.98 Gbits/sec
[ 10] 0.0000-10.0003 sec  2.30 GBytes  1.98 Gbits/sec
[  9] 0.0000-10.0000 sec  2.31 GBytes  1.98 Gbits/sec
[ 16] 0.0000-10.0002 sec  2.31 GBytes  1.98 Gbits/sec
[SUM] 0.0000-10.0003 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
88.398/101.706/114.726/24.755 ms (tot/err) = 16/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 1
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 49168 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  34.3 GBytes  29.5 Gbits/sec
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 2
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 42096 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 58667 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  4] 0.0000-10.0001 sec  18.4 GBytes  15.8 Gbits/sec
[  3] 0.0000-10.0000 sec  18.5 GBytes  15.9 Gbits/sec
[SUM] 0.0000-10.0001 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
38.155/38.997/39.839/39.839 ms (tot/err) = 2/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 4
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 36100 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 55108 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 41039 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 34868 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  7] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  5] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  3] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  6] 0.0000-10.0001 sec  9.22 GBytes  7.92 Gbits/sec
[SUM] 0.0000-10.0001 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
51.401/53.644/56.015/30.487 ms (tot/err) = 4/0

You can see with rsocket it reaches ~ 30 Gb/s with single stream, while ipoib only 2.5 Gb/s (12 X), ipoib scales with more threads until
~ 32 Gb/s, which is the link limit.

With 100 G env, rsocket also out perform ipoib, see below:

[email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 10.43.48.59 port 40588 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth
[ 3] 0.0000-10.0000 sec 80.7 GBytes 69.4 Gbits/sec [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 2
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 10.43.48.59 port 41813 connected with 10.43.48.58 port 52000 [ 5] local 10.43.48.59 port 60638 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth

[  5] 0.0000-10.0000 sec  48.9 GBytes  42.0 Gbits/sec
[  3] 0.0000-10.0000 sec  49.8 GBytes  42.8 Gbits/sec
[SUM] 0.0000-10.0000 sec  98.7 GBytes  84.8 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 6.962/7.764/8.567/8.567 ms (tot/err) = 2/0 [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 4
[ 6] local 10.43.48.59 port 58086 connected with 10.43.48.58 port 52000 [ 3] local 10.43.48.59 port 49335 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 4] local 10.43.48.59 port 44593 connected with 10.43.48.58 port 52000 [ 5] local 10.43.48.59 port 60464 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth

[  5] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[  4] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[  3] 0.0000-10.0000 sec  28.0 GBytes  24.1 Gbits/sec
[  6] 0.0000-10.0000 sec  28.0 GBytes  24.1 Gbits/sec
[SUM] 0.0000-10.0001 sec   112 GBytes  96.3 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 7.344/9.619/12.199/5.271 ms (tot/err) = 4/0 [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 8

[  3] local 10.43.48.59 port 43020 connected with 10.43.48.58 port 52000
[  7] local 10.43.48.59 port 59720 connected with 10.43.48.58 port 52000
[  4] local 10.43.48.59 port 52547 connected with 10.43.48.58 port 52000
[  8] local 10.43.48.59 port 41712 connected with 10.43.48.58 port 52000

[ 10] local 10.43.48.59 port 53126 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------

[  6] local 10.43.48.59 port 60311 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 44103 connected with 10.43.48.58 port 52000
[  9] local 10.43.48.59 port 49007 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  9] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  8] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  4] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  6] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[ 10] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  7] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  5] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  3] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[SUM] 0.0000-10.0001 sec   112 GBytes  96.3 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
6.942/12.361/18.109/4.872 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 8
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 58176 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 58180 connected with 10.43.48.58 port 52000
[  3] local 10.43.48.59 port 58178 connected with 10.43.48.58 port 52000
[ 10] local 10.43.48.59 port 58226 connected with 10.43.48.58 port 52000
[ 11] local 10.43.48.59 port 58228 connected with 10.43.48.58 port 52000
[  9] local 10.43.48.59 port 58212 connected with 10.43.48.58 port 52000
[  7] local 10.43.48.59 port 58194 connected with 10.43.48.58 port 52000
[  8] local 10.43.48.59 port 58198 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  9] 0.0000-10.0005 sec  15.8 GBytes  13.5 Gbits/sec
[  4] 0.0000-10.0002 sec  15.8 GBytes  13.6 Gbits/sec
[  3] 0.0000-10.0000 sec  15.8 GBytes  13.6 Gbits/sec
[  5] 0.0000-10.0002 sec  15.8 GBytes  13.6 Gbits/sec
[  8] 0.0000-10.0005 sec  7.89 GBytes  6.78 Gbits/sec
[ 10] 0.0000-10.0000 sec  15.8 GBytes  13.6 Gbits/sec
[ 11] 0.0000-10.0014 sec  7.94 GBytes  6.82 Gbits/sec
[  7] 0.0000-10.0009 sec  15.8 GBytes  13.6 Gbits/sec
[SUM] 0.0000-10.0009 sec   111 GBytes  95.1 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.234/0.325/0.406/0.155 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 4
[  3] local 10.43.48.59 port 42548 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 42558 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 42560 connected with 10.43.48.58 port 52000
[  6] local 10.43.48.59 port 42562 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  6] 0.0000-10.0000 sec  27.8 GBytes  23.9 Gbits/sec
[  5] 0.0000-10.0001 sec  27.3 GBytes  23.5 Gbits/sec
[  3] 0.0000-10.0001 sec  27.8 GBytes  23.9 Gbits/sec
[  4] 0.0000-10.0001 sec  27.8 GBytes  23.9 Gbits/sec
[SUM] 0.0000-10.0001 sec   111 GBytes  95.1 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.295/0.340/0.390/0.201 ms (tot/err) = 4/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 2
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 44194 connected with 10.43.48.58 port 52000
[  3] local 10.43.48.59 port 44186 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  48.3 GBytes  41.5 Gbits/sec
[  4] 0.0000-10.0000 sec  41.3 GBytes  35.5 Gbits/sec
[SUM] 0.0000-10.0000 sec  89.7 GBytes  77.0 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.227/0.233/0.240/0.240 ms (tot/err) = 2/0
[email protected]:~$ pbkvm list
 VM  State  PID  Cores  Mem  VNC  Migration
--------------------------------------------

Total: 0 VMs, Running: 0

[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 1
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  3] local 10.43.48.59 port 40364 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  51.2 GBytes  44.0 Gbits/sec

Thanks!

Quoted 5 lines

Thanks,

--
Peter Xu

Signature · 1 line
Peter Xu
#40 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter, hi Daniel,

Quoted 11 lines · Peter Xu
Quoted 1 line · Jinpu Wang

Hi Peter, hi Daniel,

Hi, Jinpu,

Thanks for sharing this test results. Sounds like a great news.

What's your plan next? Would it then be worthwhile / possible moving QEMU into that direction? Would that greatly simplify rdma code as Dan mentioned?

I'm rather not familiar with QEMU migration yet, from the test result, I think it's a possible direction,
just we need to at least based on a rather recent release like rdma-core v33 with proper 'fork' support.

Maybe Dan or you could give more detail about what you have in mind for using rsocket as a replacement for the future.
We will also look into the implementation details in the meantime.

Thx!
J

Quoted 377 lines

Thanks,

Quoted 361 lines · Peter Xu
Quoted 8 lines · Jinpu Wang
Quoted 5 lines

I had a brief check in the rsocket changelog, there seems some improvement over time,
might be worth revisiting this. due to socket abstraction, we can't
use some feature like
ODP, it won't be a small and easy task.

Quoted 5 lines

It'll be good to know whether Dan's suggestion would work first, without rewritting everything yet so far. Not sure whether some perf test could help with the rsocket APIs even without QEMU's involvements (or looking for test data supporting / invalidate such conversions).

I did a quick test with iperf on 100 G environment and 40 G environment, in summary rsocket works pretty well.

iperf tests between 2 hosts with 40 G (IB),
first a few test with different num. of threads on top of ipoib interface, later with preload rsocket on top of same ipoib interface.

[email protected]:~$ iperf -p 52000 -c 10.43.3.145
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 55602 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0001 sec  2.85 GBytes  2.44 Gbits/sec
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 2
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.3.146 port 39640 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 39626 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0012 sec  2.85 GBytes  2.45 Gbits/sec
[  4] 0.0000-10.0026 sec  2.86 GBytes  2.45 Gbits/sec
[SUM] 0.0000-10.0026 sec  5.71 GBytes  4.90 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.281/0.300/0.318/0.318 ms (tot/err) = 2/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 4
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.3.146 port 46956 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 46978 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 46944 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 46962 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0017 sec  2.85 GBytes  2.45 Gbits/sec
[  4] 0.0000-10.0015 sec  2.85 GBytes  2.45 Gbits/sec
[  5] 0.0000-10.0026 sec  2.85 GBytes  2.45 Gbits/sec
[  6] 0.0000-10.0005 sec  2.85 GBytes  2.45 Gbits/sec
[SUM] 0.0000-10.0005 sec  11.4 GBytes  9.80 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.274/0.312/0.360/0.212 ms (tot/err) = 4/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 8
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  7] local 10.43.3.146 port 35062 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 35058 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 35066 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 35074 connected with 10.43.3.145 port 52000
[  3] local 10.43.3.146 port 35038 connected with 10.43.3.145 port 52000
[ 12] local 10.43.3.146 port 35088 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 35048 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 35050 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  4] 0.0000-10.0005 sec  2.85 GBytes  2.44 Gbits/sec
[  8] 0.0000-10.0011 sec  2.85 GBytes  2.45 Gbits/sec
[  5] 0.0000-10.0000 sec  2.85 GBytes  2.45 Gbits/sec
[ 12] 0.0000-10.0021 sec  2.85 GBytes  2.44 Gbits/sec
[  3] 0.0000-10.0003 sec  2.85 GBytes  2.44 Gbits/sec
[  7] 0.0000-10.0065 sec  2.50 GBytes  2.14 Gbits/sec
[  9] 0.0000-10.0077 sec  2.52 GBytes  2.16 Gbits/sec
[  6] 0.0000-10.0003 sec  2.85 GBytes  2.44 Gbits/sec
[SUM] 0.0000-10.0003 sec  22.1 GBytes  19.0 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.096/0.226/0.339/0.109 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.3.145 -P 16
[  3] local 10.43.3.146 port 49540 connected with 10.43.3.145 port 52000
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  6] local 10.43.3.146 port 49554 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 49584 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 49552 connected with 10.43.3.145 port 52000
[ 20] local 10.43.3.146 port 49626 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 49606 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 49596 connected with 10.43.3.145 port 52000
[ 10] local 10.43.3.146 port 49604 connected with 10.43.3.145 port 52000
[ 26] local 10.43.3.146 port 49678 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 49556 connected with 10.43.3.145 port 52000
[ 25] local 10.43.3.146 port 49662 connected with 10.43.3.145 port 52000
[ 22] local 10.43.3.146 port 49636 connected with 10.43.3.145 port 52000
[ 11] local 10.43.3.146 port 49612 connected with 10.43.3.145 port 52000
[ 13] local 10.43.3.146 port 49618 connected with 10.43.3.145 port 52000
[ 23] local 10.43.3.146 port 49646 connected with 10.43.3.145 port 52000
[ 15] local 10.43.3.146 port 49688 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[ 11] 0.0000-10.0024 sec  2.28 GBytes  1.95 Gbits/sec
[ 23] 0.0000-10.0022 sec  2.28 GBytes  1.95 Gbits/sec
[ 20] 0.0000-10.0010 sec  2.28 GBytes  1.95 Gbits/sec
[  8] 0.0000-10.0032 sec  2.28 GBytes  1.95 Gbits/sec
[ 26] 0.0000-10.0038 sec  2.28 GBytes  1.95 Gbits/sec
[ 10] 0.0000-10.0002 sec  2.28 GBytes  1.95 Gbits/sec
[  7] 0.0000-10.0033 sec  2.28 GBytes  1.95 Gbits/sec
[ 15] 0.0000-10.0015 sec  2.27 GBytes  1.95 Gbits/sec
[  4] 0.0000-10.0028 sec  2.28 GBytes  1.95 Gbits/sec
[  6] 0.0000-10.0012 sec  2.28 GBytes  1.96 Gbits/sec
[ 13] 0.0000-10.0030 sec  2.28 GBytes  1.95 Gbits/sec
[ 25] 0.0000-10.0051 sec  2.28 GBytes  1.95 Gbits/sec
[  5] 0.0000-10.0001 sec  2.28 GBytes  1.96 Gbits/sec
[  9] 0.0000-10.0017 sec  2.28 GBytes  1.95 Gbits/sec
[ 22] 0.0000-10.0008 sec  2.27 GBytes  1.95 Gbits/sec
[  3] 0.0000-10.0033 sec  2.28 GBytes  1.95 Gbits/sec
[SUM] 0.0000-10.0034 sec  36.4 GBytes  31.3 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.105/0.217/0.401/0.093 ms (tot/err) = 16/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 16
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 48902 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 52777 connected with 10.43.3.145 port 52000
[  9] local 10.43.3.146 port 42911 connected with 10.43.3.145 port 52000
[ 11] local 10.43.3.146 port 56354 connected with 10.43.3.145 port 52000
[ 15] local 10.43.3.146 port 43325 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 37041 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 58828 connected with 10.43.3.145 port 52000
[ 17] local 10.43.3.146 port 48858 connected with 10.43.3.145 port 52000
[ 13] local 10.43.3.146 port 49256 connected with 10.43.3.145 port 52000
[ 16] local 10.43.3.146 port 35652 connected with 10.43.3.145 port 52000
[  8] local 10.43.3.146 port 48567 connected with 10.43.3.145 port 52000
[ 18] local 10.43.3.146 port 47394 connected with 10.43.3.145 port 52000
[ 19] local 10.43.3.146 port 48065 connected with 10.43.3.145 port 52000
[ 10] local 10.43.3.146 port 39788 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 46818 connected with 10.43.3.145 port 52000
[ 14] local 10.43.3.146 port 57174 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[ 14] 0.0000-10.0002 sec  2.30 GBytes  1.98 Gbits/sec
[  6] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[  5] 0.0000-10.0002 sec  2.31 GBytes  1.98 Gbits/sec
[  8] 0.0000-10.0001 sec  2.31 GBytes  1.98 Gbits/sec
[ 11] 0.0000-10.0003 sec  2.31 GBytes  1.98 Gbits/sec
[ 18] 0.0000-10.0003 sec  2.31 GBytes  1.98 Gbits/sec
[  3] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[  4] 0.0000-10.0005 sec  2.30 GBytes  1.98 Gbits/sec
[ 17] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[ 15] 0.0000-10.0005 sec  2.31 GBytes  1.98 Gbits/sec
[ 19] 0.0000-10.0001 sec  2.30 GBytes  1.98 Gbits/sec
[  7] 0.0000-10.0004 sec  2.31 GBytes  1.98 Gbits/sec
[ 13] 0.0000-10.0005 sec  2.31 GBytes  1.98 Gbits/sec
[ 10] 0.0000-10.0003 sec  2.30 GBytes  1.98 Gbits/sec
[  9] 0.0000-10.0000 sec  2.31 GBytes  1.98 Gbits/sec
[ 16] 0.0000-10.0002 sec  2.31 GBytes  1.98 Gbits/sec
[SUM] 0.0000-10.0003 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
88.398/101.706/114.726/24.755 ms (tot/err) = 16/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 1
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 49168 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  34.3 GBytes  29.5 Gbits/sec
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 2
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 42096 connected with 10.43.3.145 port 52000
[  4] local 10.43.3.146 port 58667 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  4] 0.0000-10.0001 sec  18.4 GBytes  15.8 Gbits/sec
[  3] 0.0000-10.0000 sec  18.5 GBytes  15.9 Gbits/sec
[SUM] 0.0000-10.0001 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
38.155/38.997/39.839/39.839 ms (tot/err) = 2/0
[email protected]:~$
LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p
52000 -c 10.43.3.145 -P 4
------------------------------------------------------------
Client connecting to 10.43.3.145, TCP port 52000
TCP window size:  128 KByte (default)
------------------------------------------------------------
[  3] local 10.43.3.146 port 36100 connected with 10.43.3.145 port 52000
[  5] local 10.43.3.146 port 55108 connected with 10.43.3.145 port 52000
[  6] local 10.43.3.146 port 41039 connected with 10.43.3.145 port 52000
[  7] local 10.43.3.146 port 34868 connected with 10.43.3.145 port 52000
[ ID] Interval       Transfer     Bandwidth
[  7] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  5] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  3] 0.0000-10.0000 sec  9.22 GBytes  7.92 Gbits/sec
[  6] 0.0000-10.0001 sec  9.22 GBytes  7.92 Gbits/sec
[SUM] 0.0000-10.0001 sec  36.9 GBytes  31.7 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
51.401/53.644/56.015/30.487 ms (tot/err) = 4/0

You can see with rsocket it reaches ~ 30 Gb/s with single stream, while ipoib only 2.5 Gb/s (12 X), ipoib scales with more threads until
~ 32 Gb/s, which is the link limit.

With 100 G env, rsocket also out perform ipoib, see below:

[email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 10.43.48.59 port 40588 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth
[ 3] 0.0000-10.0000 sec 80.7 GBytes 69.4 Gbits/sec [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 2
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 10.43.48.59 port 41813 connected with 10.43.48.58 port 52000 [ 5] local 10.43.48.59 port 60638 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth

[  5] 0.0000-10.0000 sec  48.9 GBytes  42.0 Gbits/sec
[  3] 0.0000-10.0000 sec  49.8 GBytes  42.8 Gbits/sec
[SUM] 0.0000-10.0000 sec  98.7 GBytes  84.8 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 6.962/7.764/8.567/8.567 ms (tot/err) = 2/0 [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 4
[ 6] local 10.43.48.59 port 58086 connected with 10.43.48.58 port 52000 [ 3] local 10.43.48.59 port 49335 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 4] local 10.43.48.59 port 44593 connected with 10.43.48.58 port 52000 [ 5] local 10.43.48.59 port 60464 connected with 10.43.48.58 port 52000
[ ID] Interval Transfer Bandwidth

[  5] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[  4] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[  3] 0.0000-10.0000 sec  28.0 GBytes  24.1 Gbits/sec
[  6] 0.0000-10.0000 sec  28.0 GBytes  24.1 Gbits/sec
[SUM] 0.0000-10.0001 sec   112 GBytes  96.3 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 7.344/9.619/12.199/5.271 ms (tot/err) = 4/0 [email protected]:~$ LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -p 52000 -c 10.43.48.58 -P 8

[  3] local 10.43.48.59 port 43020 connected with 10.43.48.58 port 52000
[  7] local 10.43.48.59 port 59720 connected with 10.43.48.58 port 52000
[  4] local 10.43.48.59 port 52547 connected with 10.43.48.58 port 52000
[  8] local 10.43.48.59 port 41712 connected with 10.43.48.58 port 52000

[ 10] local 10.43.48.59 port 53126 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size: 128 KByte (default)
------------------------------------------------------------

[  6] local 10.43.48.59 port 60311 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 44103 connected with 10.43.48.58 port 52000
[  9] local 10.43.48.59 port 49007 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  9] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  8] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  4] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  6] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[ 10] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  7] 0.0000-10.0000 sec  14.0 GBytes  12.0 Gbits/sec
[  5] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[  3] 0.0000-10.0001 sec  14.0 GBytes  12.0 Gbits/sec
[SUM] 0.0000-10.0001 sec   112 GBytes  96.3 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
6.942/12.361/18.109/4.872 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 8
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 58176 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 58180 connected with 10.43.48.58 port 52000
[  3] local 10.43.48.59 port 58178 connected with 10.43.48.58 port 52000
[ 10] local 10.43.48.59 port 58226 connected with 10.43.48.58 port 52000
[ 11] local 10.43.48.59 port 58228 connected with 10.43.48.58 port 52000
[  9] local 10.43.48.59 port 58212 connected with 10.43.48.58 port 52000
[  7] local 10.43.48.59 port 58194 connected with 10.43.48.58 port 52000
[  8] local 10.43.48.59 port 58198 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  9] 0.0000-10.0005 sec  15.8 GBytes  13.5 Gbits/sec
[  4] 0.0000-10.0002 sec  15.8 GBytes  13.6 Gbits/sec
[  3] 0.0000-10.0000 sec  15.8 GBytes  13.6 Gbits/sec
[  5] 0.0000-10.0002 sec  15.8 GBytes  13.6 Gbits/sec
[  8] 0.0000-10.0005 sec  7.89 GBytes  6.78 Gbits/sec
[ 10] 0.0000-10.0000 sec  15.8 GBytes  13.6 Gbits/sec
[ 11] 0.0000-10.0014 sec  7.94 GBytes  6.82 Gbits/sec
[  7] 0.0000-10.0009 sec  15.8 GBytes  13.6 Gbits/sec
[SUM] 0.0000-10.0009 sec   111 GBytes  95.1 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.234/0.325/0.406/0.155 ms (tot/err) = 8/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 4
[  3] local 10.43.48.59 port 42548 connected with 10.43.48.58 port 52000
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 42558 connected with 10.43.48.58 port 52000
[  5] local 10.43.48.59 port 42560 connected with 10.43.48.58 port 52000
[  6] local 10.43.48.59 port 42562 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  6] 0.0000-10.0000 sec  27.8 GBytes  23.9 Gbits/sec
[  5] 0.0000-10.0001 sec  27.3 GBytes  23.5 Gbits/sec
[  3] 0.0000-10.0001 sec  27.8 GBytes  23.9 Gbits/sec
[  4] 0.0000-10.0001 sec  27.8 GBytes  23.9 Gbits/sec
[SUM] 0.0000-10.0001 sec   111 GBytes  95.1 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.295/0.340/0.390/0.201 ms (tot/err) = 4/0
[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 2
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  4] local 10.43.48.59 port 44194 connected with 10.43.48.58 port 52000
[  3] local 10.43.48.59 port 44186 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  48.3 GBytes  41.5 Gbits/sec
[  4] 0.0000-10.0000 sec  41.3 GBytes  35.5 Gbits/sec
[SUM] 0.0000-10.0000 sec  89.7 GBytes  77.0 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.227/0.233/0.240/0.240 ms (tot/err) = 2/0
[email protected]:~$ pbkvm list
 VM  State  PID  Cores  Mem  VNC  Migration
--------------------------------------------

Total: 0 VMs, Running: 0

[email protected]:~$ iperf -p 52000 -c 10.43.48.58 -P 1
------------------------------------------------------------
Client connecting to 10.43.48.58, TCP port 52000
TCP window size:  165 KByte (default)
------------------------------------------------------------
[  3] local 10.43.48.59 port 40364 connected with 10.43.48.58 port 52000
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  51.2 GBytes  44.0 Gbits/sec
Quoted 9 lines

Thanks!

Quoted 5 lines

Thanks,

--
Peter Xu

--
Peter Xu

#41 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 13 lines · Jinpu Wang

Hi Peter, hi Daniel,

Quoted 11 lines · Peter Xu
Quoted 1 line · Jinpu Wang

Hi Peter, hi Daniel,

Hi, Jinpu,

Thanks for sharing this test results. Sounds like a great news.

What's your plan next? Would it then be worthwhile / possible moving QEMU into that direction? Would that greatly simplify rdma code as Dan mentioned?

Quoted 8 lines

I'm rather not familiar with QEMU migration yet, from the test result, I think it's a possible direction,
just we need to at least based on a rather recent release like rdma-core v33 with proper 'fork' support.

Maybe Dan or you could give more detail about what you have in mind for using rsocket as a replacement for the future.
We will also look into the implementation details in the meantime.

The migration/socket.c file is the entrypoint for traditional TCP based migration code. It uses the QIOChannelSocket class which is written against the traditional sockets APIs, and uses the QAPI SocketAddress data type to configure it..

My thought was that potentially SocketAddress could be extended to offer RDMA addressing eg

{ 'union': 'SocketAddress',
'base': { 'type': 'SocketAddressType' },
'discriminator': 'type',
'data': { 'inet': 'InetSocketAddress',

            'unix': 'UnixSocketAddress',
            'vsock': 'VsockSocketAddress',
            'fd': 'FdSocketAddress',
	    'rdma': 'InetSocketAddress' } }

And then QIOChannelSocket could be also extended to call the alternative 'rsockets' APIs where needed. That would mean that existing sockets migration code would almost "just work" with RDMA. Theoreticaly any other part of QEMU using QIOChannelSocket would also then magically support RDMA too, with very little (if any) extra work.

With regards,
Daniel

Signature · 3 lines
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|
#42 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu

Hi, Peter

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Regards,
-Gonglei

Quoted 123 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Wednesday, May 1, 2024 11:31 PM
To: Daniel P. Berrangé <[email protected]>
Cc: Markus Armbruster <[email protected]>; Michael Galaxy <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Gonglei (Arei) <[email protected]>; Pannengyuan <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 41 lines · Daniel P. Berrangé
Quoted 33 lines · Markus Armbruster
Quoted 24 lines · Peter Xu
Quoted 1 line · Michael Galaxy

Hi All (and Peter),

Hi, Michael,

Quoted 14 lines

My name is Michael Galaxy (formerly Hines). Yes, I changed my last name (highly irregular for a male) and yes, that's my real last name:
https://www.linkedin.com/in/mrgalaxy/)

I'm the original author of the RDMA implementation. I've been discussing with Yu Zhang for a little bit about potentially handing over maintainership of the codebase to his team.

I simply have zero access to RoCE or Infiniband hardware at all, unfortunately. so I've never been able to run tests or use what I wrote at work, and as all of you know, if you don't have a way to test something, then you can't maintain it.

Quoted 4 lines

Yu Zhang put a (very kind) proposal forward to me to ask the community if they feel comfortable training his team to maintain the codebase (and run
tests) while they learn about it.

Quoted 6 lines

The "while learning" part is fine at least to me. IMHO the "ownership" to the code, or say, taking over the responsibility, may or may not need 100% mastering the code base first. There should still be some fundamental confidence to work on the code though as a starting point, then it's about serious use case to back this up, and careful testings while getting more familiar with it.

Quoted 6 lines

How much experience we expect of maintainers depends on the subsystem and other circumstances. The hard requirement isn't experience, it's trust. See the recent attack on xz.

I do not mean to express any doubts whatsoever on Yu Zhang's integrity! I'm merely reminding y'all what's at stake.

Quoted 6 lines

I think we shouldn't overly obsess[1] about 'xz', because the overwhealmingly common scenario is that volunteer maintainers are honest people. QEMU is in a massively better peer review situation. With xz there was basically no oversight of the new maintainer. With QEMU, we have oversight from 1000's of people on the list, a huge pool of general maintainers, the specific migration maintainers, and the release

manager merging code.

Quoted 3 lines

With a lack of historical experiance with QEMU maintainership, I'd suggest that new RDMA volunteers would start by adding themselves to the

"MAINTAINERS"

Quoted 4 lines

file with only the 'Reviewer' classification. The main migration maintainers would still handle pull requests, but wait for a R-b from one of the RMDA volunteers. After some period of time the RDMA folks could graduate to full maintainer status if the migration maintainers needed

to reduce their load.

Quoted 2 lines

I suspect that might prove unneccesary though, given RDMA isn't an area of code with a high turnover of patches.

Right, and we can do that as a start, it also follows our normal rules of starting from Reviewers to maintain something. I even considered Zhijian to be the previous rdma goto guy / maintainer no matter what role he used to have in the MAINTAINERS file.

Here IMHO it's more about whether any company would like to stand up and provide help, without yet binding that to be able to send pull requests in the near future or even longer term.

What I worry more is whether this is really what we want to keep rdma in qemu, and that's also why I was trying to request for some serious performance measurements comparing rdma v.s. nics. And here when I said "we" I mean both QEMU community and any company that will support keeping rdma around.

The problem is if NICs now are fast enough to perform at least equally against rdma, and if it has a lower cost of overall maintenance, does it mean that rdma migration will only be used by whoever wants to keep them in the products and existed already? In that case we should simply ask new users to stick with tcp, and rdma users should only drop but not increase.

It seems also destined that most new migration features will not support
rdma: see how much we drop old features in migration now (which rdma _might_ still leverage, but maybe not), and how much we add mostly multifd relevant which will probably not apply to rdma at all. So in general what I am worrying is a both-loss condition, if the company might be easier to either stick with an old qemu (depending on whether other new features are requested to be used besides RDMA alone), or do periodic rebase with RDMA downstream only.

So even if we want to keep RDMA around I hope with this chance we can at least have clear picture on when we should still suggest any new user to use RDMA (with the reasons behind). Or we simply shouldn't suggest any new user to use RDMA at all (because at least it'll lose many new migration features).

Thanks,

--
Peter Xu

#43 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Gonglei (Arei)

Hi, Peter

Hey, Lei,

Happy to see you around again after years.

Quoted 4 lines

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

It's another cost to use offload cards, v.s. preparing more cpu resources?

Quoted 3 lines

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Will any of you upstream that work? I'm curious how intrusive would it be when adding it to multifd, if it can keep only 5 exported functions like what rdma.h does right now it'll be pretty nice. We also want to make sure it works with arbitrary sized loads and buffers, e.g. vfio is considering to add IO loads to multifd channels too.

One thing to note that the question here is not about a pure performance comparison between rdma and nics only. It's about help us make a decision on whether to drop rdma, iow, even if rdma performs well, the community still has the right to drop it if nobody can actively work and maintain it. It's just that if nics can perform as good it's more a reason to drop, unless companies can help to provide good support and work together.

Thanks,

Signature · 1 line
Peter Xu
#44 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hello,

Quoted 23 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Monday, May 6, 2024 11:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Michael Galaxy <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 7 lines
Quoted 1 line · Gonglei (Arei)

Hi, Peter

Hey, Lei,

Happy to see you around again after years.

Haha, me too.

Quoted 7 lines
Quoted 4 lines

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

It's another cost to use offload cards, v.s. preparing more cpu resources?

Software and hardware offload converged architecture is the way to go for all cloud vendors (Including comprehensive benefits in terms of performance, cost, security, and innovation speed), it's not just a matter of adding the resource of a DPU card.

Quoted 10 lines
Quoted 3 lines

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Will any of you upstream that work? I'm curious how intrusive would it be when adding it to multifd, if it can keep only 5 exported functions like what rdma.h does right now it'll be pretty nice. We also want to make sure it works with arbitrary sized loads and buffers, e.g. vfio is considering to add IO loads to multifd channels too.

In fact, we sent the patchset to the community in 2021. Pls see:
https://lore.kernel.org/all/20210203185906.GT2950@work-vm/T/

Quoted 7 lines

One thing to note that the question here is not about a pure performance comparison between rdma and nics only. It's about help us make a decision on whether to drop rdma, iow, even if rdma performs well, the community still has the right to drop it if nobody can actively work and maintain it. It's just that if nics can perform as good it's more a reason to drop, unless companies can help to provide good support and work together.

We are happy to provide the necessary review and maintenance work for RDMA if the community needs it.

CC'ing Chuan Zheng.

Regards,
-Gonglei

#45 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 56 lines · Gonglei (Arei)

Hello,

Quoted 23 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Monday, May 6, 2024 11:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Michael Galaxy <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 7 lines
Quoted 1 line · Gonglei (Arei)

Hi, Peter

Hey, Lei,

Happy to see you around again after years.

Haha, me too.

Quoted 7 lines
Quoted 4 lines

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

It's another cost to use offload cards, v.s. preparing more cpu resources?

Software and hardware offload converged architecture is the way to go for all cloud vendors (Including comprehensive benefits in terms of performance, cost, security, and innovation speed), it's not just a matter of adding the resource of a DPU card.

Quoted 10 lines
Quoted 3 lines

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Will any of you upstream that work? I'm curious how intrusive would it be when adding it to multifd, if it can keep only 5 exported functions like what rdma.h does right now it'll be pretty nice. We also want to make sure it works with arbitrary sized loads and buffers, e.g. vfio is considering to add IO loads to multifd channels too.

Quoted 2 lines

In fact, we sent the patchset to the community in 2021. Pls see:
https://lore.kernel.org/all/20210203185906.GT2950@work-vm/T/

I wasn't aware of that for sure in the past..

Multifd has changed quite a bit in the last 9.0 release, that may not apply anymore. One thing to mention is please look at Dan's comment on possible use of rsocket.h:

https://lore.kernel.org/all/[email protected]/

And Jinpu did help provide an initial test result over the library:

https://lore.kernel.org/qemu-devel/CAMGffEk8wiKNQmoUYxcaTHGtiEm2dwoCF_W7T0vMcD-i30tUkA@mail.gmail.com/

It looks like we have a chance to apply that in QEMU.

Quoted 10 lines
Quoted 9 lines

One thing to note that the question here is not about a pure performance comparison between rdma and nics only. It's about help us make a decision on whether to drop rdma, iow, even if rdma performs well, the community still has the right to drop it if nobody can actively work and maintain it. It's just that if nics can perform as good it's more a reason to drop, unless companies can help to provide good support and work together.

Quoted 4 lines

We are happy to provide the necessary review and maintenance work for RDMA if the community needs it.

CC'ing Chuan Zheng.

I'm not sure whether you and Jinpu's team would like to work together and provide a final solution for rdma over multifd. It could be much simpler than the original 2021 proposal if the rsocket API will work out.

Thanks,

Signature · 1 line
Peter Xu
#46 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi, Peter,Lei,Jinpu.

Quoted 57 lines · Peter Xu
Quoted 56 lines · Gonglei (Arei)

Hello,

Quoted 23 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Monday, May 6, 2024 11:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Michael Galaxy <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 7 lines
Quoted 1 line · Gonglei (Arei)

Hi, Peter

Hey, Lei,

Happy to see you around again after years.

Haha, me too.

Quoted 7 lines
Quoted 4 lines

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

It's another cost to use offload cards, v.s. preparing more cpu resources?

Software and hardware offload converged architecture is the way to go for all cloud vendors (Including comprehensive benefits in terms of performance, cost, security, and innovation speed), it's not just a matter of adding the resource of a DPU card.

Quoted 10 lines
Quoted 3 lines

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Will any of you upstream that work? I'm curious how intrusive would it be when adding it to multifd, if it can keep only 5 exported functions like what rdma.h does right now it'll be pretty nice. We also want to make sure it works with arbitrary sized loads and buffers, e.g. vfio is considering to add IO loads to multifd channels too.

Quoted 3 lines
Quoted 2 lines

In fact, we sent the patchset to the community in 2021. Pls see:
https://lore.kernel.org/all/20210203185906.GT2950@work-vm/T/

Yes, I have sent the patchset of multifd support for rdma migration by taking over my colleague, and also sorry for not keeping on this work at that time due to some reasons.
And also I am strongly agree with Lei that the RDMA protocol has some special advantages against with TCP in some scenario, and we are indeed to use it in our product.

Quoted 27 lines

I wasn't aware of that for sure in the past..

Multifd has changed quite a bit in the last 9.0 release, that may not apply anymore. One thing to mention is please look at Dan's comment on possible use of rsocket.h:

https://lore.kernel.org/all/[email protected]/

And Jinpu did help provide an initial test result over the library:

https://lore.kernel.org/qemu-devel/CAMGffEk8wiKNQmoUYxcaTHGtiEm2dwoCF_W7T0vMcD-i30tUkA@mail.gmail.com/

It looks like we have a chance to apply that in QEMU.

Quoted 13 lines
Quoted 9 lines

One thing to note that the question here is not about a pure performance comparison between rdma and nics only. It's about help us make a decision on whether to drop rdma, iow, even if rdma performs well, the community still has the right to drop it if nobody can actively work and maintain it. It's just that if nics can perform as good it's more a reason to drop, unless companies can help to provide good support and work together.

We are happy to provide the necessary review and maintenance work for RDMA if the community needs it.

Quoted 8 lines
Quoted 1 line

CC'ing Chuan Zheng.

I'm not sure whether you and Jinpu's team would like to work together and provide a final solution for rdma over multifd. It could be much simpler than the original 2021 proposal if the rsocket API will work out.

Thanks,

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Signature · 2 lines
Regards.
Chuan
#47 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 11 lines · Zheng Chuan via

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Based on what we have now, it looks like we'd better halt the deprecation process a bit, so I think we shouldn't need to rush it at least in 9.1 then, and we'll need to see how it goes on the refactoring.

It'll be perfect if rsocket works, otherwise supporting multifd with little overhead / exported APIs would also be a good thing in general with whatever approach. And obviously all based on the facts that we can get resources from companies to support this feature first.

Note that so far nobody yet compared with rdma v.s. nic perf, so I hope if any of us can provide some test results please do so. Many people are saying RDMA is better, but I yet didn't see any numbers comparing it with modern TCP networks. I don't want to have old impressions floating around even if things might have changed.. When we have consolidated results, we should share them out and also reflect that in QEMU's migration docs when a rdma document page is ready.

Chuan, please check the whole thread discussion, it may help to understand what we are looking for on rdma migrations [1]. Meanwhile please feel free to sync with Jinpu's team and see how to move forward with such a project.

[1] https://lore.kernel.org/qemu-devel/[email protected]/

Thanks,

Signature · 1 line
Peter Xu
#48 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter, Hi Chuan,

Quoted 23 lines · Peter Xu
Quoted 11 lines · Zheng Chuan via

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Based on what we have now, it looks like we'd better halt the deprecation process a bit, so I think we shouldn't need to rush it at least in 9.1 then, and we'll need to see how it goes on the refactoring.

It'll be perfect if rsocket works, otherwise supporting multifd with little overhead / exported APIs would also be a good thing in general with whatever approach. And obviously all based on the facts that we can get resources from companies to support this feature first.

Quoted 7 lines

Note that so far nobody yet compared with rdma v.s. nic perf, so I hope if any of us can provide some test results please do so. Many people are saying RDMA is better, but I yet didn't see any numbers comparing it with modern TCP networks. I don't want to have old impressions floating around even if things might have changed.. When we have consolidated results, we should share them out and also reflect that in QEMU's migration docs when a rdma document page is ready.

I also did a tests with Mellanox ConnectX-6 100 G RoCE nic, the results are mixed, for less than 3 streams native ethernet is faster, and when more than 3 streams rsocket performs better.

root@x4-right:~# iperf -c 1.1.1.16 -P 1
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size:  325 KByte (default)
------------------------------------------------------------
[  3] local 1.1.1.15 port 44214 connected with 1.1.1.16 port 5001
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  52.9 GBytes  45.4 Gbits/sec
root@x4-right:~# iperf -c 1.1.1.16 -P 2
[  3] local 1.1.1.15 port 33118 connected with 1.1.1.16 port 5001
[  4] local 1.1.1.15 port 33130 connected with 1.1.1.16 port 5001
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 4.00 MByte (default)
------------------------------------------------------------
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0001 sec  45.0 GBytes  38.7 Gbits/sec
[  4] 0.0000-10.0000 sec  43.9 GBytes  37.7 Gbits/sec
[SUM] 0.0000-10.0000 sec  88.9 GBytes  76.4 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.172/0.189/0.205/0.172 ms (tot/err) = 2/0
root@x4-right:~# iperf -c 1.1.1.16 -P 4
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size:  325 KByte (default)
------------------------------------------------------------
[  5] local 1.1.1.15 port 50748 connected with 1.1.1.16 port 5001
[  4] local 1.1.1.15 port 50734 connected with 1.1.1.16 port 5001
[  6] local 1.1.1.15 port 50764 connected with 1.1.1.16 port 5001
[  3] local 1.1.1.15 port 50730 connected with 1.1.1.16 port 5001
[ ID] Interval       Transfer     Bandwidth
[  6] 0.0000-10.0000 sec  24.7 GBytes  21.2 Gbits/sec
[  3] 0.0000-10.0004 sec  23.6 GBytes  20.3 Gbits/sec
[  4] 0.0000-10.0000 sec  27.8 GBytes  23.9 Gbits/sec
[  5] 0.0000-10.0000 sec  28.0 GBytes  24.0 Gbits/sec
[SUM] 0.0000-10.0000 sec   104 GBytes  89.4 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.104/0.156/0.204/0.124 ms (tot/err) = 4/0
root@x4-right:~# iperf -c 1.1.1.16 -P 8
[  4] local 1.1.1.15 port 55588 connected with 1.1.1.16 port 5001
[  5] local 1.1.1.15 port 55600 connected with 1.1.1.16 port 5001
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size:  325 KByte (default)
------------------------------------------------------------
[ 10] local 1.1.1.15 port 55628 connected with 1.1.1.16 port 5001
[ 15] local 1.1.1.15 port 55648 connected with 1.1.1.16 port 5001
[  7] local 1.1.1.15 port 55620 connected with 1.1.1.16 port 5001
[  3] local 1.1.1.15 port 55584 connected with 1.1.1.16 port 5001
[ 14] local 1.1.1.15 port 55644 connected with 1.1.1.16 port 5001
[  6] local 1.1.1.15 port 55610 connected with 1.1.1.16 port 5001
[ ID] Interval       Transfer     Bandwidth
[  6] 0.0000-10.0015 sec  8.47 GBytes  7.27 Gbits/sec
[  4] 0.0000-10.0011 sec  8.62 GBytes  7.40 Gbits/sec
[  7] 0.0000-10.0000 sec  18.1 GBytes  15.5 Gbits/sec
[ 14] 0.0000-10.0000 sec  8.69 GBytes  7.46 Gbits/sec
[  5] 0.0000-10.0006 sec  18.5 GBytes  15.9 Gbits/sec
[ 10] 0.0000-10.0006 sec  16.1 GBytes  13.9 Gbits/sec
[  3] 0.0000-10.0000 sec  17.1 GBytes  14.6 Gbits/sec
[ 15] 0.0000-10.0016 sec  8.54 GBytes  7.34 Gbits/sec
[SUM] 0.0000-10.0017 sec   104 GBytes  89.4 Gbits/sec
[ CT] final connect times (min/avg/max/stdev) =
0.049/0.095/0.213/0.062 ms (tot/err) = 8/0

root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 1
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 1.1.1.15 port 45596 connected with 1.1.1.16 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0000-10.0000 sec 37.8 GBytes 32.5 Gbits/sec root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 2
[ 4] local 1.1.1.15 port 46782 connected with 1.1.1.16 port 5001
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 1.1.1.15 port 43237 connected with 1.1.1.16 port 5001
[ ID] Interval Transfer Bandwidth

[  4] 0.0000-10.0000 sec  37.5 GBytes  32.2 Gbits/sec
[  3] 0.0000-10.0000 sec  40.7 GBytes  34.9 Gbits/sec
[SUM] 0.0000-10.0000 sec  78.2 GBytes  67.2 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 5.819/6.579/7.340/7.340 ms (tot/err) = 2/0
root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 4

[  4] local 1.1.1.15 port 60385 connected with 1.1.1.16 port 5001
[  7] local 1.1.1.15 port 55203 connected with 1.1.1.16 port 5001
[  6] local 1.1.1.15 port 35084 connected with 1.1.1.16 port 5001

------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 1.1.1.15 port 37253 connected with 1.1.1.16 port 5001
[ ID] Interval Transfer Bandwidth

[  6] 0.0000-10.0000 sec  28.4 GBytes  24.4 Gbits/sec
[  4] 0.0000-10.0000 sec  28.3 GBytes  24.3 Gbits/sec
[  7] 0.0000-10.0000 sec  28.4 GBytes  24.4 Gbits/sec
[  3] 0.0000-10.0001 sec  28.2 GBytes  24.3 Gbits/sec
[SUM] 0.0000-10.0001 sec   113 GBytes  97.3 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 5.311/7.579/10.019/4.165 ms (tot/err) = 4/0 root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 8
[ 8] local 1.1.1.15 port 33684 connected with 1.1.1.16 port 5001 [ 10] local 1.1.1.15 port 40620 connected with 1.1.1.16 port 5001
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 1.1.1.15 port 56988 connected with 1.1.1.16 port 5001 [ 4] local 1.1.1.15 port 51139 connected with 1.1.1.16 port 5001 [ 12] local 1.1.1.15 port 44712 connected with 1.1.1.16 port 5001

[  5] local 1.1.1.15 port 50838 connected with 1.1.1.16 port 5001
[  6] local 1.1.1.15 port 51334 connected with 1.1.1.16 port 5001
[  9] local 1.1.1.15 port 40611 connected with 1.1.1.16 port 5001
[ ID] Interval       Transfer     Bandwidth
[  3] 0.0000-10.0000 sec  13.8 GBytes  11.9 Gbits/sec
[  5] 0.0000-10.0001 sec  13.9 GBytes  11.9 Gbits/sec
[ 12] 0.0000-10.0001 sec  13.8 GBytes  11.9 Gbits/sec
[ 10] 0.0000-10.0001 sec  13.9 GBytes  11.9 Gbits/sec
[  9] 0.0000-10.0000 sec  13.8 GBytes  11.9 Gbits/sec
[  6] 0.0000-10.0000 sec  13.9 GBytes  11.9 Gbits/sec
[  8] 0.0000-10.0000 sec  13.8 GBytes  11.9 Gbits/sec
[  4] 0.0000-10.0001 sec  13.8 GBytes  11.9 Gbits/sec
[SUM] 0.0000-10.0001 sec   111 GBytes  95.1 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 5.973/10.699/15.943/4.251 ms (tot/err) = 8/0 root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 1
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 3] local 1.1.1.15 port 36960 connected with 1.1.1.16 port 5001
[ ID] Interval Transfer Bandwidth
[ 3] 0.0000-10.0000 sec 41.1 GBytes 35.3 Gbits/sec root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 2
[ 3] local 1.1.1.15 port 32799 connected with 1.1.1.16 port 5001
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------
[ 4] local 1.1.1.15 port 35912 connected with 1.1.1.16 port 5001
[ ID] Interval Transfer Bandwidth

[  4] 0.0000-10.0000 sec  36.6 GBytes  31.4 Gbits/sec
[  3] 0.0000-10.0000 sec  36.6 GBytes  31.4 Gbits/sec
[SUM] 0.0000-10.0000 sec  73.2 GBytes  62.9 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) = 5.172/5.842/6.512/6.512 ms (tot/err) = 2/0
root@x4-right:~# LD_PRELOAD=/usr/lib/x86_64-linux-gnu/rsocket/librspreload.so iperf -c 1.1.1.16 -P 4
[ 4] local 1.1.1.15 port 53311 connected with 1.1.1.16 port 5001
------------------------------------------------------------
Client connecting to 1.1.1.16, TCP port 5001
TCP window size: 128 KByte (default)
------------------------------------------------------------

[  3] local 1.1.1.15 port 37243 connected with 1.1.1.16 port 5001
[  7] local 1.1.1.15 port 60801 connected with 1.1.1.16 port 5001
[  6] local 1.1.1.15 port 49694 connected with 1.1.1.16 port 5001
[ ID] Interval       Transfer     Bandwidth
[  6] 0.0000-10.0000 sec  28.2 GBytes  24.2 Gbits/sec
[  7] 0.0000-10.0000 sec  28.2 GBytes  24.3 Gbits/sec
[  3] 0.0000-10.0000 sec  28.2 GBytes  24.2 Gbits/sec
[  4] 0.0000-10.0000 sec  28.2 GBytes  24.2 Gbits/sec
[SUM] 0.0000-10.0000 sec   113 GBytes  96.9 Gbits/sec

[ CT] final connect times (min/avg/max/stdev) =
5.570/7.762/10.045/4.265 ms (tot/err) = 4/0
root@x4-right:~#

Quoted 4 lines

Chuan, please check the whole thread discussion, it may help to understand what we are looking for on rdma migrations [1]. Meanwhile please feel free to sync with Jinpu's team and see how to move forward with such a project.

We are happy to work with community to improve rdma migration.

Quoted 4 lines

[1] https://lore.kernel.org/qemu-devel/[email protected]/

Thanks,

Regards!

Quoted 4 lines

--
Peter Xu

#49 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu

Hello Peter and all,

I did a comparison of the VM live-migration speeds between RDMA and TCP/IP on our servers
and plotted the results to get an initial impression. Unfortunately, the Ethernet NICs are not the
recent ones, therefore, it may not make much sense. I can do it on servers with more recent Ethernet
NICs and keep you updated.

It seems that the benefits of RDMA becomes obviously when the VM has large memory and is
running memory-intensive workload.

Best regards,
Yu Zhang @ IONOS Cloud

Quoted 42 lines · Peter Xu
Quoted 11 lines · Zheng Chuan via

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Based on what we have now, it looks like we'd better halt the deprecation process a bit, so I think we shouldn't need to rush it at least in 9.1 then, and we'll need to see how it goes on the refactoring.

It'll be perfect if rsocket works, otherwise supporting multifd with little overhead / exported APIs would also be a good thing in general with whatever approach. And obviously all based on the facts that we can get resources from companies to support this feature first.

Note that so far nobody yet compared with rdma v.s. nic perf, so I hope if any of us can provide some test results please do so. Many people are saying RDMA is better, but I yet didn't see any numbers comparing it with modern TCP networks. I don't want to have old impressions floating around even if things might have changed.. When we have consolidated results, we should share them out and also reflect that in QEMU's migration docs when a rdma document page is ready.

Chuan, please check the whole thread discussion, it may help to understand what we are looking for on rdma migrations [1]. Meanwhile please feel free to sync with Jinpu's team and see how to move forward with such a project.

[1] https://lore.kernel.org/qemu-devel/[email protected]/

Thanks,

--
Peter Xu

#50 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

These are very compelling results, no?

(40gbps cards, right? Are the cards active/active? or active/standby?)

- Michael

Quoted 58 lines · Yu Zhang

Hello Peter and all,

I did a comparison of the VM live-migration speeds between RDMA and TCP/IP on our servers
and plotted the results to get an initial impression. Unfortunately, the Ethernet NICs are not the
recent ones, therefore, it may not make much sense. I can do it on servers with more recent Ethernet
NICs and keep you updated.

It seems that the benefits of RDMA becomes obviously when the VM has large memory and is
running memory-intensive workload.

Best regards,
Yu Zhang @ IONOS Cloud

Quoted 40 lines · Peter Xu
Quoted 11 lines · Zheng Chuan via

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Based on what we have now, it looks like we'd better halt the deprecation process a bit, so I think we shouldn't need to rush it at least in 9.1 then, and we'll need to see how it goes on the refactoring.

It'll be perfect if rsocket works, otherwise supporting multifd with little overhead / exported APIs would also be a good thing in general with whatever approach. And obviously all based on the facts that we can get resources from companies to support this feature first.

Note that so far nobody yet compared with rdma v.s. nic perf, so I hope if any of us can provide some test results please do so. Many people are saying RDMA is better, but I yet didn't see any numbers comparing it with modern TCP networks. I don't want to have old impressions floating around even if things might have changed.. When we have consolidated results, we should share them out and also reflect that in QEMU's migration docs when a rdma document page is ready.

Chuan, please check the whole thread discussion, it may help to understand what we are looking for on rdma migrations [1]. Meanwhile please feel free to sync with Jinpu's team and see how to move forward with such a project.

[1] https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/[email protected]/__;!!GjvTz_vk!QnXDo1zSlYecz7JvJky4SOQ9I8V5MoGHbINdAQAzMJQ_yYg_8_BSUXz9kjvbSgFefhG0wi1j38KaC3g$

Thanks,

--
Peter Xu

#51 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hello Michael and Peter,

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe
InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more sense.

Best regards,
Yu Zhang @ IONOS Cloud

Quoted 66 lines · Michael Galaxy

These are very compelling results, no?

(40gbps cards, right? Are the cards active/active? or active/standby?)

- Michael

Quoted 58 lines · Yu Zhang

Hello Peter and all,

I did a comparison of the VM live-migration speeds between RDMA and TCP/IP on our servers
and plotted the results to get an initial impression. Unfortunately, the Ethernet NICs are not the
recent ones, therefore, it may not make much sense. I can do it on servers with more recent Ethernet
NICs and keep you updated.

It seems that the benefits of RDMA becomes obviously when the VM has large memory and is
running memory-intensive workload.

Best regards,
Yu Zhang @ IONOS Cloud

Quoted 40 lines · Peter Xu
Quoted 11 lines · Zheng Chuan via

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Based on what we have now, it looks like we'd better halt the deprecation process a bit, so I think we shouldn't need to rush it at least in 9.1 then, and we'll need to see how it goes on the refactoring.

It'll be perfect if rsocket works, otherwise supporting multifd with little overhead / exported APIs would also be a good thing in general with whatever approach. And obviously all based on the facts that we can get resources from companies to support this feature first.

Note that so far nobody yet compared with rdma v.s. nic perf, so I hope if any of us can provide some test results please do so. Many people are saying RDMA is better, but I yet didn't see any numbers comparing it with modern TCP networks. I don't want to have old impressions floating around even if things might have changed.. When we have consolidated results, we should share them out and also reflect that in QEMU's migration docs when a rdma document page is ready.

Chuan, please check the whole thread discussion, it may help to understand what we are looking for on rdma migrations [1]. Meanwhile please feel free to sync with Jinpu's team and see how to move forward with such a project.

[1] https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/[email protected]/__;!!GjvTz_vk!QnXDo1zSlYecz7JvJky4SOQ9I8V5MoGHbINdAQAzMJQ_yYg_8_BSUXz9kjvbSgFefhG0wi1j38KaC3g$

Thanks,

--
Peter Xu

#52 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Yu Zhang

Hello Michael and Peter,

Hi,

Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more reasonable.

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15wVAqk81vXtKzx-LfJQ@mail.gmail.com/

Appreciate a lot for everyone helping on the testings.

Quoted 11 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

Thanks,

Signature · 1 line
Peter Xu
#53 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Peter,

Quoted 55 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Wednesday, May 22, 2024 6:15 AM
To: Yu Zhang <[email protected]>
Cc: Michael Galaxy <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; zhengchuan <[email protected]>; Gonglei (Arei) <[email protected]>; Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>; Fabiano Rosas <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Yu Zhang

Hello Michael and Peter,

Hi,

Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more reasonable.

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15
[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 5 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Progress:

1.  Implement io/channel-rdma.c,
2.  Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful,
3.  Remove the original code from migration/rdma.c,
4.  Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic,
5.  Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the core logic of live migration),
6.  The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

We will be submit the patchset later.

Regards,
-Gonglei

Quoted 4 lines

Thanks,

--
Peter Xu

#54 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Gonglei,

Quoted 71 lines · Gonglei (Arei)

Hi Peter,

Quoted 55 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Wednesday, May 22, 2024 6:15 AM
To: Yu Zhang <[email protected]>
Cc: Michael Galaxy <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; zhengchuan <[email protected]>; Gonglei (Arei) <[email protected]>; Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>; Fabiano Rosas <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Yu Zhang

Hello Michael and Peter,

Hi,

Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more reasonable.

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15
[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 5 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Progress:

Quoted 9 lines
1.  Implement io/channel-rdma.c,
2.  Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful,
3.  Remove the original code from migration/rdma.c,
4.  Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic,
5.  Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the core logic of live migration),
6.  The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

We will be submit the patchset later.

Thanks for working on this PoC, and sharing progress on this, we are looking forward for the patchset.

Quoted 3 lines

Regards,
-Gonglei

Regards!
Jinpu

Quoted 6 lines
Quoted 5 lines

Thanks,

--
Peter Xu

#55 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Gonglei (Arei) via
Quoted 70 lines · Gonglei (Arei)

Hi Peter,

Quoted 55 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Wednesday, May 22, 2024 6:15 AM
To: Yu Zhang <[email protected]>
Cc: Michael Galaxy <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; zhengchuan <[email protected]>; Gonglei (Arei) <[email protected]>; Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>; Fabiano Rosas <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 1 line · Yu Zhang

Hello Michael and Peter,

Hi,

Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more reasonable.

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15
[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 5 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Progress:

Quoted 8 lines
1.  Implement io/channel-rdma.c,
2.  Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful,
3.  Remove the original code from migration/rdma.c,
4.  Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic,
5.  Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the core logic of live migration),
6.  The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

We will be submit the patchset later.

That's great news, thank you!

Signature · 1 line
Peter Xu
#56 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi,

Quoted 54 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15

Quoted 29 lines
Quoted 17 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 6 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 7 lines

Progress:
1. Implement io/channel-rdma.c,
2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 1 line
Quoted 6 lines
Quoted 1 line

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use rdma_cm to establish a connection, and then use verbs to transmit data.

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API). And cannot listen to the verbs fd event. Only some hacking methods can be used to address this problem.

Do you guys have any ideas? Thanks.

Regards,
-Gonglei

#57 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 70 lines · Gonglei (Arei)

Hi,

Quoted 54 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15

Quoted 29 lines
Quoted 17 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 6 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 7 lines

Progress:
1. Implement io/channel-rdma.c,
2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 1 line
Quoted 6 lines
Quoted 1 line

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use rdma_cm to establish a connection, and then use verbs to transmit data.

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 6 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API). And cannot listen to the verbs fd event. Only some hacking methods can be used to address this problem.

Do you guys have any ideas? Thanks.

+cc linux-rdma
+cc Sean

Quoted 4 lines

Regards,
-Gonglei

#58 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 72 lines · Jinpu Wang
Quoted 70 lines · Gonglei (Arei)

Hi,

Quoted 54 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15

Quoted 29 lines
Quoted 17 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 6 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 7 lines

Progress:
1. Implement io/channel-rdma.c,
2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 1 line
Quoted 6 lines
Quoted 1 line

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use rdma_cm to establish a connection, and then use verbs to transmit data.

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 7 lines
Quoted 6 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API). And cannot listen to the verbs fd event. Only some hacking methods can be used to address this problem.

Do you guys have any ideas? Thanks.

+cc linux-rdma

Why include rdma community?

Quoted 9 lines

+cc Sean

Quoted 4 lines

Regards,
-Gonglei

#59 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 79 lines · Greg Sword
Quoted 77 lines · Jinpu Wang
Quoted 75 lines · Gonglei (Arei)

Hi,

Quoted 54 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme BCM5720 2-port Gigabit Ethernet PCIe

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda15

Quoted 29 lines
Quoted 17 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other is active.

Quoted 6 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev 01)

The comparison between RDMA and TCP on the same NIC could make more

Quoted 9 lines

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored using rsocket.

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 7 lines

Progress:
1. Implement io/channel-rdma.c,
2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 1 line
Quoted 6 lines
Quoted 1 line

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use rdma_cm to establish a connection, and then use verbs to transmit data.

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API). And cannot listen to the verbs fd event. Only some hacking methods can be used to address this problem.

Quoted 4 lines
Quoted 2 lines
Quoted 1 line

Do you guys have any ideas? Thanks.

+cc linux-rdma

Why include rdma community?

rdma community has a lot people with experience in rdma/rsocket?

Quoted 10 lines
Quoted 10 lines

+cc Sean

Quoted 4 lines

Regards,
-Gonglei

#60 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Greg Sword
Quoted 103 lines

-----Original Message-----
From: Greg Sword [mailto:[email protected]]
Sent: Wednesday, May 29, 2024 2:06 PM
To: Jinpu Wang <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 2 lines · Jinpu Wang

On Wed, May 29, 2024 at 4:43 AM Gonglei (Arei) <[email protected]>

Quoted 14 lines · wrote:
Quoted 14 lines

Hi,

Quoted 11 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme
BCM5720 2-port Gigabit Ethernet PCIe

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

15

Quoted 10 lines
Quoted 10 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 6 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other

is active.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev
01)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines

The comparison between RDMA and TCP on the same NIC could make more

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored

using rsocket.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 7 lines
Quoted 4 lines

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 8 lines

Progress:
1. Implement io/channel-rdma.c, 2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 1 line

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit data.

Quoted 4 lines
Quoted 4 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 4 lines
Quoted 4 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link

setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API).

Quoted 8 lines
Quoted 1 line
Quoted 1 line

And cannot listen to the verbs fd event. Only some hacking methods can be

used to address this problem.

Quoted 3 lines
Quoted 2 lines

Do you guys have any ideas? Thanks.

+cc linux-rdma

Why include rdma community?

Can rdma/rsocket provide an API to expose the verbs fd?

Regards,
-Gonglei

Quoted 9 lines
Quoted 9 lines

+cc Sean

Quoted 4 lines

Regards,
-Gonglei

#61 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Gonglei,

Quoted 97 lines · Gonglei (Arei)
Quoted 97 lines

-----Original Message-----
From: Greg Sword [mailto:[email protected]]
Sent: Wednesday, May 29, 2024 2:06 PM
To: Jinpu Wang <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 2 lines · Jinpu Wang

On Wed, May 29, 2024 at 4:43 AM Gonglei (Arei) <[email protected]>

Quoted 14 lines · wrote:
Quoted 14 lines

Hi,

Quoted 11 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme
BCM5720 2-port Gigabit Ethernet PCIe

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines

15

Quoted 10 lines
Quoted 10 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 6 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other

is active.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev
01)

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 2 lines

The comparison between RDMA and TCP on the same NIC could make more

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored

using rsocket.

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines
Quoted 7 lines
Quoted 4 lines

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 8 lines

Progress:
1. Implement io/channel-rdma.c, 2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 1 line

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit data.

Quoted 4 lines
Quoted 4 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 3 lines
Quoted 3 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link

setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API).

Quoted 1 line
Quoted 1 line

And cannot listen to the verbs fd event.

I'm confused, the rs_poll_arm :https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/rsocket.c#L3290 For STREAM, rpoll setup fd for both cq fd and cm fd.

Quoted 8 lines
Quoted 6 lines
Quoted 3 lines
Quoted 2 lines

Do you guys have any ideas? Thanks.

+cc linux-rdma

Why include rdma community?

Can rdma/rsocket provide an API to expose the verbs fd?

Why do we need verbs fd? looks rsocket during rsend/rrecv is handling the new completion if any via rs_get_comp

Another question to my mind is Daniel suggested a bit different way of using rsocket: https://lore.kernel.org/qemu-devel/[email protected]/
Have you considered that?

Thx!
Jinpu

Quoted 14 lines

Regards,
-Gonglei

Quoted 9 lines
Quoted 9 lines

+cc Sean

Quoted 4 lines

Regards,
-Gonglei

#62 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 154 lines

-----Original Message-----
From: Jinpu Wang [mailto:[email protected]]
Sent: Wednesday, May 29, 2024 5:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Greg Sword <[email protected]>; Peter Xu <[email protected]>; Yu Zhang <[email protected]>; Michael Galaxy <[email protected]>; Elmar Gerdes <[email protected]>; zhengchuan <[email protected]>; Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>; Fabiano Rosas <[email protected]>; RDMA mailing list <[email protected]>; [email protected]; Haris Iqbal <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi Gonglei,

Quoted 30 lines · Gonglei (Arei)
Quoted 30 lines

-----Original Message-----
From: Greg Sword [mailto:[email protected]]
Sent: Wednesday, May 29, 2024 2:06 PM
To: Jinpu Wang <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 3 lines · Jinpu Wang

On Wed, May 29, 2024 at 4:43 AM Gonglei (Arei)
<[email protected]>

Quoted 14 lines · wrote:
Quoted 14 lines

Hi,

Quoted 11 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme
BCM5720 2-port Gigabit Ethernet PCIe

Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda

Quoted 67 lines
Quoted 67 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

15

Quoted 4 lines
Quoted 4 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other

is active.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev
01)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

The comparison between RDMA and TCP on the same NIC could make more

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored

using rsocket.

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 7 lines
Quoted 4 lines

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 8 lines

Progress:
1. Implement io/channel-rdma.c, 2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 2 lines

We will be submit the patchset later.

Quoted 7 lines
Quoted 7 lines
Quoted 4 lines

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit data.

Quoted 5 lines
Quoted 5 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link

setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd

(gotten by raccept API).

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

And cannot listen to the verbs fd event.

I'm confused, the rs_poll_arm :https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/rsocket.c# L3290
For STREAM, rpoll setup fd for both cq fd and cm fd.

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

Do you guys have any ideas? Thanks.

Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
Quoted 1 line

+cc linux-rdma

Why include rdma community?

Can rdma/rsocket provide an API to expose the verbs fd?

Why do we need verbs fd? looks rsocket during rsend/rrecv is handling the new completion if any via rs_get_comp

Actually I said the reason in the previous mail. Listing some header in librdmacm.

/* verbs.h */

struct ibv_comp_channel {
	struct ibv_context     *context;
	int			fd;
	int			refcnt;
};

/* rdma_cma.h */

struct rdma_event_channel {
	int			fd;
};

/* rdma_cma.h */

struct rdma_cm_id {
	struct ibv_context	*verbs;
	struct rdma_event_channel *channel;   //==> it can be gotten by rsocket.h
	void			*context;
	struct ibv_qp		*qp;
	struct rdma_route	 route;
	enum rdma_port_space	 ps;
	uint8_t			 port_num;
	struct rdma_cm_event	*event;
	struct ibv_comp_channel *send_cq_channel;  // ==> can't be gotten so that Qemu can't read the CQE data
	struct ibv_cq		*send_cq;
	struct ibv_comp_channel *recv_cq_channel;
	struct ibv_cq		*recv_cq;
	struct ibv_srq		*srq;
	struct ibv_pd		*pd;
	enum ibv_qp_type	qp_type;
};

/* rsocket.h */
int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen); int rpoll(struct pollfd *fds, nfds_t nfds, int timeout);

Quoted 4 lines

Another question to my mind is Daniel suggested a bit different way of using
rsocket: https://lore.kernel.org/qemu-devel/[email protected]/
Have you considered that?

We do use 'rsocket' APIs to refactor the RDMA code in QEMU and encounter the issue.

Regards,
-Gonglei

#63 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 199 lines · Gonglei (Arei)
Quoted 19 lines

-----Original Message-----
From: Jinpu Wang [mailto:[email protected]]
Sent: Wednesday, May 29, 2024 5:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Greg Sword <[email protected]>; Peter Xu <[email protected]>; Yu Zhang <[email protected]>; Michael Galaxy <[email protected]>; Elmar Gerdes <[email protected]>; zhengchuan <[email protected]>; Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song

Quoted 8 lines

Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>; Fabiano Rosas <[email protected]>; RDMA mailing list <[email protected]>; [email protected]; Haris Iqbal <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol

handling

Quoted 36 lines

Hi Gonglei,

On Wed, May 29, 2024 at 10:31 AM Gonglei (Arei) <[email protected]

Quoted 30 lines · wrote:
Quoted 30 lines

-----Original Message-----
From: Greg Sword [mailto:[email protected]]
Sent: Wednesday, May 29, 2024 2:06 PM
To: Jinpu Wang <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 3 lines · Jinpu Wang

On Wed, May 29, 2024 at 4:43 AM Gonglei (Arei)
<[email protected]>

Quoted 14 lines · wrote:
Quoted 14 lines

Hi,

Quoted 11 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Tuesday, May 28, 2024 11:55 PM

Quoted 8 lines
Quoted 8 lines
Quoted 7 lines

Exactly, not so compelling, as I did it first only on servers widely used for production in our data center. The network adapters are

Ethernet controller: Broadcom Inc. and subsidiaries NetXtreme
BCM5720 2-port Gigabit Ethernet PCIe

Quoted 48 lines
Quoted 7 lines
Quoted 7 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

Hmm... I definitely thinks Jinpu's Mellanox ConnectX-6 looks more

reasonable.

Quoted 2 lines
Quoted 2 lines

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda

Quoted 40 lines
Quoted 40 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

15

Quoted 4 lines
Quoted 4 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other

is active.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev
01)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

The comparison between RDMA and TCP on the same NIC could make more

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored

using rsocket.

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 4 lines

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Progress:
1. Implement io/channel-rdma.c, 2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from

migration/rdma.c, 4.

Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Quoted 4 lines

Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's

successful.

Quoted 11 lines
Quoted 11 lines
Quoted 11 lines
Quoted 10 lines
Quoted 10 lines
Quoted 7 lines
Quoted 2 lines

We will be submit the patchset later.

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit

data.

Quoted 9 lines
Quoted 9 lines
Quoted 9 lines
Quoted 5 lines
Quoted 5 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is

generated.

Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket

interfaces.

Quoted 10 lines
Quoted 6 lines
Quoted 6 lines
Quoted 2 lines
Quoted 2 lines

However, this library returns only the rdma_cm fd for listening to link

setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd

(gotten by raccept API).

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

And cannot listen to the verbs fd event.

I'm confused, the rs_poll_arm
:

https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/rsocket.c#

Quoted 4 lines

L3290
For STREAM, rpoll setup fd for both cq fd and cm fd.

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 7 lines
Quoted 5 lines
Quoted 2 lines
Quoted 1 line

Do you guys have any ideas? Thanks.

+cc linux-rdma

Why include rdma community?

Can rdma/rsocket provide an API to expose the verbs fd?

Why do we need verbs fd? looks rsocket during rsend/rrecv is handling

the new

Quoted 2 lines

completion if any via rs_get_comp

Actually I said the reason in the previous mail. Listing some header in librdmacm.

/* verbs.h */

struct ibv_comp_channel {
        struct ibv_context     *context;
        int                     fd;
        int                     refcnt;
};

/* rdma_cma.h */

struct rdma_event_channel {
        int                     fd;
};

/* rdma_cma.h */

struct rdma_cm_id {
        struct ibv_context      *verbs;
        struct rdma_event_channel *channel;   //==> it can be gotten by
rsocket.h
        void                    *context;
        struct ibv_qp           *qp;
        struct rdma_route        route;
        enum rdma_port_space     ps;
        uint8_t                  port_num;
        struct rdma_cm_event    *event;
        struct ibv_comp_channel *send_cq_channel;  // ==> can't be gotten
Quoted 2 lines

so that Qemu can't read the CQE dat

ok, but the send_cq_channel is set the same as recv_cq_channel:
https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/rsocket.c#L855
and also use the same recv_cq as send_cq.

Quoted 26 lines
        struct ibv_cq           *send_cq;
        struct ibv_comp_channel *recv_cq_channel;
        struct ibv_cq           *recv_cq;
        struct ibv_srq          *srq;
        struct ibv_pd           *pd;
        enum ibv_qp_type        qp_type;
};

/* rsocket.h */
int raccept(int socket, struct sockaddr *addr, socklen_t *addrlen); int rpoll(struct pollfd *fds, nfds_t nfds, int timeout);

Quoted 1 line

Another question to my mind is Daniel suggested a bit different way of

using

Quoted 3 lines

rsocket: https://lore.kernel.org/qemu-devel/[email protected]/
Have you considered that?

We do use 'rsocket' APIs to refactor the RDMA code in QEMU and encounter the issue.

Regards,
-Gonglei

#64 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Jinpu Wang

Hi,

Quoted 78 lines

-----Original Message-----

Quoted 1 line
Quoted 1 line

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda

Quoted 67 lines
Quoted 67 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

15

Quoted 4 lines
Quoted 4 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other

is active.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev
01)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

The comparison between RDMA and TCP on the same NIC could make more

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored

using rsocket.

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 7 lines
Quoted 4 lines

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 8 lines

Progress:
1. Implement io/channel-rdma.c, 2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 2 lines

We will be submit the patchset later.

Quoted 7 lines
Quoted 7 lines
Quoted 4 lines

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit data.

Quoted 5 lines
Quoted 5 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link

setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd

Quoted 7 lines

(gotten by raccept API).

Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

And cannot listen to the verbs fd event.

I'm confused, the rs_poll_arm :https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/rsocket.c# L3290
For STREAM, rpoll setup fd for both cq fd and cm fd.

Right. But the question is QEMU do not use rpoll but gilb's ppoll. :(

Regards,
-Gonglei

#65 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hello,

I am part of the storage kernel team which develops and maintains the RDMA block storage in IONOS.
We work closely with Jinpu/Yu, and currently I am supporting Jinpu with this Qemu RDMA work.

Quoted 82 lines · Gonglei (Arei) via

Hi,

Quoted 79 lines

-----Original Message-----

Quoted 1 line
Quoted 1 line

https://lore.kernel.org/qemu-devel/CAMGffEn-DKpMZ4tA71MJYdyemg0Zda

Quoted 67 lines
Quoted 67 lines
Quoted 5 lines
Quoted 5 lines
Quoted 5 lines

15

Quoted 4 lines
Quoted 4 lines

[email protected]/

Appreciate a lot for everyone helping on the testings.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

InfiniBand controller: Mellanox Technologies MT27800 Family [ConnectX-5]

which doesn't meet our purpose. I can choose RDMA or TCP for VM migration. RDMA traffic is through InfiniBand and TCP through Ethernet on these two hosts. One is standby while the other

is active.

Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines

Now I'll try on a server with more recent Ethernet and InfiniBand network adapters. One of them has:
BCM57414 NetXtreme-E 10Gb/25Gb RDMA Ethernet Controller (rev
01)

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
Quoted 2 lines

The comparison between RDMA and TCP on the same NIC could make more

sense.

It looks to me NICs are powerful now, but again as I mentioned I don't think it's a reason we need to deprecate rdma, especially if QEMU's rdma migration has the chance to be refactored

using rsocket.

Quoted 20 lines
Quoted 20 lines
Quoted 20 lines
Quoted 7 lines
Quoted 4 lines

Is there anyone who started looking into that direction? Would it make sense we start some PoC now?

My team has finished the PoC refactoring which works well.

Quoted 8 lines

Progress:
1. Implement io/channel-rdma.c, 2. Add unit test tests/unit/test-io-channel-rdma.c and verifying it is successful, 3. Remove the original code from migration/rdma.c, 4. Rewrite the rdma_start_outgoing_migration and rdma_start_incoming_migration logic, 5. Remove all rdma_xxx functions from migration/ram.c. (to prevent RDMA live migration from polluting the

core logic of live migration), 6. The soft-RoCE implemented by software is used to test the RDMA live migration. It's successful.

Quoted 2 lines

We will be submit the patchset later.

Quoted 7 lines
Quoted 7 lines
Quoted 4 lines

That's great news, thank you!

--
Peter Xu

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit data.

Quoted 5 lines
Quoted 5 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 1 line
Quoted 1 line
Quoted 8 lines
Quoted 8 lines
Quoted 4 lines
Quoted 4 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link

setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd

(gotten by raccept API).

Quoted 8 lines
Quoted 6 lines
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line

And cannot listen to the verbs fd event.

I'm confused, the rs_poll_arm :https://github.com/linux-rdma/rdma-core/blob/master/librdmacm/rsocket.c# L3290
For STREAM, rpoll setup fd for both cq fd and cm fd.

Right. But the question is QEMU do not use rpoll but gilb's ppoll. :(

I have a query around this topic. Are the fds used in socket migration polled through ppoll?
If yes, then can someone point out where; I couldn't find that piece of code.

I could only find that sendmsg/send and recvmsg/recv is being used.

Quoted 5 lines

Regards,
-Gonglei

#66 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Jinpu Wang
Quoted 14 lines
Quoted 1 line

For rdma programming, the current mainstream implementation is to use

rdma_cm to establish a connection, and then use verbs to transmit data.

Quoted 4 lines

rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When

poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 4 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link setup-related

Quoted 8 lines

events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API).

Quoted 1 line

And cannot listen to the verbs fd event. Only some hacking methods can be

used to address this problem.

Quoted 2 lines

Do you guys have any ideas? Thanks.

The current rsocket code allows calling rpoll() with non-rsocket fd's, so an app can use rpoll() directly in place of poll(). It may be easiest to add an rppoll() call to rsockets and call that when using RDMA.

In case the easy path isn't feasible:

An extension could allow extracting the actual fd's under an rsocket, in order to allow a user to call poll()/ppoll() directly. But it would be non-trivial.

The 'fd' that represents an rsocket happens to be the fd related to the RDMA CM. That's because an rsocket needs a unique integer value to report as an 'fd' value which will not conflict with any other fd value that the app may have. I would consider the fd value an implementation detail, rather than something which an app should depend upon. (For example, the 'fd' value returned for a datagram rsocket is actually a UDP socket fd).

Once an rsocket is in the connected state, it's possible an extended rgetsockopt() or rfcntl() call could return the fd related to the CQ. But if an app tried to call poll() on that fd, the results would not be as expected. For example, it's possible for data to be available to receive on the rsocket without the CQ fd being signaled. Calling poll() on the CQ fd in this state could leave the app hanging. This is a natural? result of races in the RDMA CQ signaling. If you look at the rsocket rpoll() implementation, you'll see that it checks for data prior to sleeping.

For an app to safely wait in poll/ppoll on the CQ fd, it would need to invoke some sort of 'pre-poll' routine, which would perform the same checks done in rpoll() prior to blocking. As a reference to a similar pre-poll routine, see the fi_trywait() call from this man page:

https://ofiwg.github.io/libfabric/v1.21.0/man/fi_poll.3.html

This is for a different library but deals with the same underlying problem. Obviously adding an rtrywait() to rsockets is possible but wouldn't align with any socket API equivalent.

- Sean

#67 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Gonglei (Arei) via

Lei,

Quoted 10 lines · Gonglei (Arei)

For rdma programming, the current mainstream implementation is to use rdma_cm to establish a connection, and then use verbs to transmit data. rdma_cm and ibverbs create two FDs respectively. The two FDs have different responsibilities. rdma_cm fd is used to notify connection establishment events, and verbs fd is used to notify new CQEs. When poll/epoll monitoring is directly performed on the rdma_cm fd, only a pollin event can be monitored, which means that an rdma_cm event occurs. When the verbs fd is directly polled/epolled, only the pollin event can be listened, which indicates that a new CQE is generated.

Quoted 9 lines

Rsocket is a sub-module attached to the rdma_cm library and provides rdma calls that are completely similar to socket interfaces. However, this library returns only the rdma_cm fd for listening to link setup-related events and does not expose the verbs fd (readable and writable events for listening to data). Only the rpoll interface provided by the RSocket can be used to listen to related events. However, QEMU uses the ppoll interface to listen to the rdma_cm fd (gotten by raccept API). And cannot listen to the verbs fd event. Only some hacking methods can be used to address this problem. Do you guys have any ideas? Thanks.

I saw that you mentioned this elsewhere:

Quoted 1 line

Right. But the question is QEMU do not use rpoll but gilb's ppoll. :(

So what I'm thinking may not make much sense, as I mentioned I don't think I know rdma at all.. and my idea also has involvement on coroutine stuff which I also don't know well. But just in case it shed some light in some form.

IIUC we do iochannel blockings with this no matter for read/write:

        if (len == QIO_CHANNEL_ERR_BLOCK) {
            if (qemu_in_coroutine()) {
                qio_channel_yield(ioc, G_IO_XXX);
            } else {
                qio_channel_wait(ioc, G_IO_XXX);
            }
            continue;
        }

One thing I'm wondering is whether we can provide a new feature bit for qiochannel, e.g., QIO_CHANNEL_FEATURE_POLL, so that the iochannel can define its own poll routine rather than using the default when possible.

I think it may not work if it's in a coroutine, as I guess that'll block other fds from being waked up. Hence it should look like this:

        if (len == QIO_CHANNEL_ERR_BLOCK) {
            if (qemu_in_coroutine()) {
                qio_channel_yield(ioc, G_IO_XXX);
            } else if (qio_channel_has_feature(ioc, QIO_CHANNEL_FEATURE_POLL)) {
                qio_channel_poll(ioc, G_IO_XXX);
            } else {
                qio_channel_wait(ioc, G_IO_XXX);
            }
            continue;
        }

Maybe we even want to forbid such channel to be used in coroutine already, as when QIO_CHANNEL_FEATURE_POLL set it may mean that this iochannel simply won't work with poll() like in rdma's use case.

Then rdma iochannel can implement qio_channel_poll() using rpoll().

There's one other dependent issue here in that I _think_ the migration recv side is still in a coroutine.. so we may need to move that into a thread first. IIRC we don't yet have a major blocker to do that, but I didn't further check either. I've put that issue aside just to see whether this may or may not make sense.

Thanks,

Signature · 1 line
Peter Xu
#68 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Zheng Chuan via

One thing to keep in mind here (despite me not having any hardware to test) was that one of the original goals here in the RDMA implementation was not simply raw throughput nor raw latency, but a lack of CPU utilization in kernel space due to the offload. While it is entirely possible that newer hardware w/ TCP might compete, the significant reductions in CPU usage in the TCP/IP stack were a big win at the time.

Just something to consider while you're doing the testing........

- Michael

Quoted 108 lines · Zheng Chuan

Hi, Peter,Lei,Jinpu.

Quoted 47 lines · Peter Xu
Quoted 46 lines · Gonglei (Arei)

Hello,

Quoted 23 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Monday, May 6, 2024 11:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Michael Galaxy <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 6 lines
Quoted 1 line · Gonglei (Arei)

Hi, Peter

Hey, Lei,

Happy to see you around again after years.

Haha, me too.

Quoted 6 lines
Quoted 4 lines

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

It's another cost to use offload cards, v.s. preparing more cpu resources?

Software and hardware offload converged architecture is the way to go for all cloud vendors (Including comprehensive benefits in terms of performance, cost, security, and innovation speed), it's not just a matter of adding the resource of a DPU card.

Quoted 3 lines
Quoted 3 lines

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Will any of you upstream that work? I'm curious how intrusive would it be when adding it to multifd, if it can keep only 5 exported functions like what rdma.h does right now it'll be pretty nice. We also want to make sure it works with arbitrary sized loads and buffers, e.g. vfio is considering to add IO loads to multifd channels too.

In fact, we sent the patchset to the community in 2021. Pls see:
https://urldefense.com/v3/__https://lore.kernel.org/all/20210203185906.GT2950@work-vm/T/__;!!GjvTz_vk!VfP_SV-8uRya7rBdopv8OUJkmnSi44Ktpqq1E7sr_Xcwt6zvveW51qboWOBSTChdUG1hJwfAl7HZl4NUEGc$

Yes, I have sent the patchset of multifd support for rdma migration by taking over my colleague, and also sorry for not keeping on this work at that time due to some reasons.
And also I am strongly agree with Lei that the RDMA protocol has some special advantages against with TCP in some scenario, and we are indeed to use it in our product.

Quoted 25 lines

I wasn't aware of that for sure in the past..

Multifd has changed quite a bit in the last 9.0 release, that may not apply anymore. One thing to mention is please look at Dan's comment on possible use of rsocket.h:

https://urldefense.com/v3/__https://lore.kernel.org/all/[email protected]/__;!!GjvTz_vk!VfP_SV-8uRya7rBdopv8OUJkmnSi44Ktpqq1E7sr_Xcwt6zvveW51qboWOBSTChdUG1hJwfAl7HZ0CFSE-o$

And Jinpu did help provide an initial test result over the library:

https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/CAMGffEk8wiKNQmoUYxcaTHGtiEm2dwoCF_W7T0vMcD-i30tUkA@mail.gmail.com/__;!!GjvTz_vk!VfP_SV-8uRya7rBdopv8OUJkmnSi44Ktpqq1E7sr_Xcwt6zvveW51qboWOBSTChdUG1hJwfAl7HZxPNcdb4$

It looks like we have a chance to apply that in QEMU.

Quoted 11 lines
Quoted 8 lines

One thing to note that the question here is not about a pure performance comparison between rdma and nics only. It's about help us make a decision on whether to drop rdma, iow, even if rdma performs well, the community still has the right to drop it if nobody can actively work and maintain it. It's just that if nics can perform as good it's more a reason to drop, unless companies can help to provide good support and work together.

We are happy to provide the necessary review and maintenance work for RDMA if the community needs it.

Quoted 7 lines
Quoted 1 line

CC'ing Chuan Zheng.

I'm not sure whether you and Jinpu's team would like to work together and provide a final solution for rdma over multifd. It could be much simpler than the original 2021 proposal if the rsocket API will work out.

Thanks,

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

#69 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 9 lines · * Michael Galaxy ([email protected]) wrote:

One thing to keep in mind here (despite me not having any hardware to test) was that one of the original goals here
in the RDMA implementation was not simply raw throughput nor raw latency, but a lack of CPU utilization in kernel
space due to the offload. While it is entirely possible that newer hardware w/ TCP might compete, the significant
reductions in CPU usage in the TCP/IP stack were a big win at the time.

Just something to consider while you're doing the testing........

I just noticed this thread; some random notes from a somewhat fragmented memory of this:

a) Long long ago, I also tried rsocket;
https://lists.gnu.org/archive/html/qemu-devel/2015-01/msg02040.html
as I remember the library was quite flaky at the time.

b) A lot of the complexity in the rdma migration code comes from emulating a stream to carry the migration control data and interleaving
that with the actual RAM copy. I believe the original design used
a separate TCP socket for the control data, and just used the RDMA for the data - that should be a lot simpler (but alas was rejected in review early on)

c) I can't rememmber the last benchmarks I did; but I think I did manage to beat RDMA with multifd; but yes, multifd does eat host CPU where as RDMA barely uses a whisper.

d) The 'zero-copy-send' option in migrate may well get some of that CPU time back; but if I remember we were still bottle necked on the receive side. (I can't remember if zero-copy-send worked with multifd?)

e) Someone made a good suggestion (sorry can't remember who) - that the RDMA migration structure was the wrong way around - it should be the destination which initiates an RDMA read, rather than the source doing a write; then things might become a LOT simpler; you just need to send page ranges to the destination and it can pull it. That might work nicely for postcopy.

Dave

Quoted 112 lines

- Michael

Quoted 108 lines · Zheng Chuan

Hi, Peter,Lei,Jinpu.

Quoted 47 lines · Peter Xu
Quoted 46 lines · Gonglei (Arei)

Hello,

Quoted 23 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Monday, May 6, 2024 11:18 PM
To: Gonglei (Arei) <[email protected]>
Cc: Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Michael Galaxy <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 6 lines
Quoted 1 line · Gonglei (Arei)

Hi, Peter

Hey, Lei,

Happy to see you around again after years.

Haha, me too.

Quoted 6 lines
Quoted 4 lines

RDMA features high bandwidth, low latency (in non-blocking lossless network), and direct remote memory access by bypassing the CPU (As you know, CPU resources are expensive for cloud vendors, which is one of the reasons why we introduced offload cards.), which TCP does not have.

It's another cost to use offload cards, v.s. preparing more cpu resources?

Software and hardware offload converged architecture is the way to go for all cloud vendors (Including comprehensive benefits in terms of performance, cost, security, and innovation speed), it's not just a matter of adding the resource of a DPU card.

Quoted 3 lines
Quoted 3 lines

In some scenarios where fast live migration is needed (extremely short interruption duration and migration duration) is very useful. To this end, we have also developed RDMA support for multifd.

Quoted 8 lines
Quoted 8 lines
Quoted 6 lines

Will any of you upstream that work? I'm curious how intrusive would it be when adding it to multifd, if it can keep only 5 exported functions like what rdma.h does right now it'll be pretty nice. We also want to make sure it works with arbitrary sized loads and buffers, e.g. vfio is considering to add IO loads to multifd channels too.

In fact, we sent the patchset to the community in 2021. Pls see:
https://urldefense.com/v3/__https://lore.kernel.org/all/20210203185906.GT2950@work-vm/T/__;!!GjvTz_vk!VfP_SV-8uRya7rBdopv8OUJkmnSi44Ktpqq1E7sr_Xcwt6zvveW51qboWOBSTChdUG1hJwfAl7HZl4NUEGc$

Yes, I have sent the patchset of multifd support for rdma migration by taking over my colleague, and also sorry for not keeping on this work at that time due to some reasons.
And also I am strongly agree with Lei that the RDMA protocol has some special advantages against with TCP in some scenario, and we are indeed to use it in our product.

Quoted 25 lines

I wasn't aware of that for sure in the past..

Multifd has changed quite a bit in the last 9.0 release, that may not apply anymore. One thing to mention is please look at Dan's comment on possible use of rsocket.h:

https://urldefense.com/v3/__https://lore.kernel.org/all/[email protected]/__;!!GjvTz_vk!VfP_SV-8uRya7rBdopv8OUJkmnSi44Ktpqq1E7sr_Xcwt6zvveW51qboWOBSTChdUG1hJwfAl7HZ0CFSE-o$

And Jinpu did help provide an initial test result over the library:

https://urldefense.com/v3/__https://lore.kernel.org/qemu-devel/CAMGffEk8wiKNQmoUYxcaTHGtiEm2dwoCF_W7T0vMcD-i30tUkA@mail.gmail.com/__;!!GjvTz_vk!VfP_SV-8uRya7rBdopv8OUJkmnSi44Ktpqq1E7sr_Xcwt6zvveW51qboWOBSTChdUG1hJwfAl7HZxPNcdb4$

It looks like we have a chance to apply that in QEMU.

Quoted 11 lines
Quoted 8 lines

One thing to note that the question here is not about a pure performance comparison between rdma and nics only. It's about help us make a decision on whether to drop rdma, iow, even if rdma performs well, the community still has the right to drop it if nobody can actively work and maintain it. It's just that if nics can perform as good it's more a reason to drop, unless companies can help to provide good support and work together.

We are happy to provide the necessary review and maintenance work for RDMA if the community needs it.

Quoted 7 lines
Quoted 1 line

CC'ing Chuan Zheng.

I'm not sure whether you and Jinpu's team would like to work together and provide a final solution for rdma over multifd. It could be much simpler than the original 2021 proposal if the rsocket API will work out.

Thanks,

That's a good news to see the socket abstraction for RDMA!
When I was developed the series above, the most pain is the RDMA migration has no QIOChannel abstraction and i need to take a 'fake channel' for it which is awkward in code implementation.
So, as far as I know, we can do this by
i. the first thing is that we need to evaluate the rsocket is good enough to satisfy our QIOChannel fundamental abstraction ii. if it works right, then we will continue to see if it can give us opportunity to hide the detail of rdma protocol
into rsocket by remove most of code in rdma.c and also some hack in migration main process.
iii. implement the advanced features like multi-fd and multi-uri for rdma migration.

Since I am not familiar with rsocket, I need some times to look at it and do some quick verify with rdma migration based on rsocket. But, yes, I am willing to involved in this refactor work and to see if we can make this migration feature more better:)

Signature · 4 lines
 -----Open up your eyes, open up your mind, open up your code -------   
/ Dr. David Alan Gilbert    |       Running GNU/Linux       | Happy  \ 
\        dave @ treblig.org |                               | In Hex /
 \ _________________________|_____ http://www.treblig.org   |_______/
#70 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hey, Dave!

Quoted 9 lines · Dr. David Alan Gilbert
Quoted 8 lines · * Michael Galaxy ([email protected]) wrote:

One thing to keep in mind here (despite me not having any hardware to test) was that one of the original goals here
in the RDMA implementation was not simply raw throughput nor raw latency, but a lack of CPU utilization in kernel
space due to the offload. While it is entirely possible that newer hardware w/ TCP might compete, the significant
reductions in CPU usage in the TCP/IP stack were a big win at the time.

Quoted 8 lines
Quoted 1 line

Just something to consider while you're doing the testing........

I just noticed this thread; some random notes from a somewhat fragmented memory of this:

a) Long long ago, I also tried rsocket;
https://lists.gnu.org/archive/html/qemu-devel/2015-01/msg02040.html
as I remember the library was quite flaky at the time.

Hmm interesting. There also looks like a thread doing rpoll().

Btw, not sure whether you noticed, but there's the series posted for the latest rsocket conversion here:

https://lore.kernel.org/r/[email protected]

I hope Lei and his team has tested >4G mem, otherwise definitely worth checking. Lei also mentioned there're rsocket bugs they found in the cover letter, but not sure what's that about.

Quoted 11 lines

b) A lot of the complexity in the rdma migration code comes from emulating a stream to carry the migration control data and interleaving
that with the actual RAM copy. I believe the original design used
a separate TCP socket for the control data, and just used the RDMA for the data - that should be a lot simpler (but alas was rejected in review early on)

c) I can't rememmber the last benchmarks I did; but I think I did manage to beat RDMA with multifd; but yes, multifd does eat host CPU where as RDMA barely uses a whisper.

I think my first impression on this matter came from you on this one. :)

Quoted 5 lines

d) The 'zero-copy-send' option in migrate may well get some of that CPU time back; but if I remember we were still bottle necked on the receive side. (I can't remember if zero-copy-send worked with multifd?)

Yes, and zero-copy requires multifd for now. I think it's because we didn't want to complicate the header processings in the migration stream where it may not be page aligned.

Quoted 7 lines

e) Someone made a good suggestion (sorry can't remember who) - that the RDMA migration structure was the wrong way around - it should be the destination which initiates an RDMA read, rather than the source doing a write; then things might become a LOT simpler; you just need to send page ranges to the destination and it can pull it. That might work nicely for postcopy.

I'm not sure whether it'll still be a problem if rdma recv side is based on zero-copy. It would be a matter of whether atomicity can be guaranteed so that we don't want the guest vcpus to see a partially copied page during on-flight DMAs. UFFDIO_COPY (or friend) is currently the only solution for that.

Thanks,

Signature · 1 line
Peter Xu
#71 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 7 lines · Peter Xu
Quoted 6 lines

e) Someone made a good suggestion (sorry can't remember who) - that the RDMA migration structure was the wrong way around - it should be the destination which initiates an RDMA read, rather than the source doing a write; then things might become a LOT simpler; you just need to send page ranges to the destination and it can pull it. That might work nicely for postcopy.

Quoted 5 lines

I'm not sure whether it'll still be a problem if rdma recv side is based on zero-copy. It would be a matter of whether atomicity can be guaranteed so that we don't want the guest vcpus to see a partially copied page during on-flight DMAs. UFFDIO_COPY (or friend) is currently the only solution for that.

And when thinking about this (of UFFDIO_COPY's nature on not being able to do zero-copy...), the only way this will be able to do zerocopy is to use file memories (shmem/hugetlbfs), as page cache can be prepopulated. So that when we do DMA we pass over the page cache, which can be mapped in another virtual address besides what the vcpus are using.

Then we can use UFFDIO_CONTINUE (rather than UFFDIO_COPY) to do atomic updates on the vcpu pgtables, avoiding the copy. QEMU doesn't have it, but it looks like there's one more reason we may want to have better use of shmem.. than anonymous. And actually when working on 4k faults on 1G hugetlb I added CONTINUE support.

https://github.com/xzpeter/qemu/tree/doublemap
https://github.com/xzpeter/qemu/commit/b8aff3a9d7654b1cf2c089a06894ff4899740dc5

Maybe it's worthwhile on its own now, because it also means we can use that in multifd to avoid one extra layer of buffering when supporting multifd+postcopy (which has the same issue here on directly copying data into guest pages). It'll also work with things like rmda I think in similar ways. It's just that it'll not work on anonymous.

I definitely hijacked the thread to somewhere too far away. I'll stop here..

Thanks,

Signature · 1 line
Peter Xu
#72 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu
Quoted 1 line · * Peter Xu ([email protected]) wrote:

Hey, Dave!

Hey!

Quoted 12 lines · Dr. David Alan Gilbert
Quoted 11 lines · * Michael Galaxy ([email protected]) wrote:
Quoted 9 lines

One thing to keep in mind here (despite me not having any hardware to test) was that one of the original goals here
in the RDMA implementation was not simply raw throughput nor raw latency, but a lack of CPU utilization in kernel
space due to the offload. While it is entirely possible that newer hardware w/ TCP might compete, the significant
reductions in CPU usage in the TCP/IP stack were a big win at the time.

Just something to consider while you're doing the testing........

Quoted 8 lines
Quoted 6 lines

I just noticed this thread; some random notes from a somewhat fragmented memory of this:

a) Long long ago, I also tried rsocket;
https://lists.gnu.org/archive/html/qemu-devel/2015-01/msg02040.html
as I remember the library was quite flaky at the time.

Hmm interesting. There also looks like a thread doing rpoll().

Yeh, I can't actually remember much more about what I did back then!

Quoted 4 lines

Btw, not sure whether you noticed, but there's the series posted for the latest rsocket conversion here:

https://lore.kernel.org/r/[email protected]

Oh I hadn't; I think all of the stack of qemu's file abstractions had changed in the ~10 years since I wrote my version!

Quoted 3 lines

I hope Lei and his team has tested >4G mem, otherwise definitely worth checking. Lei also mentioned there're rsocket bugs they found in the cover letter, but not sure what's that about.

It would probably be a good idea to keep track of what bugs are in flight with it, and try it on a few RDMA cards to see what problems get triggered.
I think I reported a few at the time, but I gave up after feeling it was getting very hacky.

Quoted 3 lines

Yes, and zero-copy requires multifd for now. I think it's because we didn't want to complicate the header processings in the migration stream where it may not be page aligned.

Ah yes.

Quoted 8 lines
Quoted 7 lines

e) Someone made a good suggestion (sorry can't remember who) - that the RDMA migration structure was the wrong way around - it should be the destination which initiates an RDMA read, rather than the source doing a write; then things might become a LOT simpler; you just need to send page ranges to the destination and it can pull it. That might work nicely for postcopy.

Quoted 5 lines

I'm not sure whether it'll still be a problem if rdma recv side is based on zero-copy. It would be a matter of whether atomicity can be guaranteed so that we don't want the guest vcpus to see a partially copied page during on-flight DMAs. UFFDIO_COPY (or friend) is currently the only solution for that.

Yes, but even ignoring that (and the UFFDIO_CONTINUE idea you mention), if the destination can issue an RDMA read itself, it doesn't need to send messages to the source to ask for a page fetch; it just goes and grabs it itself, that's got to be good for latency.

Dave

Quoted 6 lines

Thanks,

Signature · 1 line
Peter Xu
Signature · 4 lines
 -----Open up your eyes, open up your mind, open up your code -------   
/ Dr. David Alan Gilbert    |       Running GNU/Linux       | Happy  \ 
\        dave @ treblig.org |                               | In Hex /
 \ _________________________|_____ http://www.treblig.org   |_______/
#73 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 10 lines · Dr. David Alan Gilbert
Quoted 8 lines
Quoted 6 lines

I just noticed this thread; some random notes from a somewhat fragmented memory of this:

a) Long long ago, I also tried rsocket;
https://lists.gnu.org/archive/html/qemu-devel/2015-01/msg02040.html
as I remember the library was quite flaky at the time.

Hmm interesting. There also looks like a thread doing rpoll().

Yeh, I can't actually remember much more about what I did back then!

Heh, that's understandable and fair. :)

Quoted 9 lines
Quoted 3 lines

I hope Lei and his team has tested >4G mem, otherwise definitely worth checking. Lei also mentioned there're rsocket bugs they found in the cover letter, but not sure what's that about.

It would probably be a good idea to keep track of what bugs are in flight with it, and try it on a few RDMA cards to see what problems get triggered.
I think I reported a few at the time, but I gave up after feeling it was getting very hacky.

Agreed. Maybe we can have a list of that in the cover letter or even QEMU's migration/rmda doc page.

Lei, if you think that makes sense please do so in your upcoming posts. There'll need to have a list of things you encountered in the kernel driver and it'll be even better if there're further links to read on each problem.

Quoted 14 lines
Quoted 8 lines
Quoted 7 lines

e) Someone made a good suggestion (sorry can't remember who) - that the RDMA migration structure was the wrong way around - it should be the destination which initiates an RDMA read, rather than the source doing a write; then things might become a LOT simpler; you just need to send page ranges to the destination and it can pull it. That might work nicely for postcopy.

Quoted 5 lines

I'm not sure whether it'll still be a problem if rdma recv side is based on zero-copy. It would be a matter of whether atomicity can be guaranteed so that we don't want the guest vcpus to see a partially copied page during on-flight DMAs. UFFDIO_COPY (or friend) is currently the only solution for that.

Quoted 4 lines

Yes, but even ignoring that (and the UFFDIO_CONTINUE idea you mention), if the destination can issue an RDMA read itself, it doesn't need to send messages to the source to ask for a page fetch; it just goes and grabs it itself, that's got to be good for latency.

Oh, that's pretty internal stuff of rdma to me and beyond my knowledge.. but from what I can tell it sounds very reasonable indeed!

Thanks!

Signature · 1 line
Peter Xu
#74 lore raw

RE: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Hi,

Quoted 49 lines

-----Original Message-----
From: Peter Xu [mailto:[email protected]]
Sent: Thursday, June 6, 2024 5:19 AM
To: Dr. David Alan Gilbert <[email protected]>
Cc: Michael Galaxy <[email protected]>; zhengchuan <[email protected]>; Gonglei (Arei) <[email protected]>; Daniel P. Berrangé <[email protected]>; Markus Armbruster <[email protected]>; Yu Zhang <[email protected]>; Zhijian Li (Fujitsu) <[email protected]>; Jinpu Wang <[email protected]>; Elmar Gerdes <[email protected]>; [email protected]; Yuval Shaia <[email protected]>; Kevin Wolf <[email protected]>; Prasanna Kumar Kalever <[email protected]>; Cornelia Huck <[email protected]>; Michael Roth <[email protected]>; Prasanna Kumar Kalever <[email protected]>; [email protected]; Paolo Bonzini <[email protected]>; [email protected]; [email protected]; Hanna Reitz <[email protected]>; Michael S. Tsirkin <[email protected]>; Thomas Huth <[email protected]>; Eric Blake <[email protected]>; Song Gao <[email protected]>; Marc-André Lureau <[email protected]>; Alex Bennée <[email protected]>; Wainer dos Santos Moschetta <[email protected]>; Beraldo Leal <[email protected]>; Pannengyuan <[email protected]>; Xiexiangyou <[email protected]>
Subject: Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

Quoted 5 lines · Dr. David Alan Gilbert
Quoted 5 lines
Quoted 5 lines

I just noticed this thread; some random notes from a somewhat fragmented memory of this:

a) Long long ago, I also tried rsocket;

https://lists.gnu.org/archive/html/qemu-devel/2015-01/msg02040.html

Quoted 5 lines
Quoted 3 lines
Quoted 1 line

as I remember the library was quite flaky at the time.

Hmm interesting. There also looks like a thread doing rpoll().

Yeh, I can't actually remember much more about what I did back then!

Heh, that's understandable and fair. :)

Quoted 9 lines
Quoted 3 lines

I hope Lei and his team has tested >4G mem, otherwise definitely worth checking. Lei also mentioned there're rsocket bugs they found in the cover letter, but not sure what's that about.

It would probably be a good idea to keep track of what bugs are in flight with it, and try it on a few RDMA cards to see what problems get triggered.
I think I reported a few at the time, but I gave up after feeling it was getting very hacky.

Quoted 7 lines

Agreed. Maybe we can have a list of that in the cover letter or even QEMU's migration/rmda doc page.

Lei, if you think that makes sense please do so in your upcoming posts. There'll need to have a list of things you encountered in the kernel driver and it'll be even better if there're further links to read on each problem.

OK, no problem. There are two bugs:

Bug 1:

https://github.com/linux-rdma/rdma-core/commit/23985e25aebb559b761872313f8cab4e811c5a3d#diff-5ddbf83c6f021688166096ca96c9bba874dffc3cab88ded2e9d8b2176faa084cR3302-R3303

his commit introduces a bug that causes QEMU suspension.
When the timeout parameter of the rpoll is not -1 or 0, the program is suspended occasionally.

Problem analysis:
During the first rpoll,
In line 3297, rs_poll_enter () performs pollcnt++. In this case, the value of pollcnt is 1.
In line 3302, timeout expires and the function exits. Note that rs_poll_exit () is not --pollcnt here.
In this case, the value of pollcnt is 1.
During the second rpoll, pollcnt++ is performed in line 3297 rs_poll_enter (). In this case, the value of pollcnt is 2.
If no timeout expires and the poll return value is greater than 0, the rs_poll_stop () function is executed. Because the if (--pollcnt) condition is false, suspendpoll = 1 is executed.
Go back to the do while loop inside rpoll, again rs_poll_enter () now if (suspendpoll) condition is true, execute pthread_yield (); and return -EBUSY, Then, the do while loop in the rpoll is returned. Because the if (rs_poll_enter ()) condition is true, the rs_poll_enter () function is executed again after the continue operation. As a result, the program is suspended.

Root cause: In line 3302, rs_poll_exit () is not executed before the timeout expires function exits.

Bug 2:

In rsocket.c, there is a receive queue int accept_queue[2] implemented by socketpair. The listen_svc thread in rsocket.c is responsible for receiving connections and writing them to the accept_queue[1]. When raccept () is called, a connection is received from accept_queue[0]. In the test case, qio_channel_wait(QIO_CHANNEL(lioc), G_IO_IN); waits for a readable event (waiting for a connection), rpoll () checks if accept_queue[0] has a readable event, However, this poll does not poll accept_queue[0]. After the timeout expires, rpoll () obtains the readable event of accept_queue[0] from rs_poll_arm again.

Impaction:
The accept operation can be performed only after 5000 ms. Of course, we can shorten this time by echoing the millisecond time > /etc/rdma/rsocket/wake_up_interval.

Regards,
-Gonglei

Quoted 30 lines
Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

e) Someone made a good suggestion (sorry can't remember who) -

that the

Quoted 1 line
Quoted 1 line
Quoted 1 line

RDMA migration structure was the wrong way around - it should

be the

Quoted 2 lines
Quoted 2 lines
Quoted 2 lines

destination which initiates an RDMA read, rather than the source doing a write; then things might become a LOT simpler; you just

need

Quoted 9 lines
Quoted 8 lines
Quoted 2 lines

to send page ranges to the destination and it can pull it. That might work nicely for postcopy.

I'm not sure whether it'll still be a problem if rdma recv side is based on zero-copy. It would be a matter of whether atomicity can be guaranteed so that we don't want the guest vcpus to see a partially copied page during on-flight DMAs. UFFDIO_COPY (or friend) is currently the only solution for that.

Quoted 4 lines

Yes, but even ignoring that (and the UFFDIO_CONTINUE idea you mention), if the destination can issue an RDMA read itself, it doesn't need to send messages to the source to ask for a page fetch; it just goes and grabs it itself, that's got to be good for latency.

Oh, that's pretty internal stuff of rdma to me and beyond my knowledge.. but from what I can tell it sounds very reasonable indeed!

Thanks!

--
Peter Xu

#75 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu

Hi Peter,

Quoted 207 lines · Peter Xu
Quoted 178 lines · Jinpu Wang

Hi Peter,

Quoted 69 lines · Peter Xu
Quoted 1 line · Jinpu Wang

Hi Peter,

Jinpu,

Thanks for joining the discussion.

Quoted 55 lines · Peter Xu
Quoted 53 lines · Yu Zhang
Quoted 4 lines

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

Quoted 14 lines
Quoted 6 lines

IMHO it's more important to know whether there are still users and whether they would still like to see it around.

I admit RDMA migration was lack of testing(unit/CI test), which led to the a few obvious bugs being noticed too late.

Yes, we are a user of this subsystem. I was unaware of the lack of test coverage for this part. As soon as 8.2 was released, I saw that many of the migration test
cases failed and came to realize that there might be a bug between 8.1 and 8.2, but
was unable to confirm and report it quickly to you.

Quoted 2 lines

The maintenance of this part could be too costly or difficult from your point of view.

It may or may not be too costly, it's just that we need real users of RDMA taking some care of it. Having it broken easily for >1 releases definitely is a sign of lack of users. It is an implication to the community that we should consider dropping some features so that we can get the best use of the community resources for the things that may have a broader audience.

One thing majorly missing is a RDMA tester to guard all the merges to not break RDMA paths, hopefully in CI. That should not rely on RDMA hardwares but just to sanity check the migration+rdma code running all fine. RDMA taught us the lesson so we're requesting CI coverage for all other new features that will be merged at least for migration subsystem, so that we plan to not merge anything that is not covered by CI unless extremely necessary in the future.

For sure CI is not the only missing part, but I'd say we should start with it, then someone should also take care of the code even if only in maintenance mode (no new feature to add on top).

Quoted 6 lines

My concern is, this plan will forces a few QEMU users (not sure how many) like us
either to stick to the RDMA migration by using an increasingly older version of QEMU,
or to abandon the currently used RDMA migration.

Quoted 5 lines
Quoted 3 lines

RDMA doesn't get new features anyway, if there's specific use case for RDMA migrations, would it work if such a scenario uses the old binary? Is it possible to switch to the TCP protocol with some good NICs?

We have used rdma migration with HCA from Nvidia for years, our experience is RDMA migration works better than tcp (over ipoib).

Quoted 8 lines

Please bare with me, as I know little on rdma stuff.

I'm actually pretty confused (and since a long time ago..) on why we need to operation with rdma contexts when ipoib seems to provide all the tcp layers. I meant, can it work with the current "tcp:" protocol with ipoib even if there's rdma/ib hardwares underneath? Is it because of performance improvements so that we must use a separate path comparing to generic "tcp:" protocol here?

using rdma protocol with ib verbs , we can leverage the full benefit of RDMA by talking directly to NIC which bypasses the kernel overhead, less cpu utilization and better performance.

While IPoIB is more for compatibility to applications using tcp, but can't get full benefit of RDMA. When you have mix generation of IB devices, there are performance issue on IPoIB, we've seen 40G HCA can only reach 2 Gb/s on IPoIB, but with raw RDMA can reach full line speed.

I just run a simple iperf3 test via ipoib and ib_send_bw on same hosts:

iperf 3.9
Linux ps404a-3 5.15.137-pserver #5.15.137-6~deb11 SMP Thu Jan 4 07:19:34 UTC 2024 x86_64
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
Time: Tue, 09 Apr 2024 06:55:02 GMT
Accepted connection from 2a02:247f:401:4:2:0:b:3, port 41130
Cookie: cer2hexlldrowclq6izh7gbg5toviffqbcwt
TCP MSS: 0 (default)
[ 5] local 2a02:247f:401:4:2:0:a:3 port 5201 connected to 2a02:247f:401:4:2:0:b:3 port 41136
Starting Test: protocol: TCP, 1 streams, 131072 byte blocks, omitting 0 seconds, 10 second test, tos 0

[ ID] Interval           Transfer     Bitrate
[  5]   0.00-1.00   sec  1.80 GBytes  15.5 Gbits/sec
[  5]   1.00-2.00   sec  1.85 GBytes  15.9 Gbits/sec
[  5]   2.00-3.00   sec  1.88 GBytes  16.2 Gbits/sec
[  5]   3.00-4.00   sec  1.87 GBytes  16.1 Gbits/sec
[  5]   4.00-5.00   sec  1.88 GBytes  16.2 Gbits/sec
[  5]   5.00-6.00   sec  1.93 GBytes  16.6 Gbits/sec
[  5]   6.00-7.00   sec  2.00 GBytes  17.2 Gbits/sec
[  5]   7.00-8.00   sec  1.93 GBytes  16.6 Gbits/sec
[  5]   8.00-9.00   sec  1.86 GBytes  16.0 Gbits/sec
[  5]   9.00-10.00  sec  1.95 GBytes  16.8 Gbits/sec
[  5]  10.00-10.04  sec  85.2 MBytes  17.3 Gbits/sec

- - - - - - - - - - - - - - - - - - - - - - - - - Test Complete. Summary Results:
[ ID] Interval Transfer Bitrate
[ 5] (sender statistics not available)
[ 5] 0.00-10.04 sec 19.0 GBytes 16.3 Gbits/sec receiver
rcv_tcp_congestion cubic
iperf 3.9
Linux ps404a-3 5.15.137-pserver #5.15.137-6~deb11 SMP Thu Jan 4 07:19:34 UTC 2024 x86_64
-----------------------------------------------------------
Server listening on 5201
-----------------------------------------------------------
^Ciperf3: interrupt - the server has terminated
1 [email protected]:~$ sudo ib_send_bw -F -a

************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
Send BW Test
Dual-port : OFF Device : mlx5_0
Number of qps : 1 Transport type : IB
Connection type : RC Using SRQ : OFF
PCIe relax order: ON

 ibv_wr* API     : ON
 RX depth        : 512
 CQ Moderation   : 100
 Mtu             : 4096[B]
 Link type       : IB

Max inline data : 0[B]
rdma_cm QPs : OFF
Data ex. method : Ethernet
---------------------------------------------------------------------------------------
local address: LID 0x24 QPN 0x0174 PSN 0x300138
remote address: LID 0x17 QPN 0x004a PSN 0xc54d6f
---------------------------------------------------------------------------------------

 #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]
 2          1000             0.00               6.46       3.385977
 4          1000             0.00               10.38     2.721894
 8          1000             0.00               25.69     3.367830
 16         1000             0.00               41.46     2.716859
 32         1000             0.00               102.98    3.374577
 64         1000             0.00               206.12    3.377053
 128        1000             0.00               405.03    3.318007
 256        1000             0.00               821.52    3.364939
 512        1000             0.00               2150.78    4.404803
 1024       1000             0.00               4288.13    4.391044
 2048       1000             0.00               8518.25    4.361346
 4096       1000             0.00               11440.77    2.928836
 8192       1000             0.00               11526.45    1.475385
 16384      1000             0.00               11526.06    0.737668
 32768      1000             0.00               11524.86    0.368795
 65536      1000             0.00               11331.84    0.181309
 131072     1000             0.00               11524.75    0.092198
 262144     1000             0.00               11525.82    0.046103
 524288     1000             0.00               11524.70    0.023049
 1048576    1000             0.00               11510.84    0.011511
 2097152    1000             0.00               11524.58    0.005762
 4194304    1000             0.00               11514.26    0.002879
 8388608    1000             0.00               11511.01    0.001439
Quoted 5 lines

---------------------------------------------------------------------------------------

you can see with ipoib, it reaches 16 Gb/s using TCP, 1 streams, 131072 byte blocks
with RDMA at 4k+ message size it reaches 100 Gb/s

I get it now, thank you!

Quoted 19 lines
Quoted 7 lines
Quoted 6 lines

Switching back to TCP will lead us to the old problems which was solved by RDMA migration.

Quoted 7 lines

Can you elaborate the problems, and why tcp won't work in this case? They may not be directly relevant to the issue we're discussing, but I'm happy to learn more.

What is the NICs you were testing before? Did the test carry out with things like modern ones (50Gbps-200Gbps NICs), or the test was done when these hardwares are not common?

We use Mellanox/NVidia IB HCA from 40 Gb/s to 200 Gb/s mixed generation across globe.

Quoted 3 lines

Per my recent knowledge on the new Intel hardwares, at least the ones that support QPL, it's easy to achieve single core 50Gbps+.

Quoted 7 lines
Quoted 1 line

In good case, I've also seen 50 Gbps + on Mellanox HCA.

I see. Have you compared the HCAs v.s. the modern NICs? Now NICs can achieve similar performance from their spec as I said; I am not sure how they perform in real life, but maybe worth trying. I only tried 100G nic and I rem I can hit 70+Gbps with multifd migrations at peak bandwidth. Have you tried that before?

Yes, I recently tried 100 G Eth NIC, with only iperf not yet with qemu migration.
yes, iperf can reach 90 Gbps with multiple streams.

Quoted 29 lines

Note that here I didn't want to compare the performance between the two and find a winner. The issue we're facing now is we have the RDMA migration now mostly having its own path all over the place, while the rest protocols (socket, fd, file, etc.) all share the rest.

Then, _if_ modern NICs can work similarly v.s. rdma, I don't yet see a good reason to keep it. It could be that technology just improved so we can use less code to do as good. It's a good news to help QEMU evolve by dropping unused code.

For some details there on the rdma complications for migration:

(1) RDMA is the only protocol that doesn't yet support QIOChannel, while migration uses QIOChannels mostly everywhere now.. e.g. in multifd, it means it won't easily support any new things using QIOChannels.

(2) RDMA is the only protocol that mostly hard-coded everywhere in the RAM migrations, polluting the core logic with much more code internally to support this protocol.

For (1), see migrate_fd_connect() from rdma_start_outgoing_migration(). While the rest protocols all go via migration_channel_connect().

For (2), see all the "rdma_*" functions in migration/ram.c, where I don't think it's common to a protocol - most of the rest protocols don't need those hard-coded stuff. migration/rdma.c has 4000+ LOC for these stuff, while to do a not-so-fair comparison, migration/fd.c only has <100 LOC.

Quoted 4 lines

Then, we found we don't even know who's using it.

I hope I explained why people started this idea, and also why I think that makes sense at least to me.

Yes, I can understand rdma migration become more a burden for upstream maintainers.

Quoted 43 lines
Quoted 43 lines
Quoted 6 lines

https://lore.kernel.org/r/PH7PR11MB5941A91AC1E514BCC32896A6A3342@PH7PR11MB5941.namprd11.prod.outlook.com

Quote from Yuan:

Quoted 8 lines

Yes, I use iperf3 to check the bandwidth for one core, the bandwith is 60Gbps.

  [ ID] Interval           Transfer     Bitrate         Retr  Cwnd
  [  5]   0.00-1.00   sec  7.00 GBytes  60.1 Gbits/sec    0   2.87 MBytes
  [  5]   1.00-2.00   sec  7.05 GBytes  60.6 Gbits/sec    0   2.87 Mbytes

And in the live migration test, a multifd thread's CPU utilization is almost 100%

It boils down to what old problems were there with tcp first, though.

Yeah, this is the key reason we use RDMA. (low cpu ulitization and better performance)

Quoted 26 lines
Quoted 11 lines
Quoted 11 lines

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

Quoted 4 lines

+cc some guys from Huawei.

I'm surprised RDMA users are rare, I guess maybe many are just working with different code base.

Yes, please cc whoever might be interested (or surprised.. :) to know this, and let's be open to all possibilities.

I don't think it makes sense if there're a lot of users of a feature then we deprecate that without a good reason. However there's always the resource limitation issue we're facing, so it could still have the possibility that this gets deprecated if nobody is working on our upstream branch. Say, if people use private branches anyway to support rdma without collaborating upstream, keeping such feature upstream then may not make much sense either, unless there's some way to collaborate. We'll see.

Quoted 7 lines
Quoted 4 lines

Is there document/link about the unittest/CI for migration tests, Why are those tests missing?
Is it hard or very special to set up an environment for that? maybe we can help in this regards.

See tests/qtest/migration-test.c. We put most of our migration tests there and that's covered in CI.

Yu is looking into that see if we can run the CI on our side.

Quoted 3 lines

I think one major issue is CI systems don't normally have rdma devices. Can rdma migration test be carried out without a real hardware?

As Zhijian mentioned we can use the SoftRoCE (rxe)

Quoted 5 lines
Quoted 5 lines
Quoted 4 lines

It seems there can still be people joining this discussion. I'll hold off a bit on merging this patch to provide enough window for anyone to chim in.

Quoted 8 lines
Quoted 1 line

Thx for discussion and understanding.

Thanks for all these inputs so far. These can help us make a wiser and clearer step no matter which way we choose.

--
Peter Xu

Thx!

#76 lore raw

Re: [PATCH-for-9.1 v2 2/3] migration: Remove RDMA protocol handling

In reply to Peter Xu
Quoted 19 lines · Peter Xu
Quoted 1 line · Jinpu Wang

Hi Peter,

Jinpu,

Thanks for joining the discussion.

Quoted 12 lines · Peter Xu
Quoted 10 lines · Yu Zhang
Quoted 4 lines

Hello Peter und Zhjian,

Thank you so much for letting me know about this. I'm also a bit surprised at the plan for deprecating the RDMA migration subsystem.

It's not too late, since it looks like we do have users not yet notified from this, we'll redo the deprecation procedure even if it'll be the final plan, and it'll be 2 releases after this.

[...]

Quoted 24 lines
Quoted 8 lines
Quoted 8 lines

Per our best knowledge, RDMA users are rare, and please let anyone know if you are aware of such users. IIUC the major reason why RDMA stopped being the trend is because the network is not like ten years ago; I don't think I have good knowledge in RDMA at all nor network, but my understanding is it's pretty easy to fetch modern NIC to outperform RDMAs, then it may make little sense to maintain multiple protocols, considering RDMA migration code is so special so that it has the most custom code comparing to other protocols.

Quoted 4 lines

+cc some guys from Huawei.

I'm surprised RDMA users are rare, I guess maybe many are just working with different code base.

Yes, please cc whoever might be interested (or surprised.. :) to know this, and let's be open to all possibilities.

I don't think it makes sense if there're a lot of users of a feature then we deprecate that without a good reason. However there's always the resource limitation issue we're facing, so it could still have the possibility that this gets deprecated if nobody is working on our upstream branch. Say, if people use private branches anyway to support rdma without collaborating upstream, keeping such feature upstream then may not make much sense either, unless there's some way to collaborate. We'll see.

Quoted 2 lines

It seems there can still be people joining this discussion. I'll hold off a bit on merging this patch to provide enough window for anyone to chim in.

Users are not enough. Only maintainers are.

At some point, people cared enough about RDMA in QEMU to contribute the code. That's why have the code.

To keep the code, we need people who care enough about RDMA in QEMU to maintain it. Without such people, the case for keeping it remains dangerously weak, and no amount of talk or even benchmarks can change that.

#77 lore raw

PATCH v2 3/3 block/gluster: Remove RDMA protocol handling

In reply to Philippe Mathieu-Daudé

GlusterFS+RDMA has been deprecated 8 years ago in commit 0552ff2465 ("block/gluster: deprecate rdma support"):

gluster volfile server fetch happens through unix and/or tcp, it doesn't support volfile fetch over rdma. The rdma code may actually mislead, so to make sure things do not break, for now we fallback to tcp when requested for rdma, with a warning.

If you are wondering how this worked all these days, its the gluster libgfapi code which handles anything other than unix transport as socket/tcp, sad but true.

Besides, the whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Cc Prasanna Kumar Kalever <[email protected]>
---
 docs/system/device-url-syntax.rst.inc  |  4 +--
 docs/system/qemu-block-drivers.rst.inc |  1 -
 block/gluster.c                        | 39 --------------------------
 3 files changed, 2 insertions(+), 42 deletions(-)
docs/system/device-url-syntax.rst.inc +2 -2
index 7dbc525fa8..43b5c2596b 100644
@@ -87,8 +87,8 @@ These are specified using a special URL syntax.
 
 ``GlusterFS``
    GlusterFS is a user space distributed file system. QEMU supports the
-   use of GlusterFS volumes for hosting VM disk images using TCP, Unix
-   Domain Sockets and RDMA transport protocols.
+   use of GlusterFS volumes for hosting VM disk images using TCP and Unix
+   Domain Sockets transport protocols.
 
    Syntax for specifying a VM disk image on GlusterFS volume is
 
docs/system/qemu-block-drivers.rst.inc +0 -1
index 105cb9679c..384e95ba76 100644
@@ -737,7 +737,6 @@ Examples
   |qemu_system| -drive file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
   |qemu_system| -drive file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
   |qemu_system| -drive file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
-  |qemu_system| -drive file=gluster+rdma://1.2.3.4:24007/testvol/a.img
   |qemu_system| -drive file=gluster://1.2.3.4/testvol/a.img,file.debug=9,file.logfile=/var/log/qemu-gluster.log
   |qemu_system| 'json:{"driver":"qcow2",
                            "file":{"driver":"gluster",
block/gluster.c +0 -39
index cc74af06dc..4253c8db5e 100644
@@ -371,9 +371,6 @@ static int qemu_gluster_parse_uri(BlockdevOptionsGluster *gconf,
     } else if (!strcmp(uri->scheme, "gluster+unix")) {
         gsconf->type = SOCKET_ADDRESS_TYPE_UNIX;
         is_unix = true;
-    } else if (!strcmp(uri->scheme, "gluster+rdma")) {
-        gsconf->type = SOCKET_ADDRESS_TYPE_INET;
-        warn_report("rdma feature is not supported, falling back to tcp");
     } else {
         ret = -EINVAL;
         goto out;
@@ -1638,44 +1635,8 @@ static BlockDriver bdrv_gluster_unix = {
     .strong_runtime_opts          = gluster_strong_open_opts,
 };
 
-/* rdma is deprecated (actually never supported for volfile fetch).
- * Let's maintain it for the protocol compatibility, to make sure things
- * won't break immediately. For now, gluster+rdma will fall back to gluster+tcp
- * protocol with a warning.
- * TODO: remove gluster+rdma interface support
- */
-static BlockDriver bdrv_gluster_rdma = {
-    .format_name                  = "gluster",
-    .protocol_name                = "gluster+rdma",
-    .instance_size                = sizeof(BDRVGlusterState),
-    .bdrv_file_open               = qemu_gluster_open,
-    .bdrv_reopen_prepare          = qemu_gluster_reopen_prepare,
-    .bdrv_reopen_commit           = qemu_gluster_reopen_commit,
-    .bdrv_reopen_abort            = qemu_gluster_reopen_abort,
-    .bdrv_close                   = qemu_gluster_close,
-    .bdrv_co_create               = qemu_gluster_co_create,
-    .bdrv_co_create_opts          = qemu_gluster_co_create_opts,
-    .bdrv_co_getlength            = qemu_gluster_co_getlength,
-    .bdrv_co_get_allocated_file_size = qemu_gluster_co_get_allocated_file_size,
-    .bdrv_co_truncate             = qemu_gluster_co_truncate,
-    .bdrv_co_readv                = qemu_gluster_co_readv,
-    .bdrv_co_writev               = qemu_gluster_co_writev,
-    .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
-#ifdef CONFIG_GLUSTERFS_DISCARD
-    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
-#endif
-#ifdef CONFIG_GLUSTERFS_ZEROFILL
-    .bdrv_co_pwrite_zeroes        = qemu_gluster_co_pwrite_zeroes,
-#endif
-    .bdrv_co_block_status         = qemu_gluster_co_block_status,
-    .bdrv_refresh_limits          = qemu_gluster_refresh_limits,
-    .create_opts                  = &qemu_gluster_create_opts,
-    .strong_runtime_opts          = gluster_strong_open_opts,
-};
-
 static void bdrv_gluster_init(void)
 {
-    bdrv_register(&bdrv_gluster_rdma);
     bdrv_register(&bdrv_gluster_unix);
     bdrv_register(&bdrv_gluster_tcp);
     bdrv_register(&bdrv_gluster);
Signature · 1 line
2.41.0
Reviewed-by on v2 3/3
#78 lore raw

Re: [PATCH-for-9.1 v2 3/3] block/gluster: Remove RDMA protocol handling

Quoted 16 lines · Philippe Mathieu-Daudé

GlusterFS+RDMA has been deprecated 8 years ago in commit 0552ff2465 ("block/gluster: deprecate rdma support"):

gluster volfile server fetch happens through unix and/or tcp, it doesn't support volfile fetch over rdma. The rdma code may actually mislead, so to make sure things do not break, for now we fallback to tcp when requested for rdma, with a warning.

If you are wondering how this worked all these days, its the gluster libgfapi code which handles anything other than unix transport as socket/tcp, sad but true.

Besides, the whole RDMA subsystem was deprecated in commit e9a54265f5 ("hw/rdma: Deprecate the pvrdma device and the rdma subsystem") released in v8.2.

Quoted 7 lines
Cc Prasanna Kumar Kalever <[email protected]>
---
  docs/system/device-url-syntax.rst.inc  |  4 +--
  docs/system/qemu-block-drivers.rst.inc |  1 -
  block/gluster.c                        | 39 --------------------------
  3 files changed, 2 insertions(+), 42 deletions(-)
Reviewed-by on the v2 series
#79 lore raw
In reply to Philippe Mathieu-Daudé
Quoted 3 lines · Philippe Mathieu-Daudé

Since v1:
- split in 3 (Thomas)
- justify gluster removal

Quoted 118 lines

Philippe Mathieu-Daudé (3):
hw/rdma: Remove pvrdma device and rdmacm-mux helper
migration: Remove RDMA protocol handling block/gluster: Remove RDMA protocol handling

 MAINTAINERS                                   |   17 -
 docs/about/deprecated.rst                     |    9 -
 docs/about/removed-features.rst               |    4 +
 docs/devel/migration/main.rst                 |    6 -
 docs/pvrdma.txt                               |  345 --
 docs/rdma.txt                                 |  420 --
 docs/system/device-url-syntax.rst.inc         |    4 +-
 docs/system/loongarch/virt.rst                |    2 +-
 docs/system/qemu-block-drivers.rst.inc        |    1 -
 meson.build                                   |   59 -
 qapi/machine.json                             |   17 -
 qapi/migration.json                           |   31 +-
 qapi/qapi-schema.json                         |    1 -
 qapi/rdma.json                                |   38 -
 contrib/rdmacm-mux/rdmacm-mux.h               |   61 -
 hw/rdma/rdma_backend.h                        |  129 -
 hw/rdma/rdma_backend_defs.h                   |   76 -
 hw/rdma/rdma_rm.h                             |   97 -
 hw/rdma/rdma_rm_defs.h                        |  146 -
 hw/rdma/rdma_utils.h                          |   63 -
 hw/rdma/trace.h                               |    1 -
 hw/rdma/vmw/pvrdma.h                          |  144 -
 hw/rdma/vmw/pvrdma_dev_ring.h                 |   46 -
 hw/rdma/vmw/pvrdma_qp_ops.h                   |   28 -
 hw/rdma/vmw/trace.h                           |    1 -
 include/hw/rdma/rdma.h                        |   37 -
 include/monitor/hmp.h                         |    1 -
 .../infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h |  685 ---
 .../infiniband/hw/vmw_pvrdma/pvrdma_verbs.h   |  348 --
 .../standard-headers/rdma/vmw_pvrdma-abi.h    |  310 --
 migration/migration-stats.h                   |    6 +-
 migration/migration.h                         |    9 -
 migration/options.h                           |    2 -
 migration/rdma.h                              |   69 -
 block/gluster.c                               |   39 -
 contrib/rdmacm-mux/main.c                     |  831 ----
 hw/core/machine-qmp-cmds.c                    |   32 -
 hw/rdma/rdma.c                                |   30 -
 hw/rdma/rdma_backend.c                        | 1401 ------
 hw/rdma/rdma_rm.c                             |  812 ----
 hw/rdma/rdma_utils.c                          |  126 -
 hw/rdma/vmw/pvrdma_cmd.c                      |  815 ----
 hw/rdma/vmw/pvrdma_dev_ring.c                 |  141 -
 hw/rdma/vmw/pvrdma_main.c                     |  735 ---
 hw/rdma/vmw/pvrdma_qp_ops.c                   |  298 --
 migration/migration-stats.c                   |    5 +-
 migration/migration.c                         |   31 -
 migration/options.c                           |   16 -
 migration/qemu-file.c                         |    1 -
 migration/ram.c                               |   86 +-
 migration/rdma.c                              | 4184 -----------------
 migration/savevm.c                            |    2 +-
 monitor/qmp-cmds.c                            |    1 -
 Kconfig.host                                  |    3 -
 contrib/rdmacm-mux/meson.build                |    7 -
 hmp-commands-info.hx                          |   13 -
 hw/Kconfig                                    |    1 -
 hw/meson.build                                |    1 -
 hw/rdma/Kconfig                               |    3 -
 hw/rdma/meson.build                           |   12 -
 hw/rdma/trace-events                          |   31 -
 hw/rdma/vmw/trace-events                      |   17 -
 meson_options.txt                             |    4 -
 migration/meson.build                         |    1 -
 migration/trace-events                        |   68 +-
 qapi/meson.build                              |    1 -
 qemu-options.hx                               |    6 -
 .../org.centos/stream/8/build-environment.yml |    1 -
 .../ci/org.centos/stream/8/x86_64/configure   |    3 -
 scripts/ci/setup/build-environment.yml        |    4 -
 scripts/coverity-scan/run-coverity-scan       |    2 +-
 scripts/meson-buildoptions.sh                 |    6 -
 scripts/update-linux-headers.sh               |   27 -
 tests/lcitool/projects/qemu.yml               |    3 -
 tests/migration/guestperf/engine.py           |    4 +-
 75 files changed, 20 insertions(+), 12997 deletions(-)
 delete mode 100644 docs/pvrdma.txt
 delete mode 100644 docs/rdma.txt
 delete mode 100644 qapi/rdma.json
 delete mode 100644 contrib/rdmacm-mux/rdmacm-mux.h
 delete mode 100644 hw/rdma/rdma_backend.h
 delete mode 100644 hw/rdma/rdma_backend_defs.h
 delete mode 100644 hw/rdma/rdma_rm.h
 delete mode 100644 hw/rdma/rdma_rm_defs.h
 delete mode 100644 hw/rdma/rdma_utils.h
 delete mode 100644 hw/rdma/trace.h
 delete mode 100644 hw/rdma/vmw/pvrdma.h
 delete mode 100644 hw/rdma/vmw/pvrdma_dev_ring.h
 delete mode 100644 hw/rdma/vmw/pvrdma_qp_ops.h
 delete mode 100644 hw/rdma/vmw/trace.h
 delete mode 100644 include/hw/rdma/rdma.h
 delete mode 100644 include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
 delete mode 100644 include/standard-headers/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h
 delete mode 100644 include/standard-headers/rdma/vmw_pvrdma-abi.h
 delete mode 100644 migration/rdma.h
 delete mode 100644 contrib/rdmacm-mux/main.c
 delete mode 100644 hw/rdma/rdma.c
 delete mode 100644 hw/rdma/rdma_backend.c
 delete mode 100644 hw/rdma/rdma_rm.c
 delete mode 100644 hw/rdma/rdma_utils.c
 delete mode 100644 hw/rdma/vmw/pvrdma_cmd.c
 delete mode 100644 hw/rdma/vmw/pvrdma_dev_ring.c
 delete mode 100644 hw/rdma/vmw/pvrdma_main.c
 delete mode 100644 hw/rdma/vmw/pvrdma_qp_ops.c
 delete mode 100644 migration/rdma.c
 delete mode 100644 contrib/rdmacm-mux/meson.build
 delete mode 100644 hw/rdma/Kconfig
 delete mode 100644 hw/rdma/meson.build
 delete mode 100644 hw/rdma/trace-events
 delete mode 100644 hw/rdma/vmw/trace-events
Signature · 1 line
2.41.0