Skip to contentSkip to conversation

vduse: Add support for reconnection

PATCH Superseded Cindy Lu 3 participants 18 messages Feb 7, 2024 – Mar 20, 2024

Subsystems (2)

v4 of 5 Superseded Latest is v5 →

lore
Share & tools
lore

Cindy Lu

author
#0 lore raw

v4 cover vduse: Add support for reconnection

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)
Signature · 1 line
2.43.0
author
#1 lore raw

PATCH v4 1/5 vduse: Add new ioctl VDUSE_DEV_GET_CONFIG

The ioctl VDUSE_DEV_GET_CONFIG is used by the Userspace App to get the device configuration space.

---
 drivers/vdpa/vdpa_user/vduse_dev.c | 21 +++++++++++++++++++++
 include/uapi/linux/vduse.h         |  3 +++
 2 files changed, 24 insertions(+)
drivers/vdpa/vdpa_user/vduse_dev.c +21 -0
index 1c1d71d69026..ab246da27616 100644
@@ -1368,6 +1368,27 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		ret = 0;
 		break;
 	}
+	case VDUSE_DEV_GET_CONFIG: {
+		struct vduse_config_data config;
+		unsigned long size = offsetof(struct vduse_config_data, buffer);
+
+		ret = -EFAULT;
+		if (copy_from_user(&config, argp, size))
+			break;
+
+		ret = -EINVAL;
+		if (config.offset > dev->config_size || config.length == 0 ||
+		    config.length > dev->config_size - config.offset)
+			break;
+
+		if (copy_to_user(argp + size, dev->config + config.offset,
+				 config.length)) {
+			ret = -EFAULT;
+			break;
+		}
+		ret = 0;
+		break;
+	}
 	default:
 		ret = -ENOIOCTLCMD;
 		break;
include/uapi/linux/vduse.h +3 -0
index 11bd48c72c6c..125d7529d91b 100644
@@ -350,4 +350,7 @@ struct vduse_dev_response {
 	};
 };
 
+/* get device configuration space */
+#define VDUSE_DEV_GET_CONFIG _IOR(VDUSE_BASE, 0x1b, struct vduse_config_data)
+
 #endif /* _UAPI_VDUSE_H_ */
Signature · 1 line
2.43.0
author
#2 lore raw

PATCH v4 2/5 vduse: Add new ioctl VDUSE_DEV_GET_STATUS

In reply to Cindy Lu

The ioctl VDUSE_DEV_GET_STATUS is used by the Userspace App to get the device status

---
 drivers/vdpa/vdpa_user/vduse_dev.c | 7 +++++++
 include/uapi/linux/vduse.h         | 2 ++
 2 files changed, 9 insertions(+)
drivers/vdpa/vdpa_user/vduse_dev.c +7 -0
index ab246da27616..ef3c9681941e 100644
@@ -1389,6 +1389,13 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 		ret = 0;
 		break;
 	}
+
+	case VDUSE_DEV_GET_STATUS:
+		/*
+		 * Returns the status read from device
+		 */
+		ret = put_user(dev->status, (u8 __user *)argp);
+		break;
 	default:
 		ret = -ENOIOCTLCMD;
 		break;
include/uapi/linux/vduse.h +2 -0
index 125d7529d91b..f501173a9d69 100644
@@ -353,4 +353,6 @@ struct vduse_dev_response {
 /* get device configuration space */
 #define VDUSE_DEV_GET_CONFIG _IOR(VDUSE_BASE, 0x1b, struct vduse_config_data)
 
+#define VDUSE_DEV_GET_STATUS _IOR(VDUSE_BASE, 0x1c, __u8)
+
 #endif /* _UAPI_VDUSE_H_ */
Signature · 1 line
2.43.0
author
#3 lore raw

PATCH v4 3/5 vduse: Add function to get/free the pages for reconnection

In reply to Cindy Lu

Add the function vduse_alloc_reconnnect_info_mem
and vduse_alloc_reconnnect_info_mem
These functions allow vduse to allocate and free memory for reconnection information. The amount of memory allocated is vq_num pages.
Each VQS will map its own page where the reconnection information will be saved

---
 drivers/vdpa/vdpa_user/vduse_dev.c | 40 ++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
drivers/vdpa/vdpa_user/vduse_dev.c +40 -0
index ef3c9681941e..2da659d5f4a8 100644
@@ -65,6 +65,7 @@ struct vduse_virtqueue {
 	int irq_effective_cpu;
 	struct cpumask irq_affinity;
 	struct kobject kobj;
+	unsigned long vdpa_reconnect_vaddr;
 };
 
 struct vduse_dev;
@@ -1105,6 +1106,38 @@ static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq)
 
 	vq->irq_effective_cpu = curr_cpu;
 }
+static int vduse_alloc_reconnnect_info_mem(struct vduse_dev *dev)
+{
+	unsigned long vaddr = 0;
+	struct vduse_virtqueue *vq;
+
+	for (int i = 0; i < dev->vq_num; i++) {
+		/*page 0~ vq_num save the reconnect info for vq*/
+		vq = dev->vqs[i];
+		vaddr = get_zeroed_page(GFP_KERNEL);
+		if (vaddr == 0)
+			return -ENOMEM;
+
+		vq->vdpa_reconnect_vaddr = vaddr;
+	}
+
+	return 0;
+}
+
+static int vduse_free_reconnnect_info_mem(struct vduse_dev *dev)
+{
+	struct vduse_virtqueue *vq;
+
+	for (int i = 0; i < dev->vq_num; i++) {
+		vq = dev->vqs[i];
+
+		if (vq->vdpa_reconnect_vaddr)
+			free_page(vq->vdpa_reconnect_vaddr);
+		vq->vdpa_reconnect_vaddr = 0;
+	}
+
+	return 0;
+}
 
 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 			    unsigned long arg)
@@ -1672,6 +1705,8 @@ static int vduse_destroy_dev(char *name)
 		mutex_unlock(&dev->lock);
 		return -EBUSY;
 	}
+	vduse_free_reconnnect_info_mem(dev);
+
 	dev->connected = true;
 	mutex_unlock(&dev->lock);
 
@@ -1855,12 +1890,17 @@ static int vduse_create_dev(struct vduse_dev_config *config,
 	ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num);
 	if (ret)
 		goto err_vqs;
+	ret = vduse_alloc_reconnnect_info_mem(dev);
+	if (ret < 0)
+		goto err_mem;
 
 	__module_get(THIS_MODULE);
 
 	return 0;
 err_vqs:
 	device_destroy(&vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
+err_mem:
+	vduse_free_reconnnect_info_mem(dev);
 err_dev:
 	idr_remove(&vduse_idr, dev->minor);
 err_idr:
Signature · 1 line
2.43.0
author
#4 lore raw

PATCH v4 4/5 vduse: Add file operation for mmap

In reply to Cindy Lu

Add the operation for mmap, This function will be used by the user space application to map the pages to the user space.

---
 drivers/vdpa/vdpa_user/vduse_dev.c | 57 ++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
drivers/vdpa/vdpa_user/vduse_dev.c +57 -0
index 2da659d5f4a8..083fe0047677 100644
@@ -1465,6 +1465,61 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
 	return dev;
 }
 
+static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
+{
+	struct vduse_dev *dev = vmf->vma->vm_file->private_data;
+	struct vm_area_struct *vma = vmf->vma;
+	u16 index = vma->vm_pgoff;
+	struct vduse_virtqueue *vq;
+	unsigned long vaddr;
+
+	/* index 1+vq_number page reserved for virtqueue state*/
+	vq = dev->vqs[index];
+	vaddr = vq->vdpa_reconnect_vaddr;
+	if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
+			    PFN_DOWN(virt_to_phys((void *)vaddr)), PAGE_SIZE,
+			    vma->vm_page_prot))
+		return VM_FAULT_SIGBUS;
+	return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct vduse_vm_ops = {
+	.fault = vduse_vm_fault,
+};
+
+static int vduse_dev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct vduse_dev *dev = file->private_data;
+	unsigned long vaddr = 0;
+	unsigned long index = vma->vm_pgoff;
+	struct vduse_virtqueue *vq;
+
+	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+		return -EINVAL;
+	if ((vma->vm_flags & VM_SHARED) == 0)
+		return -EINVAL;
+
+	if (index > dev->vq_num)
+		return -EINVAL;
+
+	vq = dev->vqs[index];
+	vaddr = vq->vdpa_reconnect_vaddr;
+	if (vaddr == 0)
+		return -EOPNOTSUPP;
+
+	if (virt_to_phys((void *)vaddr) & (PAGE_SIZE - 1))
+		return -EINVAL;
+
+	/* Check if the Userspace App mapped the correct size */
+	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+		return -EOPNOTSUPP;
+
+	vm_flags_set(vma, VM_DONTEXPAND);
+	vma->vm_ops = &vduse_vm_ops;
+
+	return 0;
+}
+
 static int vduse_dev_open(struct inode *inode, struct file *file)
 {
 	int ret;
@@ -1497,6 +1552,8 @@ static const struct file_operations vduse_dev_fops = {
 	.unlocked_ioctl	= vduse_dev_ioctl,
 	.compat_ioctl	= compat_ptr_ioctl,
 	.llseek		= noop_llseek,
+	.mmap		= vduse_dev_mmap,
+
 };
 
 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
Signature · 1 line
2.43.0
#5 lore raw

Re: [PATCH v4 4/5] vduse: Add file operation for mmap

Quoted 16 lines · Cindy Lu

Add the operation for mmap, This function will be used by the user space application to map the pages to the user space.

---
 drivers/vdpa/vdpa_user/vduse_dev.c | 57 ++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
drivers/vdpa/vdpa_user/vduse_dev.c +0 -0
index 2da659d5f4a8..083fe0047677 100644
@@ -1465,6 +1465,61 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
        return dev;
 }

drivers/vdpa/vdpa_user/vduse_dev.c static struct vduse_dev *vduse_dev_get_from_minor(int minor) · line 1476

Quoted 10 lines
drivers/vdpa/vdpa_user/vduse_dev.c +9 -0
 
+static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
+{
+       struct vduse_dev *dev = vmf->vma->vm_file->private_data;
+       struct vm_area_struct *vma = vmf->vma;
+       u16 index = vma->vm_pgoff;
+       struct vduse_virtqueue *vq;
+       unsigned long vaddr;
+
+       /* index 1+vq_number page reserved for virtqueue state*/

This comment seem wrong?

THanks

Quoted 61 lines
drivers/vdpa/vdpa_user/vduse_dev.c +48 -1
+       vq = dev->vqs[index];
+       vaddr = vq->vdpa_reconnect_vaddr;
+       if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
+                           PFN_DOWN(virt_to_phys((void *)vaddr)), PAGE_SIZE,
+                           vma->vm_page_prot))
+               return VM_FAULT_SIGBUS;
+       return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct vduse_vm_ops = {
+       .fault = vduse_vm_fault,
+};
+
+static int vduse_dev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+       struct vduse_dev *dev = file->private_data;
+       unsigned long vaddr = 0;
+       unsigned long index = vma->vm_pgoff;
+       struct vduse_virtqueue *vq;
+
+       if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+               return -EINVAL;
+       if ((vma->vm_flags & VM_SHARED) == 0)
+               return -EINVAL;
+
+       if (index > dev->vq_num)
+               return -EINVAL;
+
+       vq = dev->vqs[index];
+       vaddr = vq->vdpa_reconnect_vaddr;
+       if (vaddr == 0)
+               return -EOPNOTSUPP;
+
+       if (virt_to_phys((void *)vaddr) & (PAGE_SIZE - 1))
+               return -EINVAL;
+
+       /* Check if the Userspace App mapped the correct size */
+       if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+               return -EOPNOTSUPP;
+
+       vm_flags_set(vma, VM_DONTEXPAND);
+       vma->vm_ops = &vduse_vm_ops;
+
+       return 0;
+}
+
 static int vduse_dev_open(struct inode *inode, struct file *file)
 {
        int ret;
@@ -1497,6 +1552,8 @@ static const struct file_operations vduse_dev_fops = {
        .unlocked_ioctl = vduse_dev_ioctl,
        .compat_ioctl   = compat_ptr_ioctl,
        .llseek         = noop_llseek,
+       .mmap           = vduse_dev_mmap,
+
 };
 
 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
--

2.43.0

author
#6 lore raw

Re: [PATCH v4 4/5] vduse: Add file operation for mmap

Quoted 31 lines · Jason Wang
Quoted 16 lines · Cindy Lu

Add the operation for mmap, This function will be used by the user space application to map the pages to the user space.

---
 drivers/vdpa/vdpa_user/vduse_dev.c | 57 ++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
drivers/vdpa/vdpa_user/vduse_dev.c +0 -0
index 2da659d5f4a8..083fe0047677 100644
@@ -1465,6 +1465,61 @@ static struct vduse_dev *vduse_dev_get_from_minor(int minor)
        return dev;
 }

drivers/vdpa/vdpa_user/vduse_dev.c static struct vduse_dev *vduse_dev_get_from_minor(int minor) · line 1476

Quoted 10 lines
drivers/vdpa/vdpa_user/vduse_dev.c +9 -0
 
+static vm_fault_t vduse_vm_fault(struct vm_fault *vmf)
+{
+       struct vduse_dev *dev = vmf->vma->vm_file->private_data;
+       struct vm_area_struct *vma = vmf->vma;
+       u16 index = vma->vm_pgoff;
+       struct vduse_virtqueue *vq;
+       unsigned long vaddr;
+
+       /* index 1+vq_number page reserved for virtqueue state*/

This comment seem wrong?

Quoted 2 lines

THanks

sure,will fix this
thanks
cindy

Quoted 63 lines
Quoted 62 lines
drivers/vdpa/vdpa_user/vduse_dev.c +48 -1
+       vq = dev->vqs[index];
+       vaddr = vq->vdpa_reconnect_vaddr;
+       if (remap_pfn_range(vma, vmf->address & PAGE_MASK,
+                           PFN_DOWN(virt_to_phys((void *)vaddr)), PAGE_SIZE,
+                           vma->vm_page_prot))
+               return VM_FAULT_SIGBUS;
+       return VM_FAULT_NOPAGE;
+}
+
+static const struct vm_operations_struct vduse_vm_ops = {
+       .fault = vduse_vm_fault,
+};
+
+static int vduse_dev_mmap(struct file *file, struct vm_area_struct *vma)
+{
+       struct vduse_dev *dev = file->private_data;
+       unsigned long vaddr = 0;
+       unsigned long index = vma->vm_pgoff;
+       struct vduse_virtqueue *vq;
+
+       if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+               return -EINVAL;
+       if ((vma->vm_flags & VM_SHARED) == 0)
+               return -EINVAL;
+
+       if (index > dev->vq_num)
+               return -EINVAL;
+
+       vq = dev->vqs[index];
+       vaddr = vq->vdpa_reconnect_vaddr;
+       if (vaddr == 0)
+               return -EOPNOTSUPP;
+
+       if (virt_to_phys((void *)vaddr) & (PAGE_SIZE - 1))
+               return -EINVAL;
+
+       /* Check if the Userspace App mapped the correct size */
+       if (vma->vm_end - vma->vm_start != PAGE_SIZE)
+               return -EOPNOTSUPP;
+
+       vm_flags_set(vma, VM_DONTEXPAND);
+       vma->vm_ops = &vduse_vm_ops;
+
+       return 0;
+}
+
 static int vduse_dev_open(struct inode *inode, struct file *file)
 {
        int ret;
@@ -1497,6 +1552,8 @@ static const struct file_operations vduse_dev_fops = {
        .unlocked_ioctl = vduse_dev_ioctl,
        .compat_ioctl   = compat_ptr_ioctl,
        .llseek         = noop_llseek,
+       .mmap           = vduse_dev_mmap,
+
 };
 
 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf)
--

2.43.0

author
#7 lore raw

PATCH v4 5/5 Documentation: Add reconnect process for VDUSE

In reply to Cindy Lu

Add a document explaining the reconnect process, including what the Userspace App needs to do and how it works with the kernel.

---
 Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
Documentation/userspace-api/vduse.rst +32 -0
index bdb880e01132..a2be85e0e516 100644
@@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
    after the used ring is filled.
 
 For more details on the uAPI, please see include/uapi/linux/vduse.h.
+
+HOW VDUSE devices reconnectoin works
+----------------
+0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
+   If it does not exist, need to create the instance.goto step 1
+   If it does exist, it means this is a reconnect and goto step 3.
+
+1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
+   /dev/vduse/control.
+
+2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
+   to save the reconnect information.
+
+3. Userspace App need to mmap the pages to userspace
+   Userspace App need to map Pages 0 to vq_number for vq status,
+   Users can define the structure for saving the reconnect information themselves
+   in the userspace.
+
+4. Check if the infomatin sutiable for reconnect
+   If this is reconnect:
+   Before attempting to reconnect, The userspace application need to the
+   ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...
+   to get the and confirm if these information are suitable for reconnecting.
+
+5. Start the userspace App.
+   While running, the application should store the relevant information about
+   reconnections in mapped pages.
+   When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP to get vq information, it is necessary
+   to check if this is a reconnection. If a reconnection has occurred, the vq-related information
+   must be get from the mapped pages.
+
+6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
Signature · 1 line
2.43.0
#8 lore raw

Re: [PATCH v4 5/5] Documentation: Add reconnect process for VDUSE

Quoted 14 lines · Cindy Lu

Add a document explaining the reconnect process, including what the Userspace App needs to do and how it works with the kernel.

---
 Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
Documentation/userspace-api/vduse.rst +0 -0
index bdb880e01132..a2be85e0e516 100644

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 235

Quoted 5 lines
Documentation/userspace-api/vduse.rst +2 -0
@@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
    after the used ring is filled.
 
 For more details on the uAPI, please see include/uapi/linux/vduse.h.
+
+HOW VDUSE devices reconnectoin works

Typos, let's use a spell checker.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 247

Quoted 12 lines
Documentation/userspace-api/vduse.rst +12 -0
+----------------
+0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
+   If it does not exist, need to create the instance.goto step 1
+   If it does exist, it means this is a reconnect and goto step 3.
+
+1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
+   /dev/vduse/control.
+
+2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
+   to save the reconnect information.
+
+3. Userspace App need to mmap the pages to userspace

Need to describe what kind of pages need to be mapped. And what's more:

Does this require cooperation from the disconnected application? If yes, how to distinguish from the one that is not cooperative (doesn't support reconnection)?

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 252

Quoted 5 lines
Documentation/userspace-api/vduse.rst +5 -0
+   Userspace App need to map Pages 0 to vq_number for vq status,
+   Users can define the structure for saving the reconnect information themselves
+   in the userspace.
+
+4. Check if the infomatin sutiable for reconnect

Typos again.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 255

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   If this is reconnect:
+   Before attempting to reconnect, The userspace application need to the
+   ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...

"..." is not good, let's describe the steps in detail.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 258

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   to get the and confirm if these information are suitable for reconnecting.
+
+5. Start the userspace App.

I had a hard time understanding this, does it mean the application needs to restart itself?

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows:

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   While running, the application should store the relevant information about
+   reconnections in mapped pages.
+   When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP

Abbrev is not good here. Let's use "application" for consistency.

Quoted 2 lines

to get vq information, it is necessary
+ to check if this is a reconnection.

How to check whether or not it is reconnected?

Quoted 7 lines

If a reconnection has occurred, the vq-related information

Documentation/userspace-api/vduse.rst +3 -1
+   must be get from the mapped pages.
+
+6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
--

2.43.0

Thanks

author
#9 lore raw

Re: [PATCH v4 5/5] Documentation: Add reconnect process for VDUSE

Quoted 83 lines · Jason Wang
Quoted 14 lines · Cindy Lu

Add a document explaining the reconnect process, including what the Userspace App needs to do and how it works with the kernel.

---
 Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
Documentation/userspace-api/vduse.rst +0 -0
index bdb880e01132..a2be85e0e516 100644

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 235

Quoted 5 lines
Documentation/userspace-api/vduse.rst +2 -0
@@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
    after the used ring is filled.
 
 For more details on the uAPI, please see include/uapi/linux/vduse.h.
+
+HOW VDUSE devices reconnectoin works

Typos, let's use a spell checker.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 247

Quoted 12 lines
Documentation/userspace-api/vduse.rst +12 -0
+----------------
+0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
+   If it does not exist, need to create the instance.goto step 1
+   If it does exist, it means this is a reconnect and goto step 3.
+
+1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
+   /dev/vduse/control.
+
+2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
+   to save the reconnect information.
+
+3. Userspace App need to mmap the pages to userspace

Need to describe what kind of pages need to be mapped. And what's more:

Does this require cooperation from the disconnected application? If yes, how to distinguish from the one that is not cooperative (doesn't support reconnection)?

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 252

Quoted 5 lines
Documentation/userspace-api/vduse.rst +5 -0
+   Userspace App need to map Pages 0 to vq_number for vq status,
+   Users can define the structure for saving the reconnect information themselves
+   in the userspace.
+
+4. Check if the infomatin sutiable for reconnect

Typos again.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 255

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   If this is reconnect:
+   Before attempting to reconnect, The userspace application need to the
+   ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...

"..." is not good, let's describe the steps in detail.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 258

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   to get the and confirm if these information are suitable for reconnecting.
+
+5. Start the userspace App.

I had a hard time understanding this, does it mean the application needs to restart itself?

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows:

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   While running, the application should store the relevant information about
+   reconnections in mapped pages.
+   When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP

Abbrev is not good here. Let's use "application" for consistency.

Quoted 2 lines

to get vq information, it is necessary
+ to check if this is a reconnection.

How to check whether or not it is reconnected?

Quoted 7 lines

If a reconnection has occurred, the vq-related information

Documentation/userspace-api/vduse.rst +3 -1
+   must be get from the mapped pages.
+
+6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
--

2.43.0

Quoted 1 line

Thanks

Thanks Jason, I will rewrite this part and send the new version soon

Thanks
cindy

Quoted 1 line
#10 lore raw

Re: [PATCH v4 5/5] Documentation: Add reconnect process for VDUSE

In reply to Jason Wang
Quoted 82 lines · Jason Wang
Quoted 14 lines · Cindy Lu

Add a document explaining the reconnect process, including what the Userspace App needs to do and how it works with the kernel.

---
 Documentation/userspace-api/vduse.rst | 32 +++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
Documentation/userspace-api/vduse.rst +0 -0
index bdb880e01132..a2be85e0e516 100644

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 235

Quoted 5 lines
Documentation/userspace-api/vduse.rst +2 -0
@@ -231,3 +231,35 @@ able to start the dataplane processing as follows:
    after the used ring is filled.
 
 For more details on the uAPI, please see include/uapi/linux/vduse.h.
+
+HOW VDUSE devices reconnectoin works

Typos, let's use a spell checker.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 247

Quoted 12 lines
Documentation/userspace-api/vduse.rst +12 -0
+----------------
+0. Userspace APP checks if the device /dev/vduse/vduse_name exists.
+   If it does not exist, need to create the instance.goto step 1
+   If it does exist, it means this is a reconnect and goto step 3.
+
+1. Create a new VDUSE instance with ioctl(VDUSE_CREATE_DEV) on
+   /dev/vduse/control.
+
+2. When the ioctl(VDUSE_CREATE_DEV) function is called, the kernel allocates memory
+   to save the reconnect information.
+
+3. Userspace App need to mmap the pages to userspace

Need to describe what kind of pages need to be mapped. And what's more:

Does this require cooperation from the disconnected application? If yes, how to distinguish from the one that is not cooperative (doesn't support reconnection)?

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 252

Quoted 5 lines
Documentation/userspace-api/vduse.rst +5 -0
+   Userspace App need to map Pages 0 to vq_number for vq status,
+   Users can define the structure for saving the reconnect information themselves
+   in the userspace.
+
+4. Check if the infomatin sutiable for reconnect

Typos again.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 255

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   If this is reconnect:
+   Before attempting to reconnect, The userspace application need to the
+   ioctl VDUSE_DEV_GET_CONFIG,VDUSE_DEV_GET_STATUS,VDUSE_DEV_GET_FEATURES...

"..." is not good, let's describe the steps in detail.

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows: · line 258

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   to get the and confirm if these information are suitable for reconnecting.
+
+5. Start the userspace App.

I had a hard time understanding this, does it mean the application needs to restart itself?

Documentation/userspace-api/vduse.rst able to start the dataplane processing as follows:

Quoted 3 lines
Documentation/userspace-api/vduse.rst +3 -0
+   While running, the application should store the relevant information about
+   reconnections in mapped pages.
+   When calling ioctl VDUSE_VQ_GET_INFO from the userspace APP

Abbrev is not good here. Let's use "application" for consistency.

Quoted 2 lines

to get vq information, it is necessary
+ to check if this is a reconnection.

How to check whether or not it is reconnected?

Quoted 7 lines

If a reconnection has occurred, the vq-related information

Documentation/userspace-api/vduse.rst +3 -1
+   must be get from the mapped pages.
+
+6. When the Userspace App exits, it is necessary to unmap all the reconnect pages.
--

2.43.0

Quoted 1 line

Thanks

Are comments only on patch 5? Ack the rest?

Signature · 1 line
MST
#11 lore raw
In reply to Cindy Lu
Quoted 5 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

What is "reconnect"? Not really clear from documentation - it seems to be assumed that reader has an idea but most don't.

Also what's with all the typos? reconnect with 3 nnn s, sutiable and so on. Can you pls run a speller?

Quoted 33 lines

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)
Signature · 1 line
2.43.0
author
#12 lore raw
Quoted 8 lines · Michael S. Tsirkin
Quoted 5 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

Quoted 6 lines

What is "reconnect"? Not really clear from documentation - it seems to be assumed that reader has an idea but most don't.

Also what's with all the typos? reconnect with 3 nnn s, sutiable and so on. Can you pls run a speller?

Thanks a lot, Micheal. I will fix these and also update the speller thanks
Cindy

Quoted 34 lines
Quoted 33 lines

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)

--
2.43.0

#13 lore raw
Quoted 12 lines · Cindy Lu
Quoted 11 lines · Michael S. Tsirkin
Quoted 5 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

What is "reconnect"? Not really clear from documentation - it seems to be assumed that reader has an idea but most don't.

Quoted 6 lines
Quoted 3 lines

Also what's with all the typos? reconnect with 3 nnn s, sutiable and so on. Can you pls run a speller?

Thanks a lot, Micheal. I will fix these and also update the speller thanks
Cindy

Didn't get an updated version, dropped the patch from the pull for this merge window.

Quoted 34 lines
Quoted 34 lines
Quoted 33 lines

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)

--
2.43.0

author
#14 lore raw
Quoted 17 lines · Michael S. Tsirkin
Quoted 15 lines · Cindy Lu
Quoted 14 lines · Michael S. Tsirkin
Quoted 5 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

What is "reconnect"? Not really clear from documentation - it seems to be assumed that reader has an idea but most don't.

Also what's with all the typos? reconnect with 3 nnn s, sutiable and so on. Can you pls run a speller?

Quoted 7 lines
Quoted 3 lines

Thanks a lot, Micheal. I will fix these and also update the speller thanks
Cindy

Didn't get an updated version, dropped the patch from the pull for this merge window.

Hi Micheal
Really apologize for the delay, I was working in an emergency bug, I will provide the updated version soon
apologize again for this mistake
Thanks
Cindy

Quoted 35 lines
Quoted 34 lines
Quoted 34 lines
Quoted 33 lines

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.
2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)

--
2.43.0

#15 lore raw
In reply to Cindy Lu
Quoted 21 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

Quoted 2 lines

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.

So this means the structure (e.g inflight descriptors) are application specific, we can't do cross application reconnection?

Thanks

Quoted 18 lines

2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)

--
2.43.0

author
#16 lore raw
Quoted 23 lines · Jason Wang
Quoted 21 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

Quoted 8 lines
Quoted 2 lines

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.

So this means the structure (e.g inflight descriptors) are application specific, we can't do cross application reconnection?

Thanks

yes, this is also defined by the application itself. so maybe we can't support the
cross-application reconnection
thanks
cindy

Quoted 19 lines
Quoted 18 lines

2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)

--
2.43.0

#17 lore raw
Quoted 31 lines · Cindy Lu
Quoted 29 lines · Jason Wang
Quoted 21 lines · Cindy Lu

Here is the reconnect support in vduse,

Kernel will allocate pages for reconnection.
Userspace needs to use mmap to map the memory to userspace and use these pages to save the reconnect information.

test passd in vduse+dpdk-testpmd

change in V2
1. Address the comments from v1
2. Add the document for reconnect process

change in V3
1. Move the vdpa_vq_state to the uAPI. vduse will use this to synchronize the vq info between the kernel and userspace app.
2. Add a new ioctl VDUSE_DEV_GET_CONFIG. userspace app use this to get config space
3. Rewrite the commit message.
4. Only save the address for the page address and remove the index.
5. remove the ioctl VDUSE_GET_RECONNECT_INFO, userspace app will use uAPI VDUSE_RECONNCT_MMAP_SIZE to mmap
6. Rewrite the document for the reconnect process to make it clearer.

Quoted 2 lines

change in v4
1. Change the number of map pages to VQ numbers. UserSpace APP can define and maintain the structure for saving reconnection information in userspace. The kernel will not maintain this information.

So this means the structure (e.g inflight descriptors) are application specific, we can't do cross application reconnection?

Quoted 7 lines
Quoted 2 lines

Thanks

yes, this is also defined by the application itself. so maybe we can't support the
cross-application reconnection
thanks
cindy

Well, ok. We can probably start from this but let's document this explicitly in the doc patch.

Thanks

Quoted 20 lines
Quoted 19 lines
Quoted 18 lines

2. Rewrite the document for the reconnect process to make it clearer.
3. add the new ioctl for VDUSE_DEV_GET_CONFIG/VDUSE_DEV_GET_STATUS

Cindy Lu (5):
vduse: Add new ioctl VDUSE_DEV_GET_CONFIG
vduse: Add new ioctl VDUSE_DEV_GET_STATUS
vduse: Add function to get/free the pages for reconnection
vduse: Add file operation for mmap
Documentation: Add reconnect process for VDUSE

 Documentation/userspace-api/vduse.rst |  32 +++++++
 drivers/vdpa/vdpa_user/vduse_dev.c    | 125 ++++++++++++++++++++++++++
 include/uapi/linux/vduse.h            |   5 ++
 3 files changed, 162 insertions(+)

--
2.43.0