Skip to contentSkip to conversation

s390: vfio_ap: link the vfio_ap devices to the vfio_ap bus subsystem

PATCH In review Pierre Morel 5 participants 6 messages Feb 21, 2019

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

v3 cover s390: vfio_ap: link the vfio_ap devices to the vfio_ap bus subsystem

The goal of this patch is to standardize the device-driver interface for the VFIO_AP ap_matrix_device to satisfy user-land tools working on hot-plug (UDEV/LIBVIRT).

Christian Borntraeger reported libvirt looping when a matrix device was available before the libvirt start.

Marc Hartmayer debugged this and circumvented this in libvirt:
https://www.redhat.com/archives/libvir-list/2019-February/msg00837.html

Pierre Morel (1):
s390: vfio_ap: link the vfio_ap devices to the vfio_ap bus subsystem

 drivers/s390/crypto/vfio_ap_drv.c     | 44 ++++++++++++++++++++++++++++-------
 drivers/s390/crypto/vfio_ap_ops.c     |  4 ++--
 drivers/s390/crypto/vfio_ap_private.h |  1 +
 3 files changed, 38 insertions(+), 11 deletions(-)
Signature · 1 line
2.7.4
#1 lore raw

PATCH v3 1/1 s390: vfio_ap: link the vfio_ap devices to the vfio_ap bus subsystem

Libudev relies on having a subsystem link for non-root devices. To avoid libudev (and potentially other userspace tools) choking on the matrix device let us introduce a matrix bus and with it the matrix bus subsytem. Also make the matrix device reside within the matrix bus.

Doing this we remove the forced link from the matrix device to the vfio_ap driver and the device_type we do not need anymore.

Since the associated matrix driver is not the vfio_ap driver any more, we have to change the search for the devices on the vfio_ap driver in the function vfio_ap_verify_queue_reserved.

Fixes 1fde573413b5 ("s390: vfio-ap: base implementation of VFIO AP device driver")
---
 drivers/s390/crypto/vfio_ap_drv.c     | 44 ++++++++++++++++++++++++++++-------
 drivers/s390/crypto/vfio_ap_ops.c     |  4 ++--
 drivers/s390/crypto/vfio_ap_private.h |  1 +
 3 files changed, 38 insertions(+), 11 deletions(-)
drivers/s390/crypto/vfio_ap_drv.c +35 -9
index 31c6c84..e9824c3 100644
@@ -15,7 +15,6 @@
 #include "vfio_ap_private.h"
 
 #define VFIO_AP_ROOT_NAME "vfio_ap"
-#define VFIO_AP_DEV_TYPE_NAME "ap_matrix"
 #define VFIO_AP_DEV_NAME "matrix"
 
 MODULE_AUTHOR("IBM Corporation");
@@ -24,10 +23,6 @@ MODULE_LICENSE("GPL v2");
 
 static struct ap_driver vfio_ap_drv;
 
-static struct device_type vfio_ap_dev_type = {
-	.name = VFIO_AP_DEV_TYPE_NAME,
-};
-
 struct ap_matrix_dev *matrix_dev;
 
 /* Only type 10 adapters (CEX4 and later) are supported
@@ -62,6 +57,22 @@ static void vfio_ap_matrix_dev_release(struct device *dev)
 	kfree(matrix_dev);
 }
 
+static int matrix_bus_match(struct device *dev, struct device_driver *drv)
+{
+	return 1;
+}
+
+static struct bus_type matrix_bus = {
+	.name = "matrix",
+	.match = &matrix_bus_match,
+};
+
+static struct device_driver matrix_driver = {
+	.name = "vfio_ap",
+	.bus = &matrix_bus,
+	.suppress_bind_attrs = true,
+};
+
 static int vfio_ap_matrix_dev_create(void)
 {
 	int ret;
@@ -71,6 +82,10 @@ static int vfio_ap_matrix_dev_create(void)
 	if (IS_ERR(root_device))
 		return PTR_ERR(root_device);
 
+	ret = bus_register(&matrix_bus);
+	if (ret)
+		goto bus_register_err;
+
 	matrix_dev = kzalloc(sizeof(*matrix_dev), GFP_KERNEL);
 	if (!matrix_dev) {
 		ret = -ENOMEM;
@@ -87,30 +102,41 @@ static int vfio_ap_matrix_dev_create(void)
 	mutex_init(&matrix_dev->lock);
 	INIT_LIST_HEAD(&matrix_dev->mdev_list);
 
-	matrix_dev->device.type = &vfio_ap_dev_type;
 	dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME);
 	matrix_dev->device.parent = root_device;
+	matrix_dev->device.bus = &matrix_bus;
 	matrix_dev->device.release = vfio_ap_matrix_dev_release;
-	matrix_dev->device.driver = &vfio_ap_drv.driver;
+	matrix_dev->vfio_ap_drv = &vfio_ap_drv;
 
 	ret = device_register(&matrix_dev->device);
 	if (ret)
 		goto matrix_reg_err;
 
+	ret = driver_register(&matrix_driver);
+	if (ret)
+		goto matrix_drv_err;
+
 	return 0;
 
+matrix_drv_err:
+	device_unregister(&matrix_dev->device);
 matrix_reg_err:
 	put_device(&matrix_dev->device);
 matrix_alloc_err:
+	bus_unregister(&matrix_bus);
+bus_register_err:
 	root_device_unregister(root_device);
-
 	return ret;
 }
 
 static void vfio_ap_matrix_dev_destroy(void)
 {
+	struct device *root_device = matrix_dev->device.parent;
+
+	driver_unregister(&matrix_driver);
 	device_unregister(&matrix_dev->device);
-	root_device_unregister(matrix_dev->device.parent);
+	bus_unregister(&matrix_bus);
+	root_device_unregister(root_device);
 }
 
 static int __init vfio_ap_init(void)
drivers/s390/crypto/vfio_ap_ops.c +2 -2
index 272ef42..900b9cf 100644
@@ -198,8 +198,8 @@ static int vfio_ap_verify_queue_reserved(unsigned long *apid,
 	qres.apqi = apqi;
 	qres.reserved = false;
 
-	ret = driver_for_each_device(matrix_dev->device.driver, NULL, &qres,
-				     vfio_ap_has_queue);
+	ret = driver_for_each_device(&matrix_dev->vfio_ap_drv->driver, NULL,
+				     &qres, vfio_ap_has_queue);
 	if (ret)
 		return ret;
 
drivers/s390/crypto/vfio_ap_private.h +1 -0
index 5675492..76b7f98 100644
@@ -40,6 +40,7 @@ struct ap_matrix_dev {
 	struct ap_config_info info;
 	struct list_head mdev_list;
 	struct mutex lock;
+	struct ap_driver  *vfio_ap_drv;
 };
 
 extern struct ap_matrix_dev *matrix_dev;
Signature · 1 line
2.7.4
#2 lore raw
Quoted 9 lines · On Thu, 21 Feb 2019 15:14:54 +0100 Pierre Morel <[email protected]> wrote:

Libudev relies on having a subsystem link for non-root devices. To avoid libudev (and potentially other userspace tools) choking on the matrix device let us introduce a matrix bus and with it the matrix bus subsytem. Also make the matrix device reside within the matrix bus.

Doing this we remove the forced link from the matrix device to the vfio_ap driver and the device_type we do not need anymore.

Quoted 8 lines

Since the associated matrix driver is not the vfio_ap driver any more, we have to change the search for the devices on the vfio_ap driver in the function vfio_ap_verify_queue_reserved.

Fixes 1fde573413b5 ("s390: vfio-ap: base implementation of VFIO AP device driver")

I'd kill the two empty lines.

Quoted 11 lines
---
 drivers/s390/crypto/vfio_ap_drv.c     | 44 ++++++++++++++++++++++++++++-------
 drivers/s390/crypto/vfio_ap_ops.c     |  4 ++--
 drivers/s390/crypto/vfio_ap_private.h |  1 +
 3 files changed, 38 insertions(+), 11 deletions(-)

Otherwise, still looks good.

maintainer
#3 lore raw
Quoted 16 lines · On Thu, 21 Feb 2019 16:09:10 +0100 Cornelia Huck <[email protected]> wrote:
Quoted 13 lines · On Thu, 21 Feb 2019 15:14:54 +0100 Pierre Morel <[email protected]> wrote:

Libudev relies on having a subsystem link for non-root devices. To avoid libudev (and potentially other userspace tools) choking on the matrix device let us introduce a matrix bus and with it the matrix bus subsytem. Also make the matrix device reside within the matrix bus.

Doing this we remove the forced link from the matrix device to the vfio_ap driver and the device_type we do not need anymore.

Since the associated matrix driver is not the vfio_ap driver any more, we have to change the search for the devices on the vfio_ap driver in the function vfio_ap_verify_queue_reserved.

Quoted 6 lines
Quoted 4 lines
Fixes 1fde573413b5 ("s390: vfio-ap: base implementation of VFIO AP device driver")

I'd kill the two empty lines.

The summary/title line is a bit weird. It says link to the vfio_ap bus subsystem, which sounds like establishing a relationship between two pre-existing entities. But this change actually introduces the vfio_bus. I would prefer something like "introduce vfio_ap bus".

I don't feel strongly about it.

Regards,
Halil

Quoted 15 lines
Quoted 7 lines
Quoted 5 lines
---
 drivers/s390/crypto/vfio_ap_drv.c     | 44 ++++++++++++++++++++++++++++-------
 drivers/s390/crypto/vfio_ap_ops.c     |  4 ++--
 drivers/s390/crypto/vfio_ap_private.h |  1 +
 3 files changed, 38 insertions(+), 11 deletions(-)

Otherwise, still looks good.

maintainer Reviewed-by on v3 1/1
#4 lore raw
In reply to Pierre Morel
Quoted 15 lines · Pierre Morel

Libudev relies on having a subsystem link for non-root devices. To avoid libudev (and potentially other userspace tools) choking on the matrix device let us introduce a matrix bus and with it the matrix bus subsytem. Also make the matrix device reside within the matrix bus.

Doing this we remove the forced link from the matrix device to the vfio_ap driver and the device_type we do not need anymore.

Since the associated matrix driver is not the vfio_ap driver any more, we have to change the search for the devices on the vfio_ap driver in the function vfio_ap_verify_queue_reserved.

Fixes 1fde573413b5 ("s390: vfio-ap: base implementation of VFIO AP device driver")
Quoted 7 lines
Quoted 31 lines
---
  drivers/s390/crypto/vfio_ap_drv.c     | 44 ++++++++++++++++++++++++++++-------
  drivers/s390/crypto/vfio_ap_ops.c     |  4 ++--
  drivers/s390/crypto/vfio_ap_private.h |  1 +
  3 files changed, 38 insertions(+), 11 deletions(-)
drivers/s390/crypto/vfio_ap_drv.c +0 -5
index 31c6c84..e9824c3 100644
@@ -15,7 +15,6 @@
  #include "vfio_ap_private.h"
 
  #define VFIO_AP_ROOT_NAME "vfio_ap"
-#define VFIO_AP_DEV_TYPE_NAME "ap_matrix"
  #define VFIO_AP_DEV_NAME "matrix"
 
  MODULE_AUTHOR("IBM Corporation");
@@ -24,10 +23,6 @@ MODULE_LICENSE("GPL v2");
 
  static struct ap_driver vfio_ap_drv;
 
-static struct device_type vfio_ap_dev_type = {
-	.name = VFIO_AP_DEV_TYPE_NAME,
-};
-
  struct ap_matrix_dev *matrix_dev;
 
  /* Only type 10 adapters (CEX4 and later) are supported

drivers/s390/crypto/vfio_ap_drv.c static void vfio_ap_matrix_dev_release(struct device *dev) · line 66

Quoted 10 lines
drivers/s390/crypto/vfio_ap_drv.c +7 -0
@@ -62,6 +57,22 @@ static void vfio_ap_matrix_dev_release(struct device *dev)
  	kfree(matrix_dev);
  }
 
+static int matrix_bus_match(struct device *dev, struct device_driver *drv)
+{
+	return 1;
+}
+
+static struct bus_type matrix_bus = {
+	.name = "matrix",

I'm sorry, I'm late to the game due to being in a time zone six hours behind you all, but my preference would be 'ap_matrix' for the name of the bus. It seems odd to have the bus and the device both named 'matrix'. It's just my opinion and not a show stopper.

Quoted 97 lines
drivers/s390/crypto/vfio_ap_drv.c +28 -4
+	.match = &matrix_bus_match,
+};
+
+static struct device_driver matrix_driver = {
+	.name = "vfio_ap",
+	.bus = &matrix_bus,
+	.suppress_bind_attrs = true,
+};
+
  static int vfio_ap_matrix_dev_create(void)
  {
  	int ret;
@@ -71,6 +82,10 @@ static int vfio_ap_matrix_dev_create(void)
  	if (IS_ERR(root_device))
  		return PTR_ERR(root_device);
 
+	ret = bus_register(&matrix_bus);
+	if (ret)
+		goto bus_register_err;
+
  	matrix_dev = kzalloc(sizeof(*matrix_dev), GFP_KERNEL);
  	if (!matrix_dev) {
  		ret = -ENOMEM;
@@ -87,30 +102,41 @@ static int vfio_ap_matrix_dev_create(void)
  	mutex_init(&matrix_dev->lock);
  	INIT_LIST_HEAD(&matrix_dev->mdev_list);
 
-	matrix_dev->device.type = &vfio_ap_dev_type;
  	dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME);
  	matrix_dev->device.parent = root_device;
+	matrix_dev->device.bus = &matrix_bus;
  	matrix_dev->device.release = vfio_ap_matrix_dev_release;
-	matrix_dev->device.driver = &vfio_ap_drv.driver;
+	matrix_dev->vfio_ap_drv = &vfio_ap_drv;
 
  	ret = device_register(&matrix_dev->device);
  	if (ret)
  		goto matrix_reg_err;
 
+	ret = driver_register(&matrix_driver);
+	if (ret)
+		goto matrix_drv_err;
+
  	return 0;
 
+matrix_drv_err:
+	device_unregister(&matrix_dev->device);
  matrix_reg_err:
  	put_device(&matrix_dev->device);
  matrix_alloc_err:
+	bus_unregister(&matrix_bus);
+bus_register_err:
  	root_device_unregister(root_device);
-
  	return ret;
  }
 
  static void vfio_ap_matrix_dev_destroy(void)
  {
+	struct device *root_device = matrix_dev->device.parent;
+
+	driver_unregister(&matrix_driver);
  	device_unregister(&matrix_dev->device);
-	root_device_unregister(matrix_dev->device.parent);
+	bus_unregister(&matrix_bus);
+	root_device_unregister(root_device);
  }
 
  static int __init vfio_ap_init(void)
drivers/s390/crypto/vfio_ap_ops.c +2 -2
index 272ef42..900b9cf 100644
@@ -198,8 +198,8 @@ static int vfio_ap_verify_queue_reserved(unsigned long *apid,
  	qres.apqi = apqi;
  	qres.reserved = false;
 
-	ret = driver_for_each_device(matrix_dev->device.driver, NULL, &qres,
-				     vfio_ap_has_queue);
+	ret = driver_for_each_device(&matrix_dev->vfio_ap_drv->driver, NULL,
+				     &qres, vfio_ap_has_queue);
  	if (ret)
  		return ret;
 
drivers/s390/crypto/vfio_ap_private.h +1 -0
index 5675492..76b7f98 100644
@@ -40,6 +40,7 @@ struct ap_matrix_dev {
  	struct ap_config_info info;
  	struct list_head mdev_list;
  	struct mutex lock;
+	struct ap_driver  *vfio_ap_drv;
  };
 
  extern struct ap_matrix_dev *matrix_dev;
#5 lore raw
In reply to Pierre Morel
Quoted 15 lines · Pierre Morel

Libudev relies on having a subsystem link for non-root devices. To avoid libudev (and potentially other userspace tools) choking on the matrix device let us introduce a matrix bus and with it the matrix bus subsytem. Also make the matrix device reside within the matrix bus.

Doing this we remove the forced link from the matrix device to the vfio_ap driver and the device_type we do not need anymore.

Since the associated matrix driver is not the vfio_ap driver any more, we have to change the search for the devices on the vfio_ap driver in the function vfio_ap_verify_queue_reserved.

Fixes 1fde573413b5 ("s390: vfio-ap: base implementation of VFIO AP device driver")
Quoted 8 lines

New version still works fine. Please consider the latest comments (e.g. ap_matrix for the bus name, remove the empty lines) and push to Martins tree.

Quoted 137 lines
---
 drivers/s390/crypto/vfio_ap_drv.c     | 44 ++++++++++++++++++++++++++++-------
 drivers/s390/crypto/vfio_ap_ops.c     |  4 ++--
 drivers/s390/crypto/vfio_ap_private.h |  1 +
 3 files changed, 38 insertions(+), 11 deletions(-)
drivers/s390/crypto/vfio_ap_drv.c +35 -9
index 31c6c84..e9824c3 100644
@@ -15,7 +15,6 @@
 #include "vfio_ap_private.h"
 
 #define VFIO_AP_ROOT_NAME "vfio_ap"
-#define VFIO_AP_DEV_TYPE_NAME "ap_matrix"
 #define VFIO_AP_DEV_NAME "matrix"
 
 MODULE_AUTHOR("IBM Corporation");
@@ -24,10 +23,6 @@ MODULE_LICENSE("GPL v2");
 
 static struct ap_driver vfio_ap_drv;
 
-static struct device_type vfio_ap_dev_type = {
-	.name = VFIO_AP_DEV_TYPE_NAME,
-};
-
 struct ap_matrix_dev *matrix_dev;
 
 /* Only type 10 adapters (CEX4 and later) are supported
@@ -62,6 +57,22 @@ static void vfio_ap_matrix_dev_release(struct device *dev)
 	kfree(matrix_dev);
 }
 
+static int matrix_bus_match(struct device *dev, struct device_driver *drv)
+{
+	return 1;
+}
+
+static struct bus_type matrix_bus = {
+	.name = "matrix",
+	.match = &matrix_bus_match,
+};
+
+static struct device_driver matrix_driver = {
+	.name = "vfio_ap",
+	.bus = &matrix_bus,
+	.suppress_bind_attrs = true,
+};
+
 static int vfio_ap_matrix_dev_create(void)
 {
 	int ret;
@@ -71,6 +82,10 @@ static int vfio_ap_matrix_dev_create(void)
 	if (IS_ERR(root_device))
 		return PTR_ERR(root_device);
 
+	ret = bus_register(&matrix_bus);
+	if (ret)
+		goto bus_register_err;
+
 	matrix_dev = kzalloc(sizeof(*matrix_dev), GFP_KERNEL);
 	if (!matrix_dev) {
 		ret = -ENOMEM;
@@ -87,30 +102,41 @@ static int vfio_ap_matrix_dev_create(void)
 	mutex_init(&matrix_dev->lock);
 	INIT_LIST_HEAD(&matrix_dev->mdev_list);
 
-	matrix_dev->device.type = &vfio_ap_dev_type;
 	dev_set_name(&matrix_dev->device, "%s", VFIO_AP_DEV_NAME);
 	matrix_dev->device.parent = root_device;
+	matrix_dev->device.bus = &matrix_bus;
 	matrix_dev->device.release = vfio_ap_matrix_dev_release;
-	matrix_dev->device.driver = &vfio_ap_drv.driver;
+	matrix_dev->vfio_ap_drv = &vfio_ap_drv;
 
 	ret = device_register(&matrix_dev->device);
 	if (ret)
 		goto matrix_reg_err;
 
+	ret = driver_register(&matrix_driver);
+	if (ret)
+		goto matrix_drv_err;
+
 	return 0;
 
+matrix_drv_err:
+	device_unregister(&matrix_dev->device);
 matrix_reg_err:
 	put_device(&matrix_dev->device);
 matrix_alloc_err:
+	bus_unregister(&matrix_bus);
+bus_register_err:
 	root_device_unregister(root_device);
-
 	return ret;
 }
 
 static void vfio_ap_matrix_dev_destroy(void)
 {
+	struct device *root_device = matrix_dev->device.parent;
+
+	driver_unregister(&matrix_driver);
 	device_unregister(&matrix_dev->device);
-	root_device_unregister(matrix_dev->device.parent);
+	bus_unregister(&matrix_bus);
+	root_device_unregister(root_device);
 }
 
 static int __init vfio_ap_init(void)
drivers/s390/crypto/vfio_ap_ops.c +2 -2
index 272ef42..900b9cf 100644
@@ -198,8 +198,8 @@ static int vfio_ap_verify_queue_reserved(unsigned long *apid,
 	qres.apqi = apqi;
 	qres.reserved = false;
 
-	ret = driver_for_each_device(matrix_dev->device.driver, NULL, &qres,
-				     vfio_ap_has_queue);
+	ret = driver_for_each_device(&matrix_dev->vfio_ap_drv->driver, NULL,
+				     &qres, vfio_ap_has_queue);
 	if (ret)
 		return ret;
 
drivers/s390/crypto/vfio_ap_private.h +1 -0
index 5675492..76b7f98 100644
@@ -40,6 +40,7 @@ struct ap_matrix_dev {
 	struct ap_config_info info;
 	struct list_head mdev_list;
 	struct mutex lock;
+	struct ap_driver  *vfio_ap_drv;
 };
 
 extern struct ap_matrix_dev *matrix_dev;