eal: add API for bus close
PATCH v2 1/3 eal: add API for bus close
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API rte_bus_close() for cleanup of bus objects which were acquired during probe.
---
v2:
* Moved rte_bus_close call to rte_eal_cleanup path.
lib/librte_eal/common/eal_common_bus.c | 34 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 ++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++ 4 files changed, 61 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_bus.c +33 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h>
@@ -56,6 +56,38 @@ rte_bus_scan(void)return 0 ; } +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus) { +if (vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )
lib/librte_eal/include/rte_bus.h +24 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_
@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing +* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. *
@@ -248,6 +260,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */
@@ -301,6 +314,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. *
lib/librte_eal/linux/eal.c +1 -0
@@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();
lib/librte_eal/rte_eal_version.map +3 -0
@@ -403,6 +403,9 @@ EXPERIMENTAL { rte_mp_disable; rte_thread_register; rte_thread_unregister; + + # added in 20.11 + rte_bus_close; }; INTERNAL {
Signature · 1 line
2.17.1
PATCH v2 2/3 examples/multi_process: cleanup bus objects while terminating app
From: Rohit Raj <[email protected]>
Certain bus objects may need to be closed and re-acquired while terminating and rerunning the client application. Hence a signal handler is required to catch the termination of the App and hence closing the bus objects.
This patch adds the missing signal handler in the client app and closes the Bus objects in both client and server applications when the signal Handler is called.
--- .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++ .../multi_process/client_server_mp/mp_server/main.c | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-)
examples/multi_process/client_server_mp/mp_client/client.c +11 -0
@@ -11,6 +11,7 @@#include <stdlib.h> #include <getopt.h> #include <string.h> +#include <signal.h> #include <rte_common.h> #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf) } +static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup (); +exit (0 ); +} +/* * Application main function - loops through * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[]) argc -= retval; argv += retval; +signal (SIGINT , signal_handler); +if (parse_app_args (argc, argv) <0 )rte_exit (EXIT_FAILURE ,"Invalid command-line arguments\n" );
examples/multi_process/client_server_mp/mp_server/main.c +3 -1
@@ -275,11 +275,13 @@ signal_handler(int signal) {uint16_t port_id; -if (signal ==SIGINT ) +if (signal ==SIGINT ) {RTE_ETH_FOREACH_DEV (port_id) {rte_eth_dev_stop (port_id);rte_eth_dev_close (port_id); } +rte_eal_cleanup (); + }exit (0 ); }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v2 2/3] examples/multi_process: cleanup bus objects while terminating app
PATCH v2 3/3 bus/fslmc: support bus close API
In reply to [email protected]From: Rohit Raj <[email protected]>
This patch add support for closing the bus objects which were acquired In the bus probe.
Some devices need to be cleaned while in both primary and secondary process and while some need to be cleaned only in case of primary process.
The devices are closed as per the white list used while creating the objects in a particular process.
--- drivers/bus/fslmc/fslmc_bus.c | 15 +++- drivers/bus/fslmc/fslmc_vfio.c | 89 ++++++++++++++++++++- drivers/bus/fslmc/fslmc_vfio.h | 3 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 1 + drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 31 ++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 32 +++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 34 ++++++++ drivers/bus/fslmc/rte_bus_fslmc_version.map | 1 + drivers/bus/fslmc/rte_fslmc.h | 5 +- drivers/event/dpaa2/dpaa2_hw_dpcon.c | 32 +++++++- drivers/net/dpaa2/dpaa2_mux.c | 18 ++++- 11 files changed, 253 insertions(+), 8 deletions(-)
drivers/bus/fslmc/fslmc_bus.c +14 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2018-2019 NXP +* Copyright 2016,2018-2020 NXP * */
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,return 1 ; } +static int +rte_fslmc_close (void ) +{ +int ret =0 ; + + ret =fslmc_vfio_close_group (); +if (ret) +DPAA2_BUS_ERR ("Unable to close devices %d" , ret); + +return 0 ; +} +static int rte_fslmc_probe (void ) {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .close = rte_fslmc_close, .parse = rte_fslmc_parse, .find_device = rte_fslmc_find_device, .get_iommu_class = rte_dpaa2_get_iommu_class,
drivers/bus/fslmc/fslmc_vfio.c +88 -1
107 lines not on this page · raw message
drivers/bus/fslmc/fslmc_vfio.h +2 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,int fslmc_vfio_setup_group (void );int fslmc_vfio_process_group (void ); +int fslmc_vfio_close_group (void );char *fslmc_get_container (void );int fslmc_get_container_group (int *gropuid);int rte_fslmc_vfio_dmamap (void );
drivers/bus/fslmc/mc/fsl_dpcon.h +1 -0
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,int dpcon_id, uint16_t *token); +__rte_internal int dpcon_close(struct fsl_mc_io *mc_io,uint32_t cmd_flags,uint16_t token);
drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +30 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016 NXP +* Copyright 2016,2020 NXP * */
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);static struct dpbp_dev_list dpbp_dev_list =TAILQ_HEAD_INITIALIZER (dpbp_dev_list);/*!< DPBP device list */ +static struct dpaa2_dpbp_dev *get_dpbp_from_id (uint32_t dpbp_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + +/* Get DPBP dev handle from list using index */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { +if (dpbp_dev->dpbp_id == dpbp_id) +break ; + } + +return dpbp_dev; +} + static int dpaa2_create_dpbp_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)return 0 ; } +static void +dpaa2_close_dpbp_device (int object_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + + dpbp_dev =get_dpbp_from_id ((uint32_t )object_id); + +if (dpbp_dev) { +dpaa2_free_dpbp_dev (dpbp_dev); +dpbp_close (&dpbp_dev->dpbp,CMD_PRI_LOW , dpbp_dev->token); +TAILQ_REMOVE (&dpbp_dev_list, dpbp_dev, next); +rte_free (dpbp_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { .dev_type =DPAA2_BPOOL , .create = dpaa2_create_dpbp_device, + .close = dpaa2_close_dpbp_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpbp, rte_dpaa2_dpbp_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpci.c +31 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);static struct dpci_dev_list dpci_dev_list =TAILQ_HEAD_INITIALIZER (dpci_dev_list);/*!< DPCI device list */ +static struct dpaa2_dpci_dev *get_dpci_from_id (uint32_t dpci_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + +/* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { +if (dpci_dev->dpci_id == dpci_id) +break ; + } + +return dpci_dev; +} + static int rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) } } + +static void +rte_dpaa2_close_dpci_device (int object_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + + dpci_dev =get_dpci_from_id ((uint32_t )object_id); + +if (dpci_dev) { +rte_dpaa2_free_dpci_dev (dpci_dev); +dpci_close (&dpci_dev->dpci,CMD_PRI_LOW , dpci_dev->token); +TAILQ_REMOVE (&dpci_dev_list, dpci_dev, next); +rte_free (dpci_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { .dev_type =DPAA2_CI , .create = rte_dpaa2_create_dpci_device, + .close = rte_dpaa2_close_dpci_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpci, rte_dpaa2_dpci_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +34 -0
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;* Cluster 4 (ID = x07) : CPU14, CPU15; */ +static struct dpaa2_dpio_dev *get_dpio_dev_from_id (int32_t dpio_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + +/* Get DPIO dev handle from list using index */ + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { +if (dpio_dev->hw_id == dpio_id) +break ; + } + +return dpio_dev; +} +static int dpaa2_get_core_id (void ) {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)pthread_setspecific (dpaa2_portal_key,NULL ); } +static void +dpaa2_close_dpio_device (int object_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + + dpio_dev =get_dpio_dev_from_id ((int32_t )object_id); + +if (dpio_dev) { +if (dpio_dev->dpio) { +dpio_disable (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +dpio_close (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +rte_free (dpio_dev->dpio); + } +TAILQ_REMOVE (&dpio_dev_list, dpio_dev, next); +rte_free (dpio_dev); + } +} + static int dpaa2_create_dpio_device(int vdev_fd, struct vfio_device_info *obj_info,
@@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { .dev_type =DPAA2_IO , .create = dpaa2_create_dpio_device, + .close = dpaa2_close_dpio_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpio, rte_dpaa2_dpio_obj);
drivers/bus/fslmc/rte_bus_fslmc_version.map +1 -0
@@ -30,6 +30,7 @@ INTERNAL { dpci_get_opr; dpci_set_opr; dpci_set_rx_queue; + dpcon_close; dpcon_get_attributes; dpcon_open; dpdmai_close;
drivers/bus/fslmc/rte_fslmc.h +4 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,struct vfio_device_info *obj_info, int object_id); +typedef void (*rte_dpaa2_obj_close_t )(int object_id); +/** * A structure describing a DPAA2 object. */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {const char *name;/**< Name of Object. */ enum rte_dpaa2_dev_type dev_type;/**< Type of device */ rte_dpaa2_obj_create_t create; +rte_dpaa2_obj_close_t close; };/**
drivers/event/dpaa2/dpaa2_hw_dpcon.c +31 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);static struct dpcon_dev_list dpcon_dev_list =TAILQ_HEAD_INITIALIZER (dpcon_dev_list);/*!< DPCON device list */ +static struct dpaa2_dpcon_dev *get_dpcon_from_id (uint32_t dpcon_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + +/* Get DPCONC dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { +if (dpcon_dev->dpcon_id == dpcon_id) +break ; + } + +return dpcon_dev; +} + static int rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) } } + +static void +rte_dpaa2_close_dpcon_device (int object_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + + dpcon_dev =get_dpcon_from_id ((uint32_t )object_id); + +if (dpcon_dev) { +rte_dpaa2_free_dpcon_dev (dpcon_dev); +dpcon_close (&dpcon_dev->dpcon,CMD_PRI_LOW , dpcon_dev->token); +TAILQ_REMOVE (&dpcon_dev_list, dpcon_dev, next); +rte_free (dpcon_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { .dev_type =DPAA2_CON , .create = rte_dpaa2_create_dpcon_device, + .close = rte_dpaa2_close_dpcon_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpcon, rte_dpaa2_dpcon_obj);
drivers/net/dpaa2/dpaa2_mux.c +17 -1
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id) {struct dpaa2_dpdmux_dev *dpdmux_dev =NULL ; -/* Get DPBP dev handle from list using index */ +/* Get DPDMUX dev handle from list using index */ TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {if (dpdmux_dev->dpdmux_id == dpdmux_id)break ;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,return -1 ; } +static void +dpaa2_close_dpdmux_device (int object_id) +{ +struct dpaa2_dpdmux_dev *dpdmux_dev; + + dpdmux_dev =get_dpdmux_from_id ((uint32_t )object_id); + +if (dpdmux_dev) { +dpdmux_close (&dpdmux_dev->dpdmux,CMD_PRI_LOW , + dpdmux_dev->token); +TAILQ_REMOVE (&dpdmux_dev_list, dpdmux_dev, next); +rte_free (dpdmux_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = { .dev_type =DPAA2_MUX , .create = dpaa2_create_dpdmux_device, + .close = dpaa2_close_dpdmux_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpdmux, rte_dpaa2_dpdmux_obj);
Signature · 1 line
2.17.1
lib/librte_eal/common/eal_common_bus.c rte_bus_scan(void) · line 80
Quoted 2 lines · On Mon, 24 Aug 2020 13:54:12 +0530 [email protected] wrote:
lib/librte_eal/common/eal_common_bus.c +2
-0+if (vbus) { +if (vbus->close) {
Looks good, minor nit.
This could be:
if (vbus && vbus->close) {PATCH v3 1/3 eal: add API for bus close
In reply to [email protected]From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API rte_bus_close() for cleanup of bus objects which were acquired during probe.
---
v3:
* nit: combined nested if statements
v2:
* Moved rte_bus_close call to rte_eal_cleanup path.
lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 +++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++ 4 files changed, 59 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_bus.c +31 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h>
@@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )
lib/librte_eal/include/rte_bus.h +24 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_
@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing +* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. *
@@ -248,6 +260,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */
@@ -301,6 +314,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. *
lib/librte_eal/linux/eal.c +1 -0
@@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();
lib/librte_eal/rte_eal_version.map +3 -0
@@ -397,6 +397,9 @@ EXPERIMENTAL { rte_mp_disable; rte_thread_register; rte_thread_unregister; + + # added in 20.11 + rte_bus_close; }; INTERNAL {
Signature · 1 line
2.17.1
PATCH v3 2/3 examples/multi_process: cleanup bus objects while terminating app
From: Rohit Raj <[email protected]>
Certain bus objects may need to be closed and re-acquired while terminating and rerunning the client application. Hence a signal handler is required to catch the termination of the App and hence closing the bus objects.
This patch adds the missing signal handler in the client app and closes the Bus objects in both client and server applications when the signal Handler is called.
--- .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++ .../multi_process/client_server_mp/mp_server/main.c | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-)
examples/multi_process/client_server_mp/mp_client/client.c +11 -0
@@ -11,6 +11,7 @@#include <stdlib.h> #include <getopt.h> #include <string.h> +#include <signal.h> #include <rte_common.h> #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf) } +static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup (); +exit (0 ); +} +/* * Application main function - loops through * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[]) argc -= retval; argv += retval; +signal (SIGINT , signal_handler); +if (parse_app_args (argc, argv) <0 )rte_exit (EXIT_FAILURE ,"Invalid command-line arguments\n" );
examples/multi_process/client_server_mp/mp_server/main.c +3 -1
@@ -275,11 +275,13 @@ signal_handler(int signal) {uint16_t port_id; -if (signal ==SIGINT ) +if (signal ==SIGINT ) {RTE_ETH_FOREACH_DEV (port_id) {rte_eth_dev_stop (port_id);rte_eth_dev_close (port_id); } +rte_eal_cleanup (); + }exit (0 ); }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v3 2/3] examples/multi_process: cleanup bus objects while terminating app
PATCH v3 3/3 bus/fslmc: support bus close API
In reply to [email protected]From: Rohit Raj <[email protected]>
This patch add support for closing the bus objects which were acquired In the bus probe.
Some devices need to be cleaned while in both primary and secondary process and while some need to be cleaned only in case of primary process.
The devices are closed as per the white list used while creating the objects in a particular process.
--- drivers/bus/fslmc/fslmc_bus.c | 15 +++- drivers/bus/fslmc/fslmc_vfio.c | 89 ++++++++++++++++++++- drivers/bus/fslmc/fslmc_vfio.h | 3 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 1 + drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 31 ++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 32 +++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 34 ++++++++ drivers/bus/fslmc/rte_bus_fslmc_version.map | 1 + drivers/bus/fslmc/rte_fslmc.h | 5 +- drivers/event/dpaa2/dpaa2_hw_dpcon.c | 32 +++++++- drivers/net/dpaa2/dpaa2_mux.c | 18 ++++- 11 files changed, 253 insertions(+), 8 deletions(-)
drivers/bus/fslmc/fslmc_bus.c +14 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2018-2019 NXP +* Copyright 2016,2018-2020 NXP * */
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,return 1 ; } +static int +rte_fslmc_close (void ) +{ +int ret =0 ; + + ret =fslmc_vfio_close_group (); +if (ret) +DPAA2_BUS_ERR ("Unable to close devices %d" , ret); + +return 0 ; +} +static int rte_fslmc_probe (void ) {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .close = rte_fslmc_close, .parse = rte_fslmc_parse, .find_device = rte_fslmc_find_device, .get_iommu_class = rte_dpaa2_get_iommu_class,
drivers/bus/fslmc/fslmc_vfio.c +88 -1
107 lines not on this page · raw message
drivers/bus/fslmc/fslmc_vfio.h +2 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,int fslmc_vfio_setup_group (void );int fslmc_vfio_process_group (void ); +int fslmc_vfio_close_group (void );char *fslmc_get_container (void );int fslmc_get_container_group (int *gropuid);int rte_fslmc_vfio_dmamap (void );
drivers/bus/fslmc/mc/fsl_dpcon.h +1 -0
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,int dpcon_id, uint16_t *token); +__rte_internal int dpcon_close(struct fsl_mc_io *mc_io,uint32_t cmd_flags,uint16_t token);
drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +30 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016 NXP +* Copyright 2016,2020 NXP * */
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);static struct dpbp_dev_list dpbp_dev_list =TAILQ_HEAD_INITIALIZER (dpbp_dev_list);/*!< DPBP device list */ +static struct dpaa2_dpbp_dev *get_dpbp_from_id (uint32_t dpbp_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + +/* Get DPBP dev handle from list using index */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { +if (dpbp_dev->dpbp_id == dpbp_id) +break ; + } + +return dpbp_dev; +} + static int dpaa2_create_dpbp_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)return 0 ; } +static void +dpaa2_close_dpbp_device (int object_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + + dpbp_dev =get_dpbp_from_id ((uint32_t )object_id); + +if (dpbp_dev) { +dpaa2_free_dpbp_dev (dpbp_dev); +dpbp_close (&dpbp_dev->dpbp,CMD_PRI_LOW , dpbp_dev->token); +TAILQ_REMOVE (&dpbp_dev_list, dpbp_dev, next); +rte_free (dpbp_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { .dev_type =DPAA2_BPOOL , .create = dpaa2_create_dpbp_device, + .close = dpaa2_close_dpbp_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpbp, rte_dpaa2_dpbp_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpci.c +31 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);static struct dpci_dev_list dpci_dev_list =TAILQ_HEAD_INITIALIZER (dpci_dev_list);/*!< DPCI device list */ +static struct dpaa2_dpci_dev *get_dpci_from_id (uint32_t dpci_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + +/* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { +if (dpci_dev->dpci_id == dpci_id) +break ; + } + +return dpci_dev; +} + static int rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) } } + +static void +rte_dpaa2_close_dpci_device (int object_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + + dpci_dev =get_dpci_from_id ((uint32_t )object_id); + +if (dpci_dev) { +rte_dpaa2_free_dpci_dev (dpci_dev); +dpci_close (&dpci_dev->dpci,CMD_PRI_LOW , dpci_dev->token); +TAILQ_REMOVE (&dpci_dev_list, dpci_dev, next); +rte_free (dpci_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { .dev_type =DPAA2_CI , .create = rte_dpaa2_create_dpci_device, + .close = rte_dpaa2_close_dpci_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpci, rte_dpaa2_dpci_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +34 -0
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;* Cluster 4 (ID = x07) : CPU14, CPU15; */ +static struct dpaa2_dpio_dev *get_dpio_dev_from_id (int32_t dpio_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + +/* Get DPIO dev handle from list using index */ + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { +if (dpio_dev->hw_id == dpio_id) +break ; + } + +return dpio_dev; +} +static int dpaa2_get_core_id (void ) {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)pthread_setspecific (dpaa2_portal_key,NULL ); } +static void +dpaa2_close_dpio_device (int object_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + + dpio_dev =get_dpio_dev_from_id ((int32_t )object_id); + +if (dpio_dev) { +if (dpio_dev->dpio) { +dpio_disable (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +dpio_close (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +rte_free (dpio_dev->dpio); + } +TAILQ_REMOVE (&dpio_dev_list, dpio_dev, next); +rte_free (dpio_dev); + } +} + static int dpaa2_create_dpio_device(int vdev_fd, struct vfio_device_info *obj_info,
@@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { .dev_type =DPAA2_IO , .create = dpaa2_create_dpio_device, + .close = dpaa2_close_dpio_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpio, rte_dpaa2_dpio_obj);
drivers/bus/fslmc/rte_bus_fslmc_version.map +1 -0
@@ -30,6 +30,7 @@ INTERNAL { dpci_get_opr; dpci_set_opr; dpci_set_rx_queue; + dpcon_close; dpcon_get_attributes; dpcon_open; dpdmai_close;
drivers/bus/fslmc/rte_fslmc.h +4 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,struct vfio_device_info *obj_info, int object_id); +typedef void (*rte_dpaa2_obj_close_t )(int object_id); +/** * A structure describing a DPAA2 object. */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {const char *name;/**< Name of Object. */ enum rte_dpaa2_dev_type dev_type;/**< Type of device */ rte_dpaa2_obj_create_t create; +rte_dpaa2_obj_close_t close; };/**
drivers/event/dpaa2/dpaa2_hw_dpcon.c +31 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);static struct dpcon_dev_list dpcon_dev_list =TAILQ_HEAD_INITIALIZER (dpcon_dev_list);/*!< DPCON device list */ +static struct dpaa2_dpcon_dev *get_dpcon_from_id (uint32_t dpcon_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + +/* Get DPCONC dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { +if (dpcon_dev->dpcon_id == dpcon_id) +break ; + } + +return dpcon_dev; +} + static int rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) } } + +static void +rte_dpaa2_close_dpcon_device (int object_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + + dpcon_dev =get_dpcon_from_id ((uint32_t )object_id); + +if (dpcon_dev) { +rte_dpaa2_free_dpcon_dev (dpcon_dev); +dpcon_close (&dpcon_dev->dpcon,CMD_PRI_LOW , dpcon_dev->token); +TAILQ_REMOVE (&dpcon_dev_list, dpcon_dev, next); +rte_free (dpcon_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { .dev_type =DPAA2_CON , .create = rte_dpaa2_create_dpcon_device, + .close = rte_dpaa2_close_dpcon_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpcon, rte_dpaa2_dpcon_obj);
drivers/net/dpaa2/dpaa2_mux.c +17 -1
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id) {struct dpaa2_dpdmux_dev *dpdmux_dev =NULL ; -/* Get DPBP dev handle from list using index */ +/* Get DPDMUX dev handle from list using index */ TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {if (dpdmux_dev->dpdmux_id == dpdmux_id)break ;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,return -1 ; } +static void +dpaa2_close_dpdmux_device (int object_id) +{ +struct dpaa2_dpdmux_dev *dpdmux_dev; + + dpdmux_dev =get_dpdmux_from_id ((uint32_t )object_id); + +if (dpdmux_dev) { +dpdmux_close (&dpdmux_dev->dpdmux,CMD_PRI_LOW , + dpdmux_dev->token); +TAILQ_REMOVE (&dpdmux_dev_list, dpdmux_dev, next); +rte_free (dpdmux_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = { .dev_type =DPAA2_MUX , .create = dpaa2_create_dpdmux_device, + .close = dpaa2_close_dpdmux_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpdmux, rte_dpaa2_dpdmux_obj);
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v3 3/3] bus/fslmc: support bus close API
Re: [dpdk-dev] [PATCH v3 3/3] bus/fslmc: support bus close API
In reply to [email protected]Quoted 521 lines · On 26/08/2020 06:52, [email protected] wrote:
From: Rohit Raj <[email protected]>
This patch add support for closing the bus objects which were acquired In the bus probe.
Some devices need to be cleaned while in both primary and secondary process and while some need to be cleaned only in case of primary process.
The devices are closed as per the white list used while creating the objects in a particular process.
Signed-off-by Rohit Raj <[email protected]>Acked-by Sachin Saxena <[email protected]>--- drivers/bus/fslmc/fslmc_bus.c | 15 +++- drivers/bus/fslmc/fslmc_vfio.c | 89 ++++++++++++++++++++- drivers/bus/fslmc/fslmc_vfio.h | 3 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 1 + drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 31 ++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 32 +++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 34 ++++++++ drivers/bus/fslmc/rte_bus_fslmc_version.map | 1 + drivers/bus/fslmc/rte_fslmc.h | 5 +- drivers/event/dpaa2/dpaa2_hw_dpcon.c | 32 +++++++- drivers/net/dpaa2/dpaa2_mux.c | 18 ++++- 11 files changed, 253 insertions(+), 8 deletions(-)drivers/bus/fslmc/fslmc_bus.c +14
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2018-2019 NXP +* Copyright 2016,2018-2020 NXP * */ @@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,return 1 ; } +static int +rte_fslmc_close (void ) +{ +int ret =0 ; + + ret =fslmc_vfio_close_group (); +if (ret) +DPAA2_BUS_ERR ("Unable to close devices %d" , ret); + +return 0 ; +} +static int rte_fslmc_probe (void ) {@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .close = rte_fslmc_close, .parse = rte_fslmc_parse, .find_device = rte_fslmc_find_device, .get_iommu_class = rte_dpaa2_get_iommu_class,drivers/bus/fslmc/fslmc_vfio.c +88
-1@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016-2019 NXP +* Copyright 2016-2020 NXP * */ @@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,return -1 ; } +static void +fslmc_close_iodevices (struct rte_dpaa2_device *dev) +{ +struct rte_dpaa2_object *object =NULL ; +struct rte_dpaa2_driver *drv; +int ret, probe_all; + +switch (dev->dev_type) { +case DPAA2_IO : +case DPAA2_CON : +case DPAA2_CI : +case DPAA2_BPOOL : +case DPAA2_MUX : + TAILQ_FOREACH(object, &dpaa2_obj_list, next) { +if (dev->dev_type == object->dev_type) + object->close (dev->object_id); +else +continue ; + } +break ; +case DPAA2_ETH : +case DPAA2_CRYPTO : +case DPAA2_QDMA : + probe_all = rte_fslmc_bus.bus.conf.scan_mode != +RTE_BUS_SCAN_WHITELIST ; + TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) { +if (drv->drv_type != dev->dev_type) +continue ; +if (rte_dev_is_probed (&dev->device)) +continue ; +if (probe_all || + (dev->device.devargs && + dev->device.devargs->policy == +RTE_DEV_WHITELISTED )) { + ret = drv->remove (dev); +if (ret) +DPAA2_BUS_ERR ("Unable to remove" ); + } + } +break ; +default : +break ; + } + +DPAA2_BUS_LOG (DEBUG ,"Device (%s) Closed" , + dev->device.name); +} +/* * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly * EVENT) devices. @@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)return ret; } +int +fslmc_vfio_close_group (void ) +{ +struct rte_dpaa2_device *dev, *dev_temp; + + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) { +if (dev->device.devargs && + dev->device.devargs->policy ==RTE_DEV_BLACKLISTED ) { +DPAA2_BUS_LOG (DEBUG ,"%s Blacklisted, skipping" , + dev->device.name); +TAILQ_REMOVE (&rte_fslmc_bus.device_list, dev, next); +continue ; + } +switch (dev->dev_type) { +case DPAA2_ETH : +case DPAA2_CRYPTO : +case DPAA2_QDMA : +case DPAA2_IO : +fslmc_close_iodevices (dev); +break ; +case DPAA2_CON : +case DPAA2_CI : +case DPAA2_BPOOL : +case DPAA2_MUX : +if (rte_eal_process_type () ==RTE_PROC_SECONDARY ) +continue ; + +fslmc_close_iodevices (dev); +break ; +case DPAA2_DPRTC : +default : +DPAA2_BUS_DEBUG ("Device cannot be closed: Not supported (%s)" , + dev->device.name); + } + } + +return 0 ; +} +int fslmc_vfio_process_group (void ) {drivers/bus/fslmc/fslmc_vfio.h +2
-1@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */ @@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,int fslmc_vfio_setup_group (void );int fslmc_vfio_process_group (void ); +int fslmc_vfio_close_group (void );char *fslmc_get_container (void );int fslmc_get_container_group (int *gropuid);int rte_fslmc_vfio_dmamap (void );drivers/bus/fslmc/mc/fsl_dpcon.h +1
-0@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,int dpcon_id, uint16_t *token); +__rte_internal int dpcon_close(struct fsl_mc_io *mc_io,uint32_t cmd_flags,uint16_t token);drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +30
-1@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016 NXP +* Copyright 2016,2020 NXP * */ @@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);static struct dpbp_dev_list dpbp_dev_list =TAILQ_HEAD_INITIALIZER (dpbp_dev_list);/*!< DPBP device list */ +static struct dpaa2_dpbp_dev *get_dpbp_from_id (uint32_t dpbp_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + +/* Get DPBP dev handle from list using index */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { +if (dpbp_dev->dpbp_id == dpbp_id) +break ; + } + +return dpbp_dev; +} + static int dpaa2_create_dpbp_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)return 0 ; } +static void +dpaa2_close_dpbp_device (int object_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + + dpbp_dev =get_dpbp_from_id ((uint32_t )object_id); + +if (dpbp_dev) { +dpaa2_free_dpbp_dev (dpbp_dev); +dpbp_close (&dpbp_dev->dpbp,CMD_PRI_LOW , dpbp_dev->token); +TAILQ_REMOVE (&dpbp_dev_list, dpbp_dev, next); +rte_free (dpbp_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { .dev_type =DPAA2_BPOOL , .create = dpaa2_create_dpbp_device, + .close = dpaa2_close_dpbp_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpbp, rte_dpaa2_dpbp_obj);drivers/bus/fslmc/portal/dpaa2_hw_dpci.c +31
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */ @@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);static struct dpci_dev_list dpci_dev_list =TAILQ_HEAD_INITIALIZER (dpci_dev_list);/*!< DPCI device list */ +static struct dpaa2_dpci_dev *get_dpci_from_id (uint32_t dpci_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + +/* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { +if (dpci_dev->dpci_id == dpci_id) +break ; + } + +return dpci_dev; +} + static int rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) } } + +static void +rte_dpaa2_close_dpci_device (int object_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + + dpci_dev =get_dpci_from_id ((uint32_t )object_id); + +if (dpci_dev) { +rte_dpaa2_free_dpci_dev (dpci_dev); +dpci_close (&dpci_dev->dpci,CMD_PRI_LOW , dpci_dev->token); +TAILQ_REMOVE (&dpci_dev_list, dpci_dev, next); +rte_free (dpci_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { .dev_type =DPAA2_CI , .create = rte_dpaa2_create_dpci_device, + .close = rte_dpaa2_close_dpci_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpci, rte_dpaa2_dpci_obj);drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +34
-0@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;* Cluster 4 (ID = x07) : CPU14, CPU15; */ +static struct dpaa2_dpio_dev *get_dpio_dev_from_id (int32_t dpio_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + +/* Get DPIO dev handle from list using index */ + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { +if (dpio_dev->hw_id == dpio_id) +break ; + } + +return dpio_dev; +} +static int dpaa2_get_core_id (void ) {@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)pthread_setspecific (dpaa2_portal_key,NULL ); } +static void +dpaa2_close_dpio_device (int object_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + + dpio_dev =get_dpio_dev_from_id ((int32_t )object_id); + +if (dpio_dev) { +if (dpio_dev->dpio) { +dpio_disable (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +dpio_close (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +rte_free (dpio_dev->dpio); + } +TAILQ_REMOVE (&dpio_dev_list, dpio_dev, next); +rte_free (dpio_dev); + } +} + static int dpaa2_create_dpio_device(int vdev_fd, struct vfio_device_info *obj_info,@@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { .dev_type =DPAA2_IO , .create = dpaa2_create_dpio_device, + .close = dpaa2_close_dpio_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpio, rte_dpaa2_dpio_obj);drivers/bus/fslmc/rte_bus_fslmc_version.map +1
-0@@ -30,6 +30,7 @@ INTERNAL { dpci_get_opr; dpci_set_opr; dpci_set_rx_queue; + dpcon_close; dpcon_get_attributes; dpcon_open; dpdmai_close;drivers/bus/fslmc/rte_fslmc.h +4
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */ @@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,struct vfio_device_info *obj_info, int object_id); +typedef void (*rte_dpaa2_obj_close_t )(int object_id); +/** * A structure describing a DPAA2 object. */ @@ -87,6 +89,7 @@ struct rte_dpaa2_object {const char *name;/**< Name of Object. */ enum rte_dpaa2_dev_type dev_type;/**< Type of device */ rte_dpaa2_obj_create_t create; +rte_dpaa2_obj_close_t close; };/** drivers/event/dpaa2/dpaa2_hw_dpcon.c +31
-1@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */ @@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);static struct dpcon_dev_list dpcon_dev_list =TAILQ_HEAD_INITIALIZER (dpcon_dev_list);/*!< DPCON device list */ +static struct dpaa2_dpcon_dev *get_dpcon_from_id (uint32_t dpcon_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + +/* Get DPCONC dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { +if (dpcon_dev->dpcon_id == dpcon_id) +break ; + } + +return dpcon_dev; +} + static int rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) } } + +static void +rte_dpaa2_close_dpcon_device (int object_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + + dpcon_dev =get_dpcon_from_id ((uint32_t )object_id); + +if (dpcon_dev) { +rte_dpaa2_free_dpcon_dev (dpcon_dev); +dpcon_close (&dpcon_dev->dpcon,CMD_PRI_LOW , dpcon_dev->token); +TAILQ_REMOVE (&dpcon_dev_list, dpcon_dev, next); +rte_free (dpcon_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { .dev_type =DPAA2_CON , .create = rte_dpaa2_create_dpcon_device, + .close = rte_dpaa2_close_dpcon_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpcon, rte_dpaa2_dpcon_obj);drivers/net/dpaa2/dpaa2_mux.c +13
-1@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id) {struct dpaa2_dpdmux_dev *dpdmux_dev =NULL ; -/* Get DPBP dev handle from list using index */ +/* Get DPDMUX dev handle from list using index */ TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {if (dpdmux_dev->dpdmux_id == dpdmux_id)break ;@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,return -1 ; } +static void +dpaa2_close_dpdmux_device (int object_id) +{ +struct dpaa2_dpdmux_dev *dpdmux_dev; + + dpdmux_dev =get_dpdmux_from_id ((uint32_t )object_id); + +if (dpdmux_dev) { +dpdmux_close (&dpdmux_dev->dpdmux,CMD_PRI_LOW , + dpdmux_dev->token); +TAILQ_REMOVE (&dpdmux_dev_list, dpdmux_dev, next); +rte_free (dpdmux_dev);
drivers/net/dpaa2/dpaa2_mux.c dpaa2_create_dpdmux_device(int vdev_fd __rte_unused, · line 285
Quoted 10 lines
drivers/net/dpaa2/dpaa2_mux.c +4
-0+ } +} +static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = { .dev_type =DPAA2_MUX , .create = dpaa2_create_dpdmux_device, + .close = dpaa2_close_dpdmux_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpdmux, rte_dpaa2_dpdmux_obj);
-----Original Message-----
Quoted 8 lines
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Wednesday, August 26, 2020 11:23 AM
To: Ray Kinsella <[email protected]>; Neil Horman <[email protected]>
Cc: [email protected]; Rohit Raj <[email protected]>
Subject: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus closeFrom: Rohit Raj <[email protected]>
Quoted 8 lines
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>
Quoted 2 lines
Quoted 1 line
Signed-off-by Rohit Raj <[email protected]>Acked-by Hemant Agrawal <[email protected]>
Please could you look urgently to the close of a port? All NXP drivers are late to migrate to the new behaviour:
http://inbox.dpdk.org/dev/[email protected]/
http://inbox.dpdk.org/dev/80837131.n3tlCtGQ32@thomas/
Quoted 146 lines · On 26/08/2020 06:52, [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>---
v3:
* nit: combined nested if statementsv2:
* Movedrte_bus_closecall torte_eal_cleanuppath.lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 +++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++ 4 files changed, 59 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_bus.c +31
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h> @@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )lib/librte_eal/include/rte_bus.h +24
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing +* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. * @@ -248,6 +260,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */ @@ -301,6 +314,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. * lib/librte_eal/linux/eal.c +1
-0@@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();lib/librte_eal/rte_eal_version.map +0
-0
lib/librte_eal/rte_eal_version.map EXPERIMENTAL { · line 405
Quoted 9 lines
lib/librte_eal/rte_eal_version.map +3
-0@@ -397,6 +397,9 @@ EXPERIMENTAL { rte_mp_disable; rte_thread_register; rte_thread_unregister; + + # added in 20.11 + rte_bus_close; }; INTERNAL {
Putting EXPERITMENAL on such a trivial API feels like overkill, however ...
Quoted 11 lines · On 8/26/2020 6:52 AM, [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>
<...>
Quoted 6 lines
lib/librte_eal/include/rte_bus.h +2
-0@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing
lib/librte_eal/include/rte_bus.h typedef int (*rte_bus_scan_t)(void); · line 81
Quoted 10 lines
lib/librte_eal/include/rte_bus.h +10
-0+* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +
Can you please put more comments to clarify what does "closing devices on bus" mean, what is the responsibility of that function, what is the expected state after that function, etc ?
Since 'pci' & 'vdev' bus implementations are missing, this comment should help them to clarify what exactly needs to be implemented for those buses.
Quoted 133 lines · On 26-Aug-20 11:22 AM, [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>---
v3:
* nit: combined nested if statementsv2:
* Movedrte_bus_closecall torte_eal_cleanuppath.lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 +++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++ 4 files changed, 59 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_bus.c +31
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h> @@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )lib/librte_eal/include/rte_bus.h +24
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing +* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. * @@ -248,6 +260,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */ @@ -301,6 +314,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. * lib/librte_eal/linux/eal.c +0
-0
lib/librte_eal/linux/eal.c rte_eal_cleanup(void) · line 1360
Quoted 4 lines
lib/librte_eal/linux/eal.c +1
-0@@ -1357,6 +1357,7 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); +rte_bus_close ();
Since this is common framework change, we should introduce similar call for "rte_bus_close" in FreeBSD and Windows version also.
Quoted 17 lines
lib/librte_eal/linux/eal.c +0
-0rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();lib/librte_eal/rte_eal_version.map +3
-0@@ -397,6 +397,9 @@ EXPERIMENTAL { rte_mp_disable; rte_thread_register; rte_thread_unregister; + + # added in 20.11 + rte_bus_close; }; INTERNAL {
Quoted 41 lines · On 8/26/2020 6:52 AM, [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>---
v3:
* nit: combined nested if statementsv2:
* Movedrte_bus_closecall torte_eal_cleanuppath.lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 +++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++ 4 files changed, 59 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_bus.c +2
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h> @@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int
lib/librte_eal/common/eal_common_bus.c rte_bus_scan(void) · line 69
Quoted 10 lines
lib/librte_eal/common/eal_common_bus.c +10
-0+rte_bus_close (void) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + }
This special treatment for 'vdev' bus is done in probe to be sure physically device port ids start from '0', I guess we don't need to do this for 'close'.
Quoted 39 lines
lib/librte_eal/common/eal_common_bus.c +19
-0+ +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )lib/librte_eal/include/rte_bus.h +3
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing
lib/librte_eal/include/rte_bus.h typedef int (*rte_bus_scan_t)(void); · line 81
Quoted 10 lines
lib/librte_eal/include/rte_bus.h +10
-0+* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +
As I checked the 'rte_fslmc_bus->close()' ops, it iterates on all devices in the bus instead of doing a bus level close, in that case instead of adding a new 'close' bus operations, will it work if existing 'bus->unplug(dev)' used?
Whatever done in the 'rte_fslmc_bus->close()' per device, can it be done under the 'fslmc_bus_unplug()'?
And in that case a 'rte_bus_remove()' API can be added which can call 'bus->unplug(dev)' for all buses and it will be beneficial for all buses, and it can fit well into the 'rte_eal_cleanup()'.
What do you think?
Quoted 105 lines · Ferruh Yigit
Quoted 39 lines · On 8/26/2020 6:52 AM, [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>---
v3:
* nit: combined nested if statementsv2:
* Movedrte_bus_closecall torte_eal_cleanuppath.
lib/librte_eal/common/eal_common_bus.c| 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 +++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++4 files changed, 59 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_bus.c +2
-1@@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright 2016 NXP + * Copyright 2016,2020 NXP */ #include <stdio.h>@@ -56,6 +56,36 @@ rte_bus_scan(void) return 0; } +int lib/librte_eal/common/eal_common_bus.c rte_bus_scan(void) · line 68
Quoted 10 lines
lib/librte_eal/common/eal_common_bus.c +10
-0+rte_bus_close (void) +{ + int ret; + struct rte_bus *bus, *vbus = NULL; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { + if (!strcmp(bus->name, "vdev")) { + vbus = bus; + continue; + }This special treatment for 'vdev' bus is done in probe to be sure physically device port ids start from '0', I guess we don't need to do this for 'close'.
Quoted 37 lines
lib/librte_eal/common/eal_common_bus.c +19
-0+ + if (bus->close) { + ret = bus->close(); + if (ret) + RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n", + bus->name); + } + } + + if (vbus && vbus->close) { + ret = vbus->close(); + if (ret) + RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n", + vbus->name); + } + + return 0; +} + /* Probe all devices of all buses */int
rte_bus_probe(void)lib/librte_eal/include/rte_bus.h +3
-1@@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause - * Copyright 2016 NXP + * Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t)(void); +/** +* Implementation specific close function which is responsible for closing lib/librte_eal/include/rte_bus.h typedef int (*rte_bus_scan_t)(void); · line 80
Quoted 10 lines
lib/librte_eal/include/rte_bus.h +10
-0+* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +As I checked the '
rte_fslmc_bus->close()' ops, it iterates on all devices in the bus instead of doing a bus level close, in that case instead of adding a new 'close' bus operations, will it work if existing 'bus->unplug(dev)' used?
Whatever done in the 'rte_fslmc_bus->close()' per device, can it be done under the 'fslmc_bus_unplug()'?
Quoted 3 lines
And in that case a '
rte_bus_remove()' API can be added which can call 'bus->unplug(dev)' for all buses and it will be beneficial for all buses, and it can fit well into the 'rte_eal_cleanup()'.What do you think?
Hi Rohit,
I have seen new version has been sent today, I want to remind above question, can you please check it?
Hi Ferruh,
Sorry, I missed you comment last time. I will send an updated patch series after taking care of your suggested changes.
Regards,
Rohit Raj
Software Engineer | NXP India Private Limited
Sent from Outlook Mobile
________________________________
From: Ferruh Yigit <[email protected]>
Sent: Thursday, October 8, 2020 9:54:58 PM
To: Rohit Raj <[email protected]>; Ray Kinsella <[email protected]>; Neil Horman <[email protected]>
Cc: [email protected] <[email protected]>
Subject: [EXT] Re: [dpdk-dev] [PATCH v3 1/3] eal: add API for bus close
Caution: EXT Email
Quoted 110 lines · Ferruh Yigit
Quoted 40 lines · On 8/26/2020 6:52 AM, [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>---
v3:
* nit: combined nested if statementsv2:
* Movedrte_bus_closecall torte_eal_cleanuppath.lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 25 +++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 3 +++ 4 files changed, 59 insertions(+), 2 deletions(-)lib/librte_eal/common/eal_common_bus.c +2
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h> @@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int lib/librte_eal/common/eal_common_bus.c rte_bus_scan(void) · line 68
Quoted 10 lines
lib/librte_eal/common/eal_common_bus.c +10
-0+rte_bus_close (void) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + }This special treatment for 'vdev' bus is done in probe to be sure physically device port ids start from '0', I guess we don't need to do this for 'close'.
Quoted 37 lines
lib/librte_eal/common/eal_common_bus.c +19
-0+ +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )lib/librte_eal/include/rte_bus.h +3
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_@@ -67,6 +67,18 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for closing lib/librte_eal/include/rte_bus.h typedef int (*rte_bus_scan_t)(void); · line 80
Quoted 10 lines
lib/librte_eal/include/rte_bus.h +10
-0+* devices on that bus. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +As I checked the '
rte_fslmc_bus->close()' ops, it iterates on all devices in the bus instead of doing a bus level close, in that case
instead of adding a new 'close' bus operations, will it work if existing 'bus->unplug(dev)' used?
Whatever done in the 'rte_fslmc_bus->close()' per device, can it be done under the 'fslmc_bus_unplug()'?
Quoted 5 lines
And in that case a '
rte_bus_remove()' API can be added which can call 'bus->unplug(dev)' for all buses and it will be beneficial for all buses, and it can fit well into the 'rte_eal_cleanup()'.What do you think?
Hi Rohit,
I have seen new version has been sent today, I want to remind above question, can you please check it?
Hello Rohit,
Quoted 1 line · Rohit Raj
Sorry, I missed you comment last time. I will send an updated patch series after taking care of your suggested changes.
Added few more comments.
Waiting for a respin.
Thanks.
Signature · 1 line
David Marchand
PATCH v4 1/5 eal: add API for bus close
In reply to [email protected]From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API rte_bus_close() for cleanup of bus objects which were acquired during probe.
--- lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 27 +++++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 1 + 4 files changed, 59 insertions(+), 2 deletions(-)
lib/librte_eal/common/eal_common_bus.c +31 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h>
@@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Probe all devices of all buses */ int rte_bus_probe (void )
lib/librte_eal/include/rte_bus.h +26 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_
@@ -67,6 +67,20 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for resetting all +* detected devices on the bus to a default state, closing UIO nodes or VFIO +* groups and also freeing any memory allocated during rte_bus_probe like +* private resources for device list. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. *
@@ -248,6 +262,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */
@@ -301,6 +316,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. *
lib/librte_eal/linux/eal.c +1 -0
@@ -1358,6 +1358,7 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();
lib/librte_eal/rte_eal_version.map +1 -0
@@ -400,6 +400,7 @@ EXPERIMENTAL { # added in 20.11 __rte_eal_trace_generic_size_t; rte_service_lcore_may_be_active; + rte_bus_close; }; INTERNAL {
Signature · 1 line
2.17.1
PATCH v4 2/5 examples/multi_process: cleanup bus objects while terminating app
From: Rohit Raj <[email protected]>
Certain bus objects may need to be closed and re-acquired while terminating and rerunning the client application. Hence a signal handler is required to catch the termination of the App and hence closing the bus objects.
This patch adds the missing signal handler in the client app and closes the Bus objects in both client and server applications when the signal Handler is called.
--- .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++ .../multi_process/client_server_mp/mp_server/main.c | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-)
examples/multi_process/client_server_mp/mp_client/client.c +11 -0
@@ -11,6 +11,7 @@#include <stdlib.h> #include <getopt.h> #include <string.h> +#include <signal.h> #include <rte_common.h> #include <rte_malloc.h>
@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf) } +static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup (); +exit (0 ); +} +/* * Application main function - loops through * receiving and processing packets. Never returns
@@ -217,6 +226,8 @@ main(int argc, char *argv[]) argc -= retval; argv += retval; +signal (SIGINT , signal_handler); +if (parse_app_args (argc, argv) <0 )rte_exit (EXIT_FAILURE ,"Invalid command-line arguments\n" );
examples/multi_process/client_server_mp/mp_server/main.c +3 -1
@@ -275,11 +275,13 @@ signal_handler(int signal) {uint16_t port_id; -if (signal ==SIGINT ) +if (signal ==SIGINT ) {RTE_ETH_FOREACH_DEV (port_id) {rte_eth_dev_stop (port_id);rte_eth_dev_close (port_id); } +rte_eal_cleanup (); + }exit (0 ); }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app
Quoted 31 lines · On Thu, Oct 8, 2020 at 5:31 PM <[email protected]> wrote:
From: Rohit Raj <[email protected]>
Certain bus objects may need to be closed and re-acquired while terminating and rerunning the client application. Hence a signal handler is required to catch the termination of the App and hence closing the bus objects.
This patch adds the missing signal handler in the client app and closes the Bus objects in both client and server applications when the signal Handler is called.
Signed-off-by Rohit Raj <[email protected]>--- .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++ .../multi_process/client_server_mp/mp_server/main.c | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-)examples/multi_process/client_server_mp/mp_client/client.c +1
-0@@ -11,6 +11,7 @@#include <stdlib.h> #include <getopt.h> #include <string.h> +#include <signal.h> #include <rte_common.h> #include <rte_malloc.h>
examples/multi_process/client_server_mp/mp_client/client.c handle_packet(struct rte_mbuf *buf) · line 206
Quoted 10 lines
examples/multi_process/client_server_mp/mp_client/client.c +7
-0@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf) } +static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup (); +exit (0 ); +}
Calling rte_eal_cleanup from a signal handler is a bad idea.
In most cases, you are racing with other threads still using DPDK resources.
https://git.dpdk.org/dpdk/commit?id=2c434431f4
https://git.dpdk.org/dpdk/commit?id=613ce6691c
This might not be a problem in this multi_process example, but let's keep a consistent way across all examples.
Thanks.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app
Quoted 48 lines · David Marchand
Quoted 31 lines · On Thu, Oct 8, 2020 at 5:31 PM <[email protected]> wrote:
From: Rohit Raj <[email protected]>
Certain bus objects may need to be closed and re-acquired while terminating and rerunning the client application. Hence a signal handler is required to catch the termination of the App and hence closing the bus objects.
This patch adds the missing signal handler in the client app and closes the Bus objects in both client and server applications when the signal Handler is called.
Signed-off-by Rohit Raj <[email protected]>--- .../multi_process/client_server_mp/mp_client/client.c | 11 +++++++++++ .../multi_process/client_server_mp/mp_server/main.c | 4 +++- 2 files changed, 14 insertions(+), 1 deletion(-)examples/multi_process/client_server_mp/mp_client/client.c +1
-0@@ -11,6 +11,7 @@#include <stdlib.h> #include <getopt.h> #include <string.h> +#include <signal.h> #include <rte_common.h> #include <rte_malloc.h> examples/multi_process/client_server_mp/mp_client/client.c handle_packet(struct rte_mbuf *buf) · line 206
Quoted 10 lines
examples/multi_process/client_server_mp/mp_client/client.c +7
-0@@ -196,6 +197,14 @@ handle_packet(struct rte_mbuf *buf) } +static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup (); +exit (0 ); +}Calling
rte_eal_cleanupfrom a signal handler is a bad idea.
In most cases, you are racing with other threads still using DPDK resources.
https://git.dpdk.org/dpdk/commit?id=2c434431f4
https://git.dpdk.org/dpdk/commit?id=613ce6691c
Quoted 3 lines
This might not be a problem in this
multi_processexample, but let's keep a consistent way across all examples.
Thanks.
Hi,
I want to know why you don't think he's a problem. recently, we use
the patch
https://patchwork.dpdk.org/patch/86988/
startup with multiprocess. Use the tester to send traffic and kill the slave process repeatedly.
The coredump exception occurs. According to my analysis, the cause is that resources are not released after the slave process is killed.
Thanks
Lijun Ou
Quoted 1 line
Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app
Quoted 16 lines · oulijun
Quoted 13 lines
examples/multi_process/client_server_mp/mp_client/client.c handle_packet(struct rte_mbuf *buf) · line 206
Quoted 7 lines
examples/multi_process/client_server_mp/mp_client/client.c +7
-0+static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup (); +exit (0 ); +}Calling
rte_eal_cleanupfrom a signal handler is a bad idea.
In most cases, you are racing with other threads still using DPDK resources.
https://git.dpdk.org/dpdk/commit?id=2c434431f4
https://git.dpdk.org/dpdk/commit?id=613ce6691cQuoted 3 lines
This might not be a problem in this
multi_processexample, but let's keep a consistent way across all examples.
Thanks.
Quoted 9 lines
Hi,
I want to know why you don't think he's a problem. recently, we use
the patch
https://patchwork.dpdk.org/patch/86988/
startup with multiprocess. Use the tester to send traffic and kill the slave process repeatedly.
The coredump exception occurs. According to my analysis, the cause is that resources are not released after the slave process is killed.
Not sure I get your question and I don't see the relation with the testpmd patch.
I did not say we must not release resources.
Just to rephrase my previous mail:
Generally speaking, calling rte_eal_cleanup() from a signal handler is wrong since it creates races with other threads.
I recommend putting in place a synchronisation mechanism so that all worker threads break out of their processing loop and the main thread synchronizes/joins with them before calling rte_eal_cleanup() in its exit path.
Now, for this patch, in examples/multi_process/client_server_mp/mp_client/client.c, this secondary process code seems to only have one thread (but I might be wrong).
If this is true, then no race in theory => my comment "This might not be a problem in this multi_process example".
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v4 2/5] examples/multi_process: cleanup bus objects while terminating app
In reply to [email protected]examples/multi_process/client_server_mp/mp_client/client.c handle_packet(struct rte_mbuf *buf) · line 204
Quoted 5 lines · On Thu, 8 Oct 2020 21:00:44 +0530 [email protected] wrote:
examples/multi_process/client_server_mp/mp_client/client.c +5
-0+static void +signal_handler (int signal) +{ +if (signal ==SIGINT ) +rte_eal_cleanup ();
NAK
Call rte_eal_cleanup in signal handler is not safe.
Need to set a flag and handle it in main code.
PATCH v4 3/5 bus/fslmc: support bus close API
In reply to [email protected]From: Rohit Raj <[email protected]>
This patch add support for closing the bus objects which were acquired In the bus probe.
Some devices need to be cleaned while in both primary and secondary process and while some need to be cleaned only in case of primary process.
The devices are closed as per the white list used while creating the objects in a particular process.
--- drivers/bus/fslmc/fslmc_bus.c | 15 +++- drivers/bus/fslmc/fslmc_vfio.c | 89 ++++++++++++++++++++- drivers/bus/fslmc/fslmc_vfio.h | 3 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 1 + drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 31 ++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 32 +++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 34 ++++++++ drivers/bus/fslmc/rte_bus_fslmc_version.map | 1 + drivers/bus/fslmc/rte_fslmc.h | 5 +- drivers/event/dpaa2/dpaa2_hw_dpcon.c | 32 +++++++- drivers/net/dpaa2/dpaa2_mux.c | 18 ++++- 11 files changed, 253 insertions(+), 8 deletions(-)
drivers/bus/fslmc/fslmc_bus.c +14 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2018-2019 NXP +* Copyright 2016,2018-2020 NXP * */
@@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,return 1 ; } +static int +rte_fslmc_close (void ) +{ +int ret =0 ; + + ret =fslmc_vfio_close_group (); +if (ret) +DPAA2_BUS_ERR ("Unable to close devices %d" , ret); + +return 0 ; +} +static int rte_fslmc_probe (void ) {
@@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .close = rte_fslmc_close, .parse = rte_fslmc_parse, .find_device = rte_fslmc_find_device, .get_iommu_class = rte_dpaa2_get_iommu_class,
drivers/bus/fslmc/fslmc_vfio.c +88 -1
107 lines not on this page · raw message
drivers/bus/fslmc/fslmc_vfio.h +2 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,int fslmc_vfio_setup_group (void );int fslmc_vfio_process_group (void ); +int fslmc_vfio_close_group (void );char *fslmc_get_container (void );int fslmc_get_container_group (int *gropuid);int rte_fslmc_vfio_dmamap (void );
drivers/bus/fslmc/mc/fsl_dpcon.h +1 -0
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,int dpcon_id, uint16_t *token); +__rte_internal int dpcon_close(struct fsl_mc_io *mc_io,uint32_t cmd_flags,uint16_t token);
drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +30 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016 NXP +* Copyright 2016,2020 NXP * */
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);static struct dpbp_dev_list dpbp_dev_list =TAILQ_HEAD_INITIALIZER (dpbp_dev_list);/*!< DPBP device list */ +static struct dpaa2_dpbp_dev *get_dpbp_from_id (uint32_t dpbp_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + +/* Get DPBP dev handle from list using index */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { +if (dpbp_dev->dpbp_id == dpbp_id) +break ; + } + +return dpbp_dev; +} + static int dpaa2_create_dpbp_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)return 0 ; } +static void +dpaa2_close_dpbp_device (int object_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + + dpbp_dev =get_dpbp_from_id ((uint32_t )object_id); + +if (dpbp_dev) { +dpaa2_free_dpbp_dev (dpbp_dev); +dpbp_close (&dpbp_dev->dpbp,CMD_PRI_LOW , dpbp_dev->token); +TAILQ_REMOVE (&dpbp_dev_list, dpbp_dev, next); +rte_free (dpbp_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { .dev_type =DPAA2_BPOOL , .create = dpaa2_create_dpbp_device, + .close = dpaa2_close_dpbp_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpbp, rte_dpaa2_dpbp_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpci.c +31 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);static struct dpci_dev_list dpci_dev_list =TAILQ_HEAD_INITIALIZER (dpci_dev_list);/*!< DPCI device list */ +static struct dpaa2_dpci_dev *get_dpci_from_id (uint32_t dpci_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + +/* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { +if (dpci_dev->dpci_id == dpci_id) +break ; + } + +return dpci_dev; +} + static int rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) } } + +static void +rte_dpaa2_close_dpci_device (int object_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + + dpci_dev =get_dpci_from_id ((uint32_t )object_id); + +if (dpci_dev) { +rte_dpaa2_free_dpci_dev (dpci_dev); +dpci_close (&dpci_dev->dpci,CMD_PRI_LOW , dpci_dev->token); +TAILQ_REMOVE (&dpci_dev_list, dpci_dev, next); +rte_free (dpci_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { .dev_type =DPAA2_CI , .create = rte_dpaa2_create_dpci_device, + .close = rte_dpaa2_close_dpci_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpci, rte_dpaa2_dpci_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +34 -0
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;* Cluster 4 (ID = x07) : CPU14, CPU15; */ +static struct dpaa2_dpio_dev *get_dpio_dev_from_id (int32_t dpio_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + +/* Get DPIO dev handle from list using index */ + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { +if (dpio_dev->hw_id == dpio_id) +break ; + } + +return dpio_dev; +} +static int dpaa2_get_core_id (void ) {
@@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)pthread_setspecific (dpaa2_portal_key,NULL ); } +static void +dpaa2_close_dpio_device (int object_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + + dpio_dev =get_dpio_dev_from_id ((int32_t )object_id); + +if (dpio_dev) { +if (dpio_dev->dpio) { +dpio_disable (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +dpio_close (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +rte_free (dpio_dev->dpio); + } +TAILQ_REMOVE (&dpio_dev_list, dpio_dev, next); +rte_free (dpio_dev); + } +} + static int dpaa2_create_dpio_device(int vdev_fd, struct vfio_device_info *obj_info,
@@ -628,6 +661,7 @@ dpaa2_free_eq_descriptors(void)static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { .dev_type =DPAA2_IO , .create = dpaa2_create_dpio_device, + .close = dpaa2_close_dpio_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpio, rte_dpaa2_dpio_obj);
drivers/bus/fslmc/rte_bus_fslmc_version.map +1 -0
@@ -30,6 +30,7 @@ INTERNAL { dpci_get_opr; dpci_set_opr; dpci_set_rx_queue; + dpcon_close; dpcon_get_attributes; dpcon_open; dpdmai_close;
drivers/bus/fslmc/rte_fslmc.h +4 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2019 NXP +* Copyright 2016,2019-2020 NXP * */
@@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,struct vfio_device_info *obj_info, int object_id); +typedef void (*rte_dpaa2_obj_close_t )(int object_id); +/** * A structure describing a DPAA2 object. */
@@ -87,6 +89,7 @@ struct rte_dpaa2_object {const char *name;/**< Name of Object. */ enum rte_dpaa2_dev_type dev_type;/**< Type of device */ rte_dpaa2_obj_create_t create; +rte_dpaa2_obj_close_t close; };/**
drivers/event/dpaa2/dpaa2_hw_dpcon.c +31 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2020 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);static struct dpcon_dev_list dpcon_dev_list =TAILQ_HEAD_INITIALIZER (dpcon_dev_list);/*!< DPCON device list */ +static struct dpaa2_dpcon_dev *get_dpcon_from_id (uint32_t dpcon_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + +/* Get DPCONC dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { +if (dpcon_dev->dpcon_id == dpcon_id) +break ; + } + +return dpcon_dev; +} + static int rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) } } + +static void +rte_dpaa2_close_dpcon_device (int object_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + + dpcon_dev =get_dpcon_from_id ((uint32_t )object_id); + +if (dpcon_dev) { +rte_dpaa2_free_dpcon_dev (dpcon_dev); +dpcon_close (&dpcon_dev->dpcon,CMD_PRI_LOW , dpcon_dev->token); +TAILQ_REMOVE (&dpcon_dev_list, dpcon_dev, next); +rte_free (dpcon_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { .dev_type =DPAA2_CON , .create = rte_dpaa2_create_dpcon_device, + .close = rte_dpaa2_close_dpcon_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpcon, rte_dpaa2_dpcon_obj);
drivers/net/dpaa2/dpaa2_mux.c +17 -1
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id) {struct dpaa2_dpdmux_dev *dpdmux_dev =NULL ; -/* Get DPBP dev handle from list using index */ +/* Get DPDMUX dev handle from list using index */ TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {if (dpdmux_dev->dpdmux_id == dpdmux_id)break ;
@@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,return -1 ; } +static void +dpaa2_close_dpdmux_device (int object_id) +{ +struct dpaa2_dpdmux_dev *dpdmux_dev; + + dpdmux_dev =get_dpdmux_from_id ((uint32_t )object_id); + +if (dpdmux_dev) { +dpdmux_close (&dpdmux_dev->dpdmux,CMD_PRI_LOW , + dpdmux_dev->token); +TAILQ_REMOVE (&dpdmux_dev_list, dpdmux_dev, next); +rte_free (dpdmux_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = { .dev_type =DPAA2_MUX , .create = dpaa2_create_dpdmux_device, + .close = dpaa2_close_dpdmux_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpdmux, rte_dpaa2_dpdmux_obj);
Signature · 1 line
2.17.1
PATCH v4 4/5 eal/freebsd: added support for rte_bus_close API
In reply to [email protected]From: Rohit Raj <[email protected]>
Added support for rte_bus_close API on freebsd to reset the devices on the bus to default state.
--- lib/librte_eal/freebsd/eal.c | 1 + 1 file changed, 1 insertion(+)
lib/librte_eal/freebsd/eal.c +1 -0
@@ -962,6 +962,7 @@ rte_eal_cleanup(void) {struct internal_config *internal_conf =eal_get_internal_configuration (); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();
Signature · 1 line
2.17.1
PATCH v4 5/5 eal/windows: added support for rte_bus_close API
In reply to [email protected]From: Rohit Raj <[email protected]>
Added support for rte_bus_close API on windows to reset the devices on the bus to default state.
--- lib/librte_eal/windows/eal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/librte_eal/windows/eal.c +1 -1
@@ -251,7 +251,7 @@ rte_eal_cleanup(void) {struct internal_config *internal_conf =eal_get_internal_configuration (); - +rte_bus_close ();eal_cleanup_config (internal_conf);return 0 ; }
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 5/5] eal/windows: added support for rte_bus_close API
Quoted 29 lines
Subject: [dpdk-dev] [PATCH v4 5/5] eal/windows: added support for
rte_bus_closeAPIExternal email: Use caution opening links or attachments
From: Rohit Raj <[email protected]>
Added support for
rte_bus_closeAPI on windows to reset the devices on the bus to default state.Signed-off-by Rohit Raj <[email protected]>--- lib/librte_eal/windows/eal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)lib/librte_eal/windows/eal.c index renamed from lib/librte_eal/windows/eal.c +0
-0
bc48f27ab..9eb32077b100644lib/librte_eal/windows/eal.c +1
-2@@ -251,7 +251,7 @@ rte_eal_cleanup(void) {struct internal_config *internal_conf =eal_get_internal_configuration (); - +rte_bus_close ();eal_cleanup_config (internal_conf);return 0 ; } --
Quoted 1 line
2.17.1
Hi Rohit,
You need to add rte_bus_close to lib/librte_eal/rte_eal_exports.def as it is currently the way to export functions from eal in Windows clang build.
Thanks,
Tal.
Re: [dpdk-dev] [PATCH v4 5/5] eal/windows: added support for rte_bus_close API
In reply to [email protected]Quoted 15 lines · On Thu, Oct 08, 2020 at 09:00:47PM +0530, [email protected] wrote:
From: Rohit Raj <[email protected]>
Added support for
rte_bus_closeAPI on windows to reset the devices on the bus to default state.Signed-off-by Rohit Raj <[email protected]>--- lib/librte_eal/windows/eal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)lib/librte_eal/windows/eal.c +0
-0
lib/librte_eal/windows/eal.c rte_eal_cleanup(void) · line 254
Quoted 5 lines
lib/librte_eal/windows/eal.c +1
-1@@ -251,7 +251,7 @@ rte_eal_cleanup(void) {struct internal_config *internal_conf =eal_get_internal_configuration (); - +rte_bus_close ();
Should we verify the return code here?
Quoted 3 lines
lib/librte_eal/windows/eal.c +0
-0eal_cleanup_config (internal_conf);return 0 ; }
Quoted 11 lines · On Thu, Oct 8, 2020 at 5:31 PM <[email protected]> wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.
Quoted 7 lines
Signed-off-by Rohit Raj <[email protected]>--- lib/librte_eal/common/eal_common_bus.c | 32 +++++++++++++++++++++++++- lib/librte_eal/include/rte_bus.h | 27 +++++++++++++++++++++- lib/librte_eal/linux/eal.c | 1 + lib/librte_eal/rte_eal_version.map | 1 + 4 files changed, 59 insertions(+), 2 deletions(-)
This is a new EAL API, please update the release notes.
Quoted 34 lines
lib/librte_eal/common/eal_common_bus.c +19
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #include <stdio.h> @@ -56,6 +56,36 @@ rte_bus_scan(void)return 0 ; } +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + }
lib/librte_eal/common/eal_common_bus.c rte_bus_scan(void) · line 86
Quoted 10 lines
lib/librte_eal/common/eal_common_bus.c +10
-0+ } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ;
We should propagate that an error occurred so that the caller is aware something went wrong.
Quoted 46 lines
lib/librte_eal/common/eal_common_bus.c +2
-0+} +/* Probe all devices of all buses */ int rte_bus_probe (void )lib/librte_eal/include/rte_bus.h +16
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2020 NXP */ #ifndef _RTE_BUS_H_@@ -67,6 +67,20 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* Implementation specific close function which is responsible for resetting all +* detected devices on the bus to a default state, closing UIO nodes or VFIO +* groups and also freeing any memory allocated during rte_bus_probe like +* private resources for device list. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. * @@ -248,6 +262,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */
lib/librte_eal/include/rte_bus.h int rte_bus_scan(void); · line 319
Quoted 4 lines
lib/librte_eal/include/rte_bus.h +1
-0@@ -301,6 +316,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/**
Missing a banner:
+ * @warning
+ * @b EXPERIMENTAL: this API may change, or be removed, without prior notice
Quoted 17 lines
lib/librte_eal/include/rte_bus.h +9
-0+* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. * lib/librte_eal/linux/eal.c +0
-0
lib/librte_eal/linux/eal.c rte_eal_cleanup(void) · line 1364
Quoted 7 lines
lib/librte_eal/linux/eal.c +1
-0@@ -1358,6 +1358,7 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();rte_trace_save ();
You can squash patch 4 and patch 5 in this patch so that we update all OS-specific bits at once.
lib/librte_eal/rte_eal_version.map EXPERIMENTAL { · line 403
Quoted 9 lines
lib/librte_eal/rte_eal_version.map +1
-0@@ -400,6 +400,7 @@ EXPERIMENTAL { # added in 20.11 __rte_eal_trace_generic_size_t; rte_service_lcore_may_be_active; + rte_bus_close;
Please, alphabetical order.
Quoted 6 lines
lib/librte_eal/rte_eal_version.map +0
-1}; INTERNAL { --2.17.1
Signature · 1 line
David Marchand
PATCH v5 1/2 eal: add API for bus close
In reply to [email protected]From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API rte_bus_close() for cleanup of bus objects which were acquired during probe.
---
Rebased on this patch series:
https://patches.dpdk.org/project/dpdk/list/?series=21049
v5:
* Updated release notes for new feature and API change.
* Added support for error checking while closing bus.
* Added experimental banner for new API.
* Squashed changes related to freebsd and windows into single patch.
* Discarded patch to fix a bug which is already fixed on latest release.
v4:
* Added comments to clarify responsibility of rte_bus_close.
* Added support for rte_bus_close on freebsd.
* Added support for rte_bus_close on windows.
v3:
* nit: combined nested if statements.
v2:
* Moved rte_bus_close call to rte_eal_cleanup path.
doc/guides/rel_notes/release_22_03.rst | 8 +++++++ lib/eal/common/eal_common_bus.c | 33 +++++++++++++++++++++++++- lib/eal/freebsd/eal.c | 1 + lib/eal/include/rte_bus.h | 30 ++++++++++++++++++++++- lib/eal/linux/eal.c | 8 +++++++ lib/eal/version.map | 3 +++ lib/eal/windows/eal.c | 1 + 7 files changed, 82 insertions(+), 2 deletions(-)
doc/guides/rel_notes/release_22_03.rst +8 -0
@@ -55,6 +55,11 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= + * **Added support to close bus.** + + Added capability to allow a user to do cleanup of bus objects which + were acquired during bus probe. + Removed Items -------------
@@ -84,6 +89,9 @@ API Changes Also, make sure to start the actual text at the margin. ======================================================= + * eal: Added new API ``rte_bus_close`` to perform cleanup bus objects which + were acquired during bus probe. + ABI Changes -----------
lib/eal/common/eal_common_bus.c +32 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2022 NXP */ #include <stdio.h>
@@ -85,6 +85,37 @@ rte_bus_probe(void)return 0 ; } +/* Close all devices of all buses */ +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + } + +return 0 ; +} +/* Dump information of a single bus */ static int bus_dump_one (FILE *f,struct rte_bus *bus)
lib/eal/freebsd/eal.c +1 -0
@@ -984,6 +984,7 @@ rte_eal_cleanup(void) {struct internal_config *internal_conf =eal_get_internal_configuration (); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();/* after this point, any DPDK pointers will become dangling */
lib/eal/include/rte_bus.h +29 -1
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2022 NXP */ #ifndef _RTE_BUS_H_
@@ -66,6 +66,23 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Implementation specific close function which is responsible for resetting all +* detected devices on the bus to a default state, closing UIO nodes or VFIO +* groups and also freeing any memory allocated during rte_bus_probe like +* private resources for device list. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. *
@@ -263,6 +280,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */
@@ -317,6 +335,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. *
lib/eal/linux/eal.c +8 -0
@@ -1362,6 +1362,14 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); + +/* Close all the buses and devices/drivers on them */ +if (rte_bus_close ()) { +rte_eal_init_alert ("Cannot close devices" ); + rte_errno =ENOTSUP ; +return -1 ; + } +rte_service_finalize ();rte_mp_channel_cleanup ();/* after this point, any DPDK pointers will become dangling */
lib/eal/version.map +3 -0
@@ -420,6 +420,9 @@ EXPERIMENTAL { rte_intr_instance_free; rte_intr_type_get; rte_intr_type_set; + + # added in 22.03 + rte_bus_close; }; INTERNAL {
lib/eal/windows/eal.c +1 -0
@@ -260,6 +260,7 @@ rte_eal_cleanup(void)struct internal_config *internal_conf =eal_get_internal_configuration (); +rte_bus_close ();eal_intr_thread_cancel ();eal_mem_virt2iova_cleanup ();/* after this point, any DPDK pointers will become dangling */
Signature · 1 line
2.17.1
PATCH v5 2/2 bus/fslmc: support bus close API
From: Rohit Raj <[email protected]>
This patch add support for closing the bus objects which were acquired In the bus probe.
Some devices need to be cleaned while in both primary and secondary process and while some need to be cleaned only in case of primary process.
The devices are closed as per the white list used while creating the objects in a particular process.
--- drivers/bus/fslmc/fslmc_bus.c | 15 +++- drivers/bus/fslmc/fslmc_vfio.c | 89 +++++++++++++++++++++++- drivers/bus/fslmc/fslmc_vfio.h | 3 +- drivers/bus/fslmc/mc/fsl_dpcon.h | 3 +- drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 30 +++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 30 +++++++- drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 36 +++++++++- drivers/bus/fslmc/portal/dpaa2_hw_dprc.c | 31 ++++++++- drivers/bus/fslmc/rte_fslmc.h | 5 +- drivers/bus/fslmc/version.map | 1 + drivers/event/dpaa2/dpaa2_hw_dpcon.c | 31 ++++++++- drivers/net/dpaa2/dpaa2_mux.c | 20 +++++- 12 files changed, 282 insertions(+), 12 deletions(-)
drivers/bus/fslmc/fslmc_bus.c +14 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2018-2021 NXP +* Copyright 2016,2018-2022 NXP * */
@@ -493,6 +493,18 @@ rte_fslmc_probe(void)return 0 ; } +static int +rte_fslmc_close (void ) +{ +int ret =0 ; + + ret =fslmc_vfio_close_group (); +if (ret) +DPAA2_BUS_ERR ("Unable to close devices %d" , ret); + +return 0 ; +} +static struct rte_device *rte_fslmc_find_device (const struct rte_device *start,rte_dev_cmp_t cmp,const void *data)
@@ -668,6 +680,7 @@ struct rte_fslmc_bus rte_fslmc_bus = { .bus = { .scan = rte_fslmc_scan, .probe = rte_fslmc_probe, + .close = rte_fslmc_close, .parse = rte_fslmc_parse, .find_device = rte_fslmc_find_device, .get_iommu_class = rte_dpaa2_get_iommu_class,
drivers/bus/fslmc/fslmc_vfio.c +88 -1
98 lines not on this page · raw message
drivers/bus/fslmc/fslmc_vfio.h +2 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016,2019 NXP +* Copyright 2016,2019,2021-2022 NXP * */
@@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle *intr_handle,int fslmc_vfio_setup_group (void );int fslmc_vfio_process_group (void ); +int fslmc_vfio_close_group (void );char *fslmc_get_container (void );int fslmc_get_container_group (int *gropuid);int rte_fslmc_vfio_dmamap (void );
drivers/bus/fslmc/mc/fsl_dpcon.h +2 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) * * Copyright 2013-2016 Freescale Semiconductor Inc. -* Copyright 2017-2019 NXP +* Copyright 2017-2019,2021-2022 NXP * */ #ifndef __FSL_DPCON_H
@@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,int dpcon_id, uint16_t *token); +__rte_internal int dpcon_close(struct fsl_mc_io *mc_io,uint32_t cmd_flags,uint16_t token);
drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c +29 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016 NXP +* Copyright 2016,2021-2022 NXP * */
@@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);static struct dpbp_dev_list dpbp_dev_list =TAILQ_HEAD_INITIALIZER (dpbp_dev_list);/*!< DPBP device list */ +static struct dpaa2_dpbp_dev *get_dpbp_from_id (uint32_t dpbp_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + +/* Get DPBP dev handle from list using index */ + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) { +if (dpbp_dev->dpbp_id == dpbp_id) +break ; + } + +return dpbp_dev; +} + static int dpaa2_create_dpbp_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -109,6 +122,20 @@ void dpaa2_free_dpbp_dev(struct dpaa2_dpbp_dev *dpbp) } } +static void dpaa2_close_dpbp_device (int object_id) +{ +struct dpaa2_dpbp_dev *dpbp_dev =NULL ; + + dpbp_dev =get_dpbp_from_id ((uint32_t )object_id); + +if (dpbp_dev) { +dpaa2_free_dpbp_dev (dpbp_dev); +dpbp_close (&dpbp_dev->dpbp,CMD_PRI_LOW , dpbp_dev->token); +TAILQ_REMOVE (&dpbp_dev_list, dpbp_dev, next); +rte_free (dpbp_dev); + } +} +int dpaa2_dpbp_supported (void ) {if (TAILQ_EMPTY (&dpbp_dev_list))
@@ -119,6 +146,7 @@ int dpaa2_dpbp_supported(void)static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = { .dev_type =DPAA2_BPOOL , .create = dpaa2_create_dpbp_device, + .close = dpaa2_close_dpbp_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpbp, rte_dpaa2_dpbp_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpci.c +29 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2021-2022 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);static struct dpci_dev_list dpci_dev_list =TAILQ_HEAD_INITIALIZER (dpci_dev_list);/*!< DPCI device list */ +static struct dpaa2_dpci_dev *get_dpci_from_id (uint32_t dpci_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + +/* Get DPCI dev handle from list using index */ + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) { +if (dpci_dev->dpci_id == dpci_id) +break ; + } + +return dpci_dev; +} + static int rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -179,9 +192,24 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci) } } +static void rte_dpaa2_close_dpci_device (int object_id) +{ +struct dpaa2_dpci_dev *dpci_dev =NULL ; + + dpci_dev =get_dpci_from_id ((uint32_t )object_id); + +if (dpci_dev) { +rte_dpaa2_free_dpci_dev (dpci_dev); +dpci_close (&dpci_dev->dpci,CMD_PRI_LOW , dpci_dev->token); +TAILQ_REMOVE (&dpci_dev_list, dpci_dev, next); +rte_free (dpci_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpci_obj = { .dev_type =DPAA2_CI , .create = rte_dpaa2_create_dpci_device, + .close = rte_dpaa2_close_dpci_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpci, rte_dpaa2_dpci_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dpio.c +35 -1
@@ -1,7 +1,7 @@/* SPDX-License-Identifier: BSD-3-Clause * * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. -* Copyright 2016-2019 NXP +* Copyright 2016-2019,2021-2022 NXP * */ #include <unistd.h>
@@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;* Cluster 4 (ID = x07) : CPU14, CPU15; */ +static struct dpaa2_dpio_dev *get_dpio_dev_from_id (int32_t dpio_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + +/* Get DPIO dev handle from list using index */ + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) { +if (dpio_dev->hw_id == dpio_id) +break ; + } + +return dpio_dev; +} +static int dpaa2_get_core_id (void ) {
@@ -634,9 +647,30 @@ dpaa2_free_eq_descriptors(void)MAX_EQ_RESP_ENTRIES -1 ; } +static void +dpaa2_close_dpio_device (int object_id) +{ +struct dpaa2_dpio_dev *dpio_dev =NULL ; + + dpio_dev =get_dpio_dev_from_id ((int32_t )object_id); + +if (dpio_dev) { +if (dpio_dev->dpio) { +dpio_disable (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +dpio_close (dpio_dev->dpio,CMD_PRI_LOW , + dpio_dev->token); +rte_free (dpio_dev->dpio); + } +TAILQ_REMOVE (&dpio_dev_list, dpio_dev, next); +rte_free (dpio_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpio_obj = { .dev_type =DPAA2_IO , .create = dpaa2_create_dpio_device, + .close = dpaa2_close_dpio_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpio, rte_dpaa2_dpio_obj);
drivers/bus/fslmc/portal/dpaa2_hw_dprc.c +30 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2021 NXP +* Copyright 2021-2022 NXP * */
@@ -92,9 +92,38 @@ rte_dpaa2_create_dprc_device(int vdev_fd __rte_unused,return 0 ; } +static struct dpaa2_dprc_dev * +get_dprc_from_id (uint32_t dprc_id) +{ +struct dpaa2_dprc_dev *dprc_dev =NULL ; + +/* Get DPRC dev handle from list using index */ + TAILQ_FOREACH(dprc_dev, &dprc_dev_list, next) { +if (dprc_dev->dprc_id == dprc_id) +break ; + } + +return dprc_dev; +} + +static void +rte_dpaa2_close_dprc_device (int object_id) +{ +struct dpaa2_dprc_dev *dprc_dev =NULL ; + + dprc_dev =get_dprc_from_id ((uint32_t )object_id); + +if (dprc_dev) { +dprc_close (&dprc_dev->dprc,CMD_PRI_LOW , dprc_dev->token); +TAILQ_REMOVE (&dprc_dev_list, dprc_dev, next); +rte_free (dprc_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dprc_obj = { .dev_type =DPAA2_DPRC , .create = rte_dpaa2_create_dprc_device, + .close = rte_dpaa2_close_dprc_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dprc, rte_dpaa2_dprc_obj);
drivers/bus/fslmc/rte_fslmc.h +4 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2016,2021 NXP +* Copyright 2016,2021-2022 NXP * */
@@ -104,6 +104,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,struct vfio_device_info *obj_info, int object_id); +typedef void (*rte_dpaa2_obj_close_t )(int object_id); +/** * A structure describing a DPAA2 object. */
@@ -112,6 +114,7 @@ struct rte_dpaa2_object {const char *name;/**< Name of Object. */ enum rte_dpaa2_dev_type dev_type;/**< Type of device */ rte_dpaa2_obj_create_t create; +rte_dpaa2_obj_close_t close; };/**
drivers/bus/fslmc/version.map +1 -0
@@ -34,6 +34,7 @@ INTERNAL { dpci_get_opr; dpci_set_opr; dpci_set_rx_queue; + dpcon_close; dpcon_get_attributes; dpcon_open; dpdmai_close;
drivers/event/dpaa2/dpaa2_hw_dpcon.c +30 -1
@@ -1,6 +1,6 @@/* SPDX-License-Identifier: BSD-3-Clause * -* Copyright 2017 NXP +* Copyright 2017,2022 NXP * */
@@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);static struct dpcon_dev_list dpcon_dev_list =TAILQ_HEAD_INITIALIZER (dpcon_dev_list);/*!< DPCON device list */ +static struct dpaa2_dpcon_dev *get_dpcon_from_id (uint32_t dpcon_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + +/* Get DPCONC dev handle from list using index */ + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) { +if (dpcon_dev->dpcon_id == dpcon_id) +break ; + } + +return dpcon_dev; +} + static int rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused, struct vfio_device_info *obj_info __rte_unused,
@@ -105,9 +118,25 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev *dpcon) } } +static void +rte_dpaa2_close_dpcon_device (int object_id) +{ +struct dpaa2_dpcon_dev *dpcon_dev =NULL ; + + dpcon_dev =get_dpcon_from_id ((uint32_t )object_id); + +if (dpcon_dev) { +rte_dpaa2_free_dpcon_dev (dpcon_dev); +dpcon_close (&dpcon_dev->dpcon,CMD_PRI_LOW , dpcon_dev->token); +TAILQ_REMOVE (&dpcon_dev_list, dpcon_dev, next); +rte_free (dpcon_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = { .dev_type =DPAA2_CON , .create = rte_dpaa2_create_dpcon_device, + .close = rte_dpaa2_close_dpcon_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpcon, rte_dpaa2_dpcon_obj);
drivers/net/dpaa2/dpaa2_mux.c +18 -2
@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2018-2021 NXP +* Copyright 2018-2022 NXP */ #include <sys/queue.h>
@@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t dpdmux_id) {struct dpaa2_dpdmux_dev *dpdmux_dev =NULL ; -/* Get DPBP dev handle from list using index */ +/* Get DPDMUX dev handle from list using index */ TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {if (dpdmux_dev->dpdmux_id == dpdmux_id)break ;
@@ -359,9 +359,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,return -1 ; } +static void +dpaa2_close_dpdmux_device (int object_id) +{ +struct dpaa2_dpdmux_dev *dpdmux_dev; + + dpdmux_dev =get_dpdmux_from_id ((uint32_t )object_id); + +if (dpdmux_dev) { +dpdmux_close (&dpdmux_dev->dpdmux,CMD_PRI_LOW , + dpdmux_dev->token); +TAILQ_REMOVE (&dpdmux_dev_list, dpdmux_dev, next); +rte_free (dpdmux_dev); + } +} +static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = { .dev_type =DPAA2_MUX , .create = dpaa2_create_dpdmux_device, + .close = dpaa2_close_dpdmux_device, };RTE_PMD_REGISTER_DPAA2_OBJECT (dpdmux, rte_dpaa2_dpdmux_obj);
Signature · 1 line
2.17.1
Hi,
10/01/2022 06:26, [email protected]:
Quoted 9 lines
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.
I don't understand how closing all devices of a bus will help better than just closing all devices.
As Ferruh already suggested in the past,
we could force closing all devices in rte_eal_cleanup(). And we already have the function rte_dev_remove().
Hi Thomas,
This "rte_bus_close" API is introduced to do the opposite of what "rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and unplugging a single device.
The API you mentioned, "rte_dev_remove" supports only rte_device. But we also need to close/remove private devices of dpaa and fslmc buses which are not exposed directly to user (eg: mempool device). Note that these private devices/bus objects are not associated with a particular rte_device but they are available as a resource to be used by any of the device based on these hardware specific buses. So, to close these devices, we need a new API which can do this for us. That is why "rte_bus_close" is required.
Regards,
Rohit Raj
Quoted 35 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Wednesday, January 19, 2022 4:02 PM
To: Rohit Raj <[email protected]>
Cc: Bruce Richardson <[email protected]>; Ray Kinsella <[email protected]>; Dmitry Kozlyuk <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Dmitry Malloy <[email protected]>; Pallavi Kadam <[email protected]>; [email protected]; Nipun Gupta <[email protected]>; Sachin Saxena <[email protected]>; Hemant Agrawal <[email protected]>; [email protected]; [email protected]
Subject: [EXT] Re: [PATCH v5 1/2] eal: add API for bus closeCaution: EXT Email
Hi,
10/01/2022 06:26, [email protected]:
Quoted 8 lines
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.I don't understand how closing all devices of a bus will help better than just closing all devices.
As Ferruh already suggested in the past, we could force closing all devices in
rte_eal_cleanup().
And we already have the functionrte_dev_remove().
20/01/2022 15:51, Rohit Raj:
Quoted 7 lines
Hi Thomas,
This "
rte_bus_close" API is introduced to do the opposite of what "rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and unplugging a single device.The API you mentioned, "
rte_dev_remove" supports onlyrte_device. But we also need to close/remove private devices of dpaa and fslmc buses which are not exposed directly to user (eg: mempool device). Note that these private devices/bus objects are not associated with a particularrte_devicebut they are available as a resource to be used by any of the device based on these hardware specific buses. So, to close these devices, we need a new API which can do this for us. That is why "rte_bus_close" is required.
You mean some resources are associated to a bus but not to a device? It lools very weird. A resource on a bus *is* a device.
PS: please avoid top-post
Quoted 17 lines
From: Thomas Monjalon <[email protected]>
Quoted 16 lines
10/01/2022 06:26, [email protected]:
Quoted 8 lines
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.I don't understand how closing all devices of a bus will help better than just closing all devices.
As Ferruh already suggested in the past, we could force closing all devices in
rte_eal_cleanup().
And we already have the functionrte_dev_remove().
Quoted 30 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Thursday, January 20, 2022 8:28 PM
To: Rohit Raj <[email protected]>
Cc: Bruce Richardson <[email protected]>; Ray Kinsella <[email protected]>; Dmitry Kozlyuk <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Dmitry Malloy <[email protected]>; Pallavi Kadam <[email protected]>; [email protected]; Nipun Gupta <[email protected]>; Sachin Saxena <[email protected]>; Hemant Agrawal <[email protected]>; [email protected]; [email protected]
Subject: Re: [EXT] Re: [PATCH v5 1/2] eal: add API for bus closeCaution: EXT Email
20/01/2022 15:51, Rohit Raj:
Quoted 3 lines
Hi Thomas,
This "
rte_bus_close" API is introduced to do the opposite of what"
rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and unplugging a single device.Quoted 2 lines
The API you mentioned, "
rte_dev_remove" supports onlyrte_device. But wealso need to close/remove private devices of dpaa and fslmc buses which are not exposed directly to user (eg: mempool device).
Quoted 1 line
Note that these private devices/bus objects are not associated with a
particular
rte_devicebut they are available as a resource to be used by any of the device based on these hardware specific buses.
Quoted 7 lines
Quoted 1 line
So, to close these devices, we need a new API which can do this for us. That is
why "
rte_bus_close" is required.You mean some resources are associated to a bus but not to a device? It lools very weird. A resource on a bus *is* a device.
PS: please avoid top-post
FSLMC bus has hardware resources for memory pools, queues, hardware access lock(called portal).
These are common resources, which can be associated with any device. So they don't belong to a specific device.
Eg: mempool resource can be used by both eth and crypto device. So, we cannot close mempool while closing just one of the device(It can happen in multiprocess applications). So, these resources should be specifically closed/freed with the bus instead with a device.
There is no need to expose these devices to users and their usages is limited to other devices on the bus. There is no reason to create yet another type of device in DPDK to expose these internal only resources.
Quoted 19 lines
Quoted 17 lines
From: Thomas Monjalon <[email protected]>
Quoted 16 lines
10/01/2022 06:26, [email protected]:
Quoted 8 lines
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.I don't understand how closing all devices of a bus will help better than just closing all devices.
As Ferruh already suggested in the past, we could force closing all devices in
rte_eal_cleanup().
And we already have the functionrte_dev_remove().
01/02/2022 06:40, Rohit Raj:
Quoted 20 lines
From: Thomas Monjalon <[email protected]>
Quoted 19 lines
20/01/2022 15:51, Rohit Raj:
Quoted 3 lines
Hi Thomas,
This "
rte_bus_close" API is introduced to do the opposite of what"
rte_bus_probe" does. Just like there are plug and unplug APIs for plugging and unplugging a single device.Quoted 2 lines
The API you mentioned, "
rte_dev_remove" supports onlyrte_device. But wealso need to close/remove private devices of dpaa and fslmc buses which are not exposed directly to user (eg: mempool device).
Quoted 1 line
Note that these private devices/bus objects are not associated with a
particular
rte_devicebut they are available as a resource to be used by any of the device based on these hardware specific buses.Quoted 1 line
So, to close these devices, we need a new API which can do this for us. That is
why "
rte_bus_close" is required.You mean some resources are associated to a bus but not to a device? It lools very weird. A resource on a bus *is* a device.
Quoted 7 lines
Quoted 1 line
PS: please avoid top-post
FSLMC bus has hardware resources for memory pools, queues, hardware access lock(called portal).
These are common resources, which can be associated with any device. So they don't belong to a specific device.
Eg: mempool resource can be used by both eth and crypto device. So, we cannot close mempool while closing just one of the device(It can happen in multiprocess applications). So, these resources should be specifically closed/freed with the bus instead with a device.There is no need to expose these devices to users and their usages is limited to other devices on the bus. There is no reason to create yet another type of device in DPDK to expose these internal only resources.
OK I understand better now, thanks.
Quoted 17 lines
Quoted 17 lines
Quoted 17 lines
From: Thomas Monjalon <[email protected]>
Quoted 16 lines
10/01/2022 06:26, [email protected]:
Quoted 8 lines
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.I don't understand how closing all devices of a bus will help better than just closing all devices.
As Ferruh already suggested in the past, we could force closing all devices in
rte_eal_cleanup().
And we already have the functionrte_dev_remove().
Quoted 7 lines · On Mon, Jan 10, 2022 at 6:26 AM <[email protected]> wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
After an application crash, how does this bus resets the associated resources?
Quoted 3 lines
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.
The patch in its current form breaks the ABI on rte_bus object. This can be seen in GHA, or calling DPDK_ABI_REF_VERSION=v21.11 ./devtools/test-meson-builds.sh.
[snip]
doc/guides/rel_notes/release_22_03.rst New Features · line 62
Quoted 13 lines
doc/guides/rel_notes/release_22_03.rst +5
-0@@ -55,6 +55,11 @@ New Features Also, make sure to start the actual text at the margin. ======================================================= + * **Added support to close bus.** + + Added capability to allow a user to do cleanup of bus objects which + were acquired during bus probe. +
Wrongly indented.
Quoted 45 lines
doc/guides/rel_notes/release_22_03.rst +3
-0Removed Items -------------@@ -84,6 +89,9 @@ API Changes Also, make sure to start the actual text at the margin. ======================================================= + * eal: Added new API ``rte_bus_close`` to perform cleanup bus objects which + were acquired during bus probe. + ABI Changes -----------lib/eal/common/eal_common_bus.c +18
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2022 NXP */ #include <stdio.h> @@ -85,6 +85,37 @@ rte_bus_probe(void)return 0 ; } +/* Close all devices of all buses */ +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) { + vbus = bus; +continue ; + } + +if (bus->close) { + ret = bus->close (); +if (ret) + RTE_LOG(ERR, EAL, "Bus (%s) close failed.\n",
lib/eal/common/eal_common_bus.c rte_bus_probe(void) · line 114
Quoted 10 lines
lib/eal/common/eal_common_bus.c +10
-0+ bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + }
The vdev bus is special in that some drivers can reference objects from other buses (see f4ce209a8ce5 ("eal: postpone vdev initialization") and da76cc02342b ("eal: probe new virtual bus after other bus devices")).
For this reason, I would expect that the vdev bus is closed before the other buses.
Quoted 84 lines
lib/eal/common/eal_common_bus.c +4
-0+ +return 0 ; +} +/* Dump information of a single bus */ static int bus_dump_one (FILE *f,struct rte_bus *bus)lib/eal/freebsd/eal.c +1
-0@@ -984,6 +984,7 @@ rte_eal_cleanup(void) {struct internal_config *internal_conf =eal_get_internal_configuration (); +rte_bus_close ();rte_service_finalize ();rte_mp_channel_cleanup ();/* after this point, any DPDK pointers will become dangling */ lib/eal/include/rte_bus.h +29
-1@@ -1,5 +1,5 @@/* SPDX-License-Identifier: BSD-3-Clause -* Copyright 2016 NXP +* Copyright 2016,2022 NXP */ #ifndef _RTE_BUS_H_@@ -66,6 +66,23 @@ typedef int (*rte_bus_scan_t)(void);*/ typedef int (*rte_bus_probe_t )(void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Implementation specific close function which is responsible for resetting all +* detected devices on the bus to a default state, closing UIO nodes or VFIO +* groups and also freeing any memory allocated during rte_bus_probe like +* private resources for device list. +* +* This is called while iterating over each registered bus. +* +* @return +* 0 for successful close +* !0 for any error while closing +*/ +typedef int (*rte_bus_close_t )(void ); +/** * Device iterator to find a device on a bus. * @@ -263,6 +280,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */ rte_bus_find_device_t find_device;/**< Find a device on the bus */ rte_bus_plug_t plug;/**< Probe single device for drivers */ rte_bus_unplug_t unplug;/**< Remove single device from driver */ @@ -317,6 +335,16 @@ int rte_bus_scan(void);*/ int rte_bus_probe (void ); +/** +* For each device on the buses, call the device specific close. +* +* @return +* 0 for successful close +* !0 otherwise +*/ +__rte_experimental +int rte_bus_close(void); +/** * Dump information of all the buses registered with EAL. * lib/eal/linux/eal.c +0
-0
lib/eal/linux/eal.c rte_eal_cleanup(void) · line 1368
Quoted 7 lines
lib/eal/linux/eal.c +4
-0@@ -1362,6 +1362,14 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); + +/* Close all the buses and devices/drivers on them */ +if (rte_bus_close ()) { +rte_eal_init_alert ("Cannot close devices" );
You can't call rte_eal_*init*_alert in rte_eal_cleanup.
There is not much to do if the bus close fails, I'd rather leave the cleanup continue.
Quoted 36 lines
lib/eal/linux/eal.c +4
-0+ rte_errno =ENOTSUP ; +return -1 ; + } +rte_service_finalize ();rte_mp_channel_cleanup ();/* after this point, any DPDK pointers will become dangling */ lib/eal/version.map +3
-0@@ -420,6 +420,9 @@ EXPERIMENTAL { rte_intr_instance_free; rte_intr_type_get; rte_intr_type_set; + + # added in 22.03 + rte_bus_close; }; INTERNAL {lib/eal/windows/eal.c +1
-1@@ -260,6 +260,7 @@ rte_eal_cleanup(void)struct internal_config *internal_conf =eal_get_internal_configuration (); +rte_bus_close ();eal_intr_thread_cancel ();eal_mem_virt2iova_cleanup ();/* after this point, any DPDK pointers will become dangling */ --2.17.1
Signature · 1 line
David Marchand
09/02/2022 12:04, David Marchand:
Quoted 17 lines · On Mon, Jan 10, 2022 at 6:26 AM <[email protected]> wrote:
Quoted 7 lines
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
After an application crash, how does this bus resets the associated resources?
Quoted 2 lines
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.The patch in its current form breaks the ABI on
rte_busobject. This can be seen in GHA, or calling DPDK_ABI_REF_VERSION=v21.11./devtools/test-meson-builds.sh.
[...]
Quoted 9 lines
lib/eal/common/eal_common_bus.c rte_bus_probe(void) · line 96
Quoted 9 lines
lib/eal/common/eal_common_bus.c +9
-0+/* Close all devices of all buses */ +int +rte_bus_close (void ) +{ +int ret; +struct rte_bus *bus, *vbus =NULL ; + + TAILQ_FOREACH(bus, &rte_bus_list, next) { +if (!strcmp (bus->name,"vdev" )) {
Please do an explicit comparison "== 0".
Quoted 5 lines
lib/eal/common/eal_common_bus.c rte_bus_probe(void) · line 101
Quoted 5 lines
lib/eal/common/eal_common_bus.c +5
-0+ vbus = bus; +continue ; + } + +if (bus->close) {
Please do an explicit comparison with "!= NULL".
We can also completely remove this check and implement the callback in all buses. It should loop in all remaining devices and remove them.
Quoted 14 lines
lib/eal/common/eal_common_bus.c rte_bus_probe(void) · line 114
Quoted 13 lines
lib/eal/common/eal_common_bus.c +13
-0+ ret = bus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + bus->name); + } + } + +if (vbus && vbus->close) { + ret = vbus->close (); +if (ret) +RTE_LOG (ERR ,EAL ,"Bus (%s) close failed.\n" , + vbus->name); + }
Quoted 6 lines
The vdev bus is special in that some drivers can reference objects from other buses (see
f4ce209a8ce5("eal: postpone vdev initialization") andda76cc02342b("eal: probe new virtual bus after other bus devices")).
For this reason, I would expect that the vdev bus is closed before the other buses.
Yes, good catch.
We don't have to expose this function as API.
This can be an internal function called only in rte_eal_cleanup().
Instead, it would be more useful to have a public function to close a single bus by its name.
Quoted 5 lines
lib/eal/include/rte_bus.h struct rte_bus { · line 283
Quoted 5 lines
lib/eal/include/rte_bus.h +1
-0@@ -263,6 +280,7 @@ struct rte_bus {const char *name;/**< Name of the bus */ rte_bus_scan_t scan;/**< Scan for devices attached to bus */ rte_bus_probe_t probe;/**< Probe devices on bus */ +rte_bus_close_t close;/**< Close devices on bus */
As David said, it is breaking the ABI.
[...]
Quoted 11 lines
lib/eal/linux/eal.c rte_eal_cleanup(void) · line 1368
Quoted 8 lines
lib/eal/linux/eal.c +4
-0@@ -1362,6 +1362,14 @@ rte_eal_cleanup(void)if (rte_eal_process_type () ==RTE_PROC_PRIMARY )rte_memseg_walk (mark_freeable,NULL ); + +/* Close all the buses and devices/drivers on them */ +if (rte_bus_close ()) { +rte_eal_init_alert ("Cannot close devices" );You can't call rte_eal_*init*_alert in
rte_eal_cleanup.
Quoted 2 lines
There is not much to do if the bus close fails, I'd rather leave the cleanup continue.
+1, just log and save the error code for return at the end.
Quoted 11 lines · On Mon, 24 Aug 2020 13:54:12 +0530 [email protected] wrote:
From: Rohit Raj <[email protected]>
As per the current code we have API for bus probe, but the bus close API is missing. This breaks the multi process scenarios as objects are not cleaned while terminating the secondary processes.
This patch adds a new API
rte_bus_close()for cleanup of bus objects which were acquired during probe.Signed-off-by Rohit Raj <[email protected]>
This patchset got some good feedback but stalled at version 5. Marking it as "Changes requested".
Please rebase and resubmit