Skip to contentSkip to conversation

V1 event/dlb add Intel DLB PMD

PATCH In review McDaniel, Timothy 7 participants 312 messages Jun 12, 2020 – Nov 11, 2020 Arm/Amlogic Meson SoC support Page 3 of 6

lore
Share & tools
lore
#120 lore raw

PATCH v11 8/23 event/dlb: add probe-time hardware init

In reply to Timothy McDaniel

This commit adds probe-time low level hardware initialization. It also adds probe-time init for both primary and secondary DPDK processes.

---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
drivers/event/dlb/dlb.c +153 -5
index 8008a50..57b2837 100644
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
drivers/event/dlb/meson.build +2 -1
index 8707d3d..9777178 100644
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/pf/base/dlb_resource.c new file +302 -0

302 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.c +19 -1
index c10c36c..2f4a828 100644
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
drivers/event/dlb/pf/dlb_pf.c +85 -1

94 lines not on this page · raw message

Signature · 1 line
2.6.4
#121 lore raw

PATCH v11 9/23 event/dlb: add xstats

In reply to Timothy McDaniel

Add support for DLB xstats. Perform initialization and add standard xstats entry points

---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c
drivers/event/dlb/dlb.c +23 -0
index 57b2837..62b9695 100644
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
drivers/event/dlb/dlb_xstats.c new file +1222 -0

1,222 lines not on this page · raw message

drivers/event/dlb/meson.build +1 -0
index 9777178..552ff9d 100644
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
Signature · 1 line
2.6.4
#122 lore raw

PATCH v11 10/23 event/dlb: add infos get and configure

In reply to Timothy McDaniel

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB hardware's scheduling domain, such that it is provisioned with the requested number of ports and queues, provided sufficient resources are available. Individual queues and ports are configured later in port setup and eventdev start.

---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)
doc/guides/eventdevs/dlb.rst +48 -0
index 92341c0..2d7999b 100644
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
drivers/event/dlb/dlb.c +397 -0

415 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +11 -0
index dd72120..f3e82f2 100644
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +11 -0
index 416d1b3..d576232 100644
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +4007 -93

2,488 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +88 -0

88 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.6.4

Part of the diff is not shown here. View the raw message.

#123 lore raw

PATCH v11 11/23 event/dlb: add queue and port default conf

In reply to Timothy McDaniel

Add support for getting the queue and port default configuration.

---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
drivers/event/dlb/dlb.c +29 -0
index c038794..e98a438 100644
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#124 lore raw

PATCH v11 12/23 event/dlb: add queue setup

In reply to Timothy McDaniel

Load balanced (ldb) queues are setup here. Directed queues are not set up until link time, at which point we know the directed port ID. Directed queue setup will only fail if this queue is already setup or there are no directed queues left to configure.

---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)
doc/guides/eventdevs/dlb.rst +35 -0
index 2d7999b..d8e936a 100644
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
drivers/event/dlb/dlb.c +293 -0

305 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +12 -0
index f3e82f2..219f79e 100644
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
drivers/event/dlb/dlb_iface.h +12 -0
index d576232..af1416d 100644
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
drivers/event/dlb/pf/base/dlb_resource.c +386 -0

389 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +81 -0

94 lines not on this page · raw message

Signature · 1 line
2.6.4
#125 lore raw

PATCH v11 13/23 event/dlb: add port setup

In reply to Timothy McDaniel

Configure the load balanced (ldb) or directed (dir) port. The consumer queue (CQ) and producer port (PP) are also set up here.

---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)
doc/guides/eventdevs/dlb.rst +40 -0
index d8e936a..f106a07 100644
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
drivers/event/dlb/dlb.c +515 -1

546 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +11 -0
index 219f79e..fbbf9d7 100644
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +14 -0
index af1416d..d578185 100644
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +1433 -3

1,457 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +210 -0

222 lines not on this page · raw message

Signature · 1 line
2.6.4
#126 lore raw

PATCH v11 14/23 event/dlb: add port link

In reply to Timothy McDaniel

Add port link entry point. Directed queues are identified and created at this stage. Their setup deferred until link-time, at which point we know the directed port ID. Directed queue setup will only fail if this queue is already setup or there are no directed queues left to configure.

---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)
drivers/event/dlb/dlb.c +306 -0
index 4d91ddd..2ad195d 100644
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
drivers/event/dlb/dlb_iface.c +9 -0
index fbbf9d7..aaf4506 100644
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +9 -0
index d578185..c0f5f2e 100644
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +641 -0

644 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +69 -0

81 lines not on this page · raw message

Signature · 1 line
2.6.4
#127 lore raw

PATCH v11 15/23 event/dlb: add port unlink and port unlinks in progress

In reply to Timothy McDaniel

Add supports for the port unlink(s) eventdev entry points. The unlink operation is an asynchronous operation executed by a control thread, and the unlinks-in-progress function reads a counter shared with the control thread. Port QE and memzone memory is freed here.

---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)
drivers/event/dlb/dlb.c +166 -0
index 2ad195d..c64f559 100644
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#128 lore raw

PATCH v11 16/23 event/dlb: add eventdev start

In reply to Timothy McDaniel

Add support for the eventdev start entry point. DLB delays setting up single link resources until eventdev start, because it is only then that it can ascertain which ports have just one linked queue.

---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)
drivers/event/dlb/dlb.c +180 -44
index c64f559..780ff7d 100644
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
drivers/event/dlb/dlb_iface.c +3 -0
index aaf4506..22d524b 100644
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
drivers/event/dlb/dlb_iface.h +3 -0
index c0f5f2e..8c905ab 100644
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
drivers/event/dlb/pf/base/dlb_resource.c +142 -0

151 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +23 -0
index fed6719..1d2e133 100644
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
Signature · 1 line
2.6.4
#129 lore raw

PATCH v11 17/23 event/dlb: add enqueue and its burst variants

In reply to Timothy McDaniel

Add support for enqueue and its variants.

---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)
doc/guides/eventdevs/dlb.rst +162 -1
index f106a07..ae126c4 100644
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
drivers/event/dlb/dlb.c +682 -0

700 lines not on this page · raw message

Signature · 1 line
2.6.4
#130 lore raw

PATCH v11 18/23 event/dlb: add dequeue and its burst variants

In reply to Timothy McDaniel

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses umonitor/umwait if supported by the processor. This allows the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the hardware will pack 4 QEs into each cache line. In sparse cq mode, the hardware will only populate one QE per cache line. Software must be aware of the cq mode, and take the appropriate actions, based on the mode.

---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 679 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 700 insertions(+)
doc/guides/eventdevs/dlb.rst +21 -0
index ae126c4..4c4f56b 100644
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
drivers/event/dlb/dlb.c +679 -0

700 lines not on this page · raw message

Signature · 1 line
2.6.4
#131 lore raw

PATCH v11 19/23 event/dlb: add eventdev stop and close

In reply to Timothy McDaniel

Add support for eventdev stop and close entry points.

---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)
drivers/event/dlb/dlb.c +245 -11
index e2f769c..a8efcc1 100644
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3480,6 +3469,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3489,6 +3721,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
drivers/event/dlb/dlb_iface.c +6 -0
index 22d524b..44f958f 100644
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
drivers/event/dlb/dlb_iface.h +6 -0
index 8c905ab..9f61135 100644
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
drivers/event/dlb/pf/base/dlb_resource.c +89 -0

92 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +47 -0

63 lines not on this page · raw message

Signature · 1 line
2.6.4
#132 lore raw

PATCH v11 20/23 event/dlb: add PMD's token pop public interface

In reply to Timothy McDaniel

The PMD uses a public interface to allow applications to control the token pop mode. Supported token pop modes are as follows, and they impact core scheduling affinity for ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
are released. Supported on load-balanced ports only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
doc/api/doxy-api-index.md +1 -0
index a9c12d1..1c83bf4 100644
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
drivers/event/dlb/dlb.c +109 -12

247 lines not on this page · raw message

drivers/event/dlb/dlb_priv.h +3 -0
index adb1f7a..58ff428 100644
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
drivers/event/dlb/meson.build +3 -1
index 552ff9d..7f38c30 100644
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
drivers/event/dlb/rte_pmd_dlb.c new file +38 -0
new file mode 100644
index 0000000..bc802d3
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
drivers/event/dlb/rte_pmd_dlb.h new file +77 -0

77 lines not on this page · raw message

drivers/event/dlb/version.map +6 -0
index 4a76d1d..3338a22 100644
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
Signature · 1 line
2.6.4
#133 lore raw

PATCH v11 21/23 event/dlb: add PMD self-tests

In reply to Timothy McDaniel

Add a variety of self-tests for both ldb and directed ports/queues, as well as configure, start, stop, link, etc...

---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c
app/test/test_eventdev.c +7 -0
index 62019c1..ba27bed 100644
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
drivers/event/dlb/dlb.c +1 -0
index 31e4d50..c37de85 100644
@@ -3829,6 +3829,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
drivers/event/dlb/dlb_selftest.c new file +1539 -0

1,539 lines not on this page · raw message

drivers/event/dlb/meson.build +1 -0
index 7f38c30..875cf89 100644
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
Signature · 1 line
2.6.4
#134 lore raw

PATCH v11 22/23 event/dlb: add queue and port release

In reply to Timothy McDaniel

These entry points are NO-OPS. DLB does not support reconfiguring individual queues or ports. The entire device must be reconfigured.

---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
drivers/event/dlb/dlb.c +27 -0
index c37de85..41ccb4a 100644
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3805,6 +3808,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3819,7 +3844,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
Signature · 1 line
2.6.4
#135 lore raw

PATCH v11 23/23 event/dlb: add timeout ticks entry point

In reply to Timothy McDaniel

Adds the timeout ticks conversion function.

---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)
drivers/event/dlb/dlb.c +13 -0
index 41ccb4a..4aab9f6 100644
@@ -3830,6 +3830,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3851,6 +3863,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#136 lore raw

v12 cover Add DLB PMD

In reply to McDaniel, Timothy

The following patch series adds support for a new eventdev PMD. The DLB PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware. The DLB is a PCIe device that provides load-balanced, prioritized scheduling of core-to-core communication. The device consists of queues and arbiters that connect producer and consumer cores, and implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced distribution.

The DLB hardware supports both load balanced and directed ports and queues. Unlike other eventdev devices already in the repo, not all DLB ports and queues are equally capable. In particular, directed ports are limited to a single link, and must be connected to a directed queue.
Additionally, even though LDB ports may link multiple queues, the number of queues that may be linked is limited by hardware. Another difference is that DLB does not have a straightforward way of carrying the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full control over the DLB hardware. Intel will be extending the DLB PMD in the future (not as part of this first series) with a mode that we refer to as the bifurcated PMD. The bifurcated PMD communicates with a kernel driver to configure the device, ports, and queues, and memory maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in this patchset, and is why the iface.[ch] layer is present.

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of umonitor/umwait, rather than the new functions in the power patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The software is very similar in form so some DLB2 reviews comments were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ memory for depths < 8. Hardware requires minimum allocation to be at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias. I will address the remaining items identified by Mattias in the next patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
event/dlb: add documentation and meson infrastructure event/dlb: add dynamic logging
event/dlb: add private data structures and constants event/dlb: add definitions shared with LKM or shared code event/dlb: add inline functions
event/dlb: add eventdev probe
event/dlb: add flexible interface
event/dlb: add probe-time hardware init event/dlb: add xstats
event/dlb: add infos get and configure event/dlb: add queue and port default conf event/dlb: add queue setup
event/dlb: add port setup
event/dlb: add port link
event/dlb: add port unlink and port unlinks in progress event/dlb: add eventdev start
event/dlb: add enqueue and its burst variants event/dlb: add dequeue and its burst variants event/dlb: add eventdev stop and close event/dlb: add PMD's token pop public interface event/dlb: add PMD self-tests
event/dlb: add queue and port release
event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   59 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21703 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map
Signature · 1 line
2.6.4
#137 lore raw

PATCH v12 1/23 event/dlb: add documentation and meson infrastructure

Note that config/rte_config.h contains several configuration switches, providing for fine control of the PMD's runtime behaviour.

The meson infrastructure is expanded as additional files are added to this patchset.

Adds announcement of availabililty of the new driver for Intel Dynamic Load Balancer 1.0 hardware.

---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map
MAINTAINERS +5 -1
index a3d1927..b904132 100644
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <[email protected]>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <[email protected]>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <[email protected]>
 M: Jerin Jacob <[email protected]>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <[email protected]>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
config/rte_config.h +6 -0
index b78c6aa..9ebe4cc 100644
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
doc/guides/eventdevs/dlb.rst new file +36 -0
new file mode 100644
index 0000000..92341c0
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
doc/guides/eventdevs/index.rst +1 -0
index bb66a5e..4b915bf 100644
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
doc/guides/rel_notes/release_20_11.rst +5 -0
index d8ac359..0a95bf0 100644
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
drivers/event/dlb/meson.build new file +13 -0
new file mode 100644
index 0000000..5324043
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/version.map new file +3 -0
new file mode 100644
index 0000000..4a76d1d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
drivers/event/meson.build +1 -1
index a7dac99..6601e62 100644
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
Signature · 1 line
2.6.4
#138 lore raw

PATCH v12 2/23 event/dlb: add dynamic logging

In reply to Timothy McDaniel

This commit adds base support for dynamic logging. The default log level is NOTICE. Dynamic logging is used exclusively throughout this patchset.

---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h
drivers/event/dlb/dlb.c new file +7 -0
new file mode 100644
index 0000000..e03aa21
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
drivers/event/dlb/dlb_log.h new file +25 -0
new file mode 100644
index 0000000..c69c9e5
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
drivers/event/dlb/meson.build +1 -2
index 5324043..1e7d5ad 100644
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
Signature · 1 line
2.6.4
#139 lore raw

PATCH v12 3/23 event/dlb: add private data structures and constants

In reply to Timothy McDaniel

Add headers used internally by the PMD. They include constants, macros for device resources, structure definitions for hardware interfaces and software state, and various forward-declarations.

---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h
drivers/event/dlb/dlb_priv.h new file +508 -0
new file mode 100644
index 0000000..f9ff0a5
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
Signature · 1 line
2.6.4
#140 lore raw

PATCH v12 4/23 event/dlb: add definitions shared with LKM or shared code

In reply to Timothy McDaniel

Add headers containing structs and constants shared between the PMD and the shared code. The term shared code refers to the code that implements the hardware interface. The shared code is introduced in the probe patch, and then is extended as additional eventdev PMD entry points are added to the patchset. In the case of the bifurcated PMD (to be introduced in the future), the shared code is contained in the Linux kernel module itself.

---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h
drivers/event/dlb/dlb_user.h new file +814 -0
new file mode 100644
index 0000000..2d9582b
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
Signature · 1 line
2.6.4
#141 lore raw

PATCH v12 5/23 event/dlb: add inline functions

In reply to Timothy McDaniel

Add miscellaneous inline functions that may be called from multiple files. These functions include inline assembly of new x86 instructions, such as movdir64b, since they are not available as builtin functions in the minimum supported GCC version.

---
 drivers/event/dlb/dlb_inline_fns.h | 59 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
drivers/event/dlb/dlb_inline_fns.h new file +59 -0
new file mode 100644
index 0000000..82edbb2
@@ -0,0 +1,59 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_umonitor(volatile void *addr)
+{
+	asm volatile(".byte 0xf3, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (addr));
+}
+
+static inline void
+dlb_umwait(int state, uint64_t timeout)
+{
+	uint32_t eax = timeout & UINT32_MAX;
+	uint32_t edx = timeout >> 32;
+
+	asm volatile(".byte 0xf2, 0x0f, 0xae, 0xf7\t\n"
+			:
+			: "D" (state),  "a" (eax), "d" (edx));
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
Signature · 1 line
2.6.4
#142 lore raw

PATCH v12 6/23 event/dlb: add eventdev probe

In reply to Timothy McDaniel

Add the eventdev portion of probe, and parse command line options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and will be introduced in subsequent patches contained in this patch-set.

Hardware init has been moved to a subsequent patch in order to minimize the patch size.

Initialization of the flexible interface layer has been moved to a subsequent patch in order to minimize patch size.

---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
drivers/event/dlb/dlb.c +327 -0
index e03aa21..1659f93 100644
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
drivers/event/dlb/dlb_priv.h +2 -0
index f9ff0a5..adb1f7a 100644
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
drivers/event/dlb/meson.build +4 -1
index 1e7d5ad..b4bdc8b 100644
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/pf/base/dlb_hw_types.h new file +334 -0

334 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep.h new file +310 -0

310 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep_bitmap.h new file +441 -0

441 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep_list.h new file +131 -0

131 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep_types.h new file +31 -0
new file mode 100644
index 0000000..2e9d7d8
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
drivers/event/dlb/pf/base/dlb_regs.h new file +2368 -0

1,404 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_resource.h new file +876 -0

876 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.c new file +568 -0

568 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.h new file +47 -0

47 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c new file +147 -0

147 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.6.4

Part of the diff is not shown here. View the raw message.

#143 lore raw

PATCH v12 7/23 event/dlb: add flexible interface

In reply to Timothy McDaniel

This commit introduces the flexible interface. This interface allows the core code to operate in PF mode (direct hardware access) or bifurcated mode (hardware configured via kernel driver). This driver currently only supports PF modei, but bifurcated mode will be added in a future patch-set. Note that the flexible interface is not used for data path operations, and thus there are no performance concerns related to the use of function pointers.

---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
drivers/event/dlb/dlb.c +1 -0
index 1659f93..8008a50 100644
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
drivers/event/dlb/dlb_iface.c new file +27 -0
new file mode 100644
index 0000000..dd72120
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
drivers/event/dlb/dlb_iface.h new file +27 -0
new file mode 100644
index 0000000..416d1b3
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
drivers/event/dlb/meson.build +1 -0
index b4bdc8b..8707d3d 100644
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
drivers/event/dlb/pf/dlb_pf.c +1 -0
index 3f836f3..05fd76c 100644
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
Signature · 1 line
2.6.4
#144 lore raw

PATCH v12 8/23 event/dlb: add probe-time hardware init

In reply to Timothy McDaniel

This commit adds probe-time low level hardware initialization. It also adds probe-time init for both primary and secondary DPDK processes.

---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
drivers/event/dlb/dlb.c +153 -5
index 8008a50..57b2837 100644
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
drivers/event/dlb/meson.build +2 -1
index 8707d3d..9777178 100644
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/pf/base/dlb_resource.c new file +302 -0

302 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.c +19 -1
index c10c36c..2f4a828 100644
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
drivers/event/dlb/pf/dlb_pf.c +85 -1

94 lines not on this page · raw message

Signature · 1 line
2.6.4
#145 lore raw

PATCH v12 9/23 event/dlb: add xstats

In reply to Timothy McDaniel

Add support for DLB xstats. Perform initialization and add standard xstats entry points

---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c
drivers/event/dlb/dlb.c +23 -0
index 57b2837..62b9695 100644
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
drivers/event/dlb/dlb_xstats.c new file +1222 -0

1,222 lines not on this page · raw message

drivers/event/dlb/meson.build +1 -0
index 9777178..552ff9d 100644
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
Signature · 1 line
2.6.4
#146 lore raw

PATCH v12 10/23 event/dlb: add infos get and configure

In reply to Timothy McDaniel

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB hardware's scheduling domain, such that it is provisioned with the requested number of ports and queues, provided sufficient resources are available. Individual queues and ports are configured later in port setup and eventdev start.

---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)
doc/guides/eventdevs/dlb.rst +48 -0
index 92341c0..2d7999b 100644
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
drivers/event/dlb/dlb.c +397 -0

415 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +11 -0
index dd72120..f3e82f2 100644
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +11 -0
index 416d1b3..d576232 100644
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +4007 -93

2,488 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +88 -0

88 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.6.4

Part of the diff is not shown here. View the raw message.

#147 lore raw

PATCH v12 11/23 event/dlb: add queue and port default conf

In reply to Timothy McDaniel

Add support for getting the queue and port default configuration.

---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
drivers/event/dlb/dlb.c +29 -0
index c038794..e98a438 100644
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#148 lore raw

PATCH v12 12/23 event/dlb: add queue setup

In reply to Timothy McDaniel

Load balanced (ldb) queues are setup here. Directed queues are not set up until link time, at which point we know the directed port ID. Directed queue setup will only fail if this queue is already setup or there are no directed queues left to configure.

---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)
doc/guides/eventdevs/dlb.rst +35 -0
index 2d7999b..d8e936a 100644
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
drivers/event/dlb/dlb.c +293 -0

305 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +12 -0
index f3e82f2..219f79e 100644
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
drivers/event/dlb/dlb_iface.h +12 -0
index d576232..af1416d 100644
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
drivers/event/dlb/pf/base/dlb_resource.c +386 -0

389 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +81 -0

94 lines not on this page · raw message

Signature · 1 line
2.6.4
#149 lore raw

PATCH v12 13/23 event/dlb: add port setup

In reply to Timothy McDaniel

Configure the load balanced (ldb) or directed (dir) port. The consumer queue (CQ) and producer port (PP) are also set up here.

---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)
doc/guides/eventdevs/dlb.rst +40 -0
index d8e936a..f106a07 100644
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
drivers/event/dlb/dlb.c +515 -1

546 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +11 -0
index 219f79e..fbbf9d7 100644
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +14 -0
index af1416d..d578185 100644
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +1433 -3

1,457 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +210 -0

222 lines not on this page · raw message

Signature · 1 line
2.6.4
#150 lore raw

PATCH v12 14/23 event/dlb: add port link

In reply to Timothy McDaniel

Add port link entry point. Directed queues are identified and created at this stage. Their setup deferred until link-time, at which point we know the directed port ID. Directed queue setup will only fail if this queue is already setup or there are no directed queues left to configure.

---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)
drivers/event/dlb/dlb.c +306 -0
index 4d91ddd..2ad195d 100644
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
drivers/event/dlb/dlb_iface.c +9 -0
index fbbf9d7..aaf4506 100644
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +9 -0
index d578185..c0f5f2e 100644
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +641 -0

644 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +69 -0

81 lines not on this page · raw message

Signature · 1 line
2.6.4
#151 lore raw

PATCH v12 15/23 event/dlb: add port unlink and port unlinks in progress

In reply to Timothy McDaniel

Add supports for the port unlink(s) eventdev entry points. The unlink operation is an asynchronous operation executed by a control thread, and the unlinks-in-progress function reads a counter shared with the control thread. Port QE and memzone memory is freed here.

---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)
drivers/event/dlb/dlb.c +166 -0
index 2ad195d..c64f559 100644
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#152 lore raw

PATCH v12 16/23 event/dlb: add eventdev start

In reply to Timothy McDaniel

Add support for the eventdev start entry point. DLB delays setting up single link resources until eventdev start, because it is only then that it can ascertain which ports have just one linked queue.

---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)
drivers/event/dlb/dlb.c +180 -44
index c64f559..780ff7d 100644
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
drivers/event/dlb/dlb_iface.c +3 -0
index aaf4506..22d524b 100644
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
drivers/event/dlb/dlb_iface.h +3 -0
index c0f5f2e..8c905ab 100644
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
drivers/event/dlb/pf/base/dlb_resource.c +142 -0

151 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +23 -0
index fed6719..1d2e133 100644
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
Signature · 1 line
2.6.4
#153 lore raw

PATCH v12 17/23 event/dlb: add enqueue and its burst variants

In reply to Timothy McDaniel

Add support for enqueue and its variants.

---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)
doc/guides/eventdevs/dlb.rst +162 -1
index f106a07..ae126c4 100644
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
drivers/event/dlb/dlb.c +682 -0

700 lines not on this page · raw message

Signature · 1 line
2.6.4
#154 lore raw

PATCH v12 18/23 event/dlb: add dequeue and its burst variants

In reply to Timothy McDaniel

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses umonitor/umwait if supported by the processor. This allows the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the hardware will pack 4 QEs into each cache line. In sparse cq mode, the hardware will only populate one QE per cache line. Software must be aware of the cq mode, and take the appropriate actions, based on the mode.

---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 679 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 700 insertions(+)
doc/guides/eventdevs/dlb.rst +21 -0
index ae126c4..4c4f56b 100644
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
drivers/event/dlb/dlb.c +679 -0

700 lines not on this page · raw message

Signature · 1 line
2.6.4
#155 lore raw

PATCH v12 19/23 event/dlb: add eventdev stop and close

In reply to Timothy McDaniel

Add support for eventdev stop and close entry points.

---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)
drivers/event/dlb/dlb.c +245 -11
index e2f769c..a8efcc1 100644
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3480,6 +3469,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3489,6 +3721,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
drivers/event/dlb/dlb_iface.c +6 -0
index 22d524b..44f958f 100644
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
drivers/event/dlb/dlb_iface.h +6 -0
index 8c905ab..9f61135 100644
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
drivers/event/dlb/pf/base/dlb_resource.c +89 -0

92 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +47 -0

63 lines not on this page · raw message

Signature · 1 line
2.6.4
#156 lore raw

PATCH v12 20/23 event/dlb: add PMD's token pop public interface

In reply to Timothy McDaniel

The PMD uses a public interface to allow applications to control the token pop mode. Supported token pop modes are as follows, and they impact core scheduling affinity for ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
are released. Supported on load-balanced ports only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
doc/api/doxy-api-index.md +1 -0
index a9c12d1..1c83bf4 100644
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
drivers/event/dlb/dlb.c +109 -12

247 lines not on this page · raw message

drivers/event/dlb/dlb_priv.h +3 -0
index adb1f7a..58ff428 100644
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
drivers/event/dlb/meson.build +3 -1
index 552ff9d..7f38c30 100644
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
drivers/event/dlb/rte_pmd_dlb.c new file +38 -0
new file mode 100644
index 0000000..bc802d3
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
drivers/event/dlb/rte_pmd_dlb.h new file +77 -0

77 lines not on this page · raw message

drivers/event/dlb/version.map +6 -0
index 4a76d1d..3338a22 100644
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
Signature · 1 line
2.6.4
#157 lore raw

PATCH v12 21/23 event/dlb: add PMD self-tests

In reply to Timothy McDaniel

Add a variety of self-tests for both ldb and directed ports/queues, as well as configure, start, stop, link, etc...

---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c
app/test/test_eventdev.c +7 -0
index 62019c1..ba27bed 100644
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
drivers/event/dlb/dlb.c +1 -0
index 31e4d50..c37de85 100644
@@ -3829,6 +3829,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
drivers/event/dlb/dlb_selftest.c new file +1539 -0

1,539 lines not on this page · raw message

drivers/event/dlb/meson.build +1 -0
index 7f38c30..875cf89 100644
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
Signature · 1 line
2.6.4
#158 lore raw

PATCH v12 22/23 event/dlb: add queue and port release

In reply to Timothy McDaniel

These entry points are NO-OPS. DLB does not support reconfiguring individual queues or ports. The entire device must be reconfigured.

---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
drivers/event/dlb/dlb.c +27 -0
index c37de85..41ccb4a 100644
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3805,6 +3808,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3819,7 +3844,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
Signature · 1 line
2.6.4
#159 lore raw

PATCH v12 23/23 event/dlb: add timeout ticks entry point

In reply to Timothy McDaniel

Adds the timeout ticks conversion function.

---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)
drivers/event/dlb/dlb.c +13 -0
index 41ccb4a..4aab9f6 100644
@@ -3830,6 +3830,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3851,6 +3863,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#160 lore raw

v13 cover Add DLB PMD

In reply to McDaniel, Timothy

The following patch series adds support for a new eventdev PMD. The DLB PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware. The DLB is a PCIe device that provides load-balanced, prioritized scheduling of core-to-core communication. The device consists of queues and arbiters that connect producer and consumer cores, and implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced distribution.

The DLB hardware supports both load balanced and directed ports and queues. Unlike other eventdev devices already in the repo, not all DLB ports and queues are equally capable. In particular, directed ports are limited to a single link, and must be connected to a directed queue.
Additionally, even though LDB ports may link multiple queues, the number of queues that may be linked is limited by hardware. Another difference is that DLB does not have a straightforward way of carrying the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full control over the DLB hardware. Intel will be extending the DLB PMD in the future (not as part of this first series) with a mode that we refer to as the bifurcated PMD. The bifurcated PMD communicates with a kernel driver to configure the device, ports, and queues, and memory maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in this patchset, and is why the iface.[ch] layer is present.

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of umonitor/umwait, rather than the new functions in the power patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The software is very similar in form so some DLB2 reviews comments were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ memory for depths < 8. Hardware requires minimum allocation to be at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias. I will address the remaining items identified by Mattias in the next patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
event/dlb: add documentation and meson infrastructure event/dlb: add dynamic logging
event/dlb: add private data structures and constants event/dlb: add definitions shared with LKM or shared code event/dlb: add inline functions
event/dlb: add eventdev probe
event/dlb: add flexible interface
event/dlb: add probe-time hardware init event/dlb: add xstats
event/dlb: add infos get and configure event/dlb: add queue and port default conf event/dlb: add queue setup
event/dlb: add port setup
event/dlb: add port link
event/dlb: add port unlink and port unlinks in progress event/dlb: add eventdev start
event/dlb: add enqueue and its burst variants event/dlb: add dequeue and its burst variants event/dlb: add eventdev stop and close event/dlb: add PMD's token pop public interface event/dlb: add PMD self-tests
event/dlb: add queue and port release
event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   40 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21684 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map
Signature · 1 line
2.6.4
#161 lore raw

PATCH v13 1/23 event/dlb: add documentation and meson infrastructure

Note that config/rte_config.h contains several configuration switches, providing for fine control of the PMD's runtime behaviour.

The meson infrastructure is expanded as additional files are added to this patchset.

Adds announcement of availabililty of the new driver for Intel Dynamic Load Balancer 1.0 hardware.

---
 MAINTAINERS                            |  6 +++++-
 config/rte_config.h                    |  6 ++++++
 doc/guides/eventdevs/dlb.rst           | 36 ++++++++++++++++++++++++++++++++++
 doc/guides/eventdevs/index.rst         |  1 +
 doc/guides/rel_notes/release_20_11.rst |  5 +++++
 drivers/event/dlb/meson.build          | 13 ++++++++++++
 drivers/event/dlb/version.map          |  3 +++
 drivers/event/meson.build              |  2 +-
 8 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/version.map
MAINTAINERS +5 -1
index a3d1927..b904132 100644
@@ -1162,6 +1162,11 @@ Cavium OCTEON TX timvf
 M: Pavan Nikhilesh <[email protected]>
 F: drivers/event/octeontx/timvf_*
 
+Intel DLB
+M: Timothy McDaniel <[email protected]>
+F: drivers/event/dlb/
+F: doc/guides/eventdevs/dlb.rst
+
 Marvell OCTEON TX2
 M: Pavan Nikhilesh <[email protected]>
 M: Jerin Jacob <[email protected]>
@@ -1198,7 +1203,6 @@ M: Peter Mccarthy <[email protected]>
 F: drivers/event/opdl/
 F: doc/guides/eventdevs/opdl.rst
 
-
 Rawdev Drivers
 --------------
 
config/rte_config.h +6 -0
index b78c6aa..9ebe4cc 100644
@@ -135,4 +135,10 @@
 /* QEDE PMD defines */
 #define RTE_LIBRTE_QEDE_FW ""
 
+/* DLB PMD defines */
+#define RTE_LIBRTE_PMD_DLB_POLL_INTERVAL 1000
+#define RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE  0
+#undef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define RTE_LIBRTE_PMD_DLB_SW_CREDIT_QUANTA 32
+
 #endif /* _RTE_CONFIG_H_ */
doc/guides/eventdevs/dlb.rst new file +36 -0
new file mode 100644
index 0000000..92341c0
@@ -0,0 +1,36 @@
+..  SPDX-License-Identifier: BSD-3-Clause
+    Copyright(c) 2020 Intel Corporation.
+
+Driver for the Intel® Dynamic Load Balancer (DLB)
+==================================================
+
+The DPDK dlb poll mode driver supports the Intel® Dynamic Load Balancer.
+
+Prerequisites
+-------------
+
+Follow the DPDK :ref:`Getting Started Guide for Linux <linux_gsg>` to setup
+the basic DPDK environment.
+
+Configuration
+-------------
+
+The DLB PF PMD is a user-space PMD that uses VFIO to gain direct
+device access. To use this operation mode, the PCIe PF device must be bound
+to a DPDK-compatible VFIO driver, such as vfio-pci.
+
+Eventdev API Notes
+------------------
+
+The DLB provides the functions of a DPDK event device; specifically, it
+supports atomic, ordered, and parallel scheduling events from queues to ports.
+However, the DLB hardware is not a perfect match to the eventdev API. Some DLB
+features are abstracted by the PMD (e.g. directed ports), some are only
+accessible as vdev command-line parameters, and certain eventdev features are
+not supported (e.g. the event flow ID is not maintained during scheduling).
+
+In general the dlb PMD is designed for ease-of-use and does not require a
+detailed understanding of the hardware, but these details are important when
+writing high-performance code. This section describes the places where the
+eventdev API and DLB misalign.
+
doc/guides/eventdevs/index.rst +1 -0
index bb66a5e..4b915bf 100644
@@ -11,6 +11,7 @@ application through the eventdev API.
     :maxdepth: 2
     :numbered:
 
+    dlb
     dpaa
     dpaa2
     dsw
doc/guides/rel_notes/release_20_11.rst +5 -0
index d8ac359..0a95bf0 100644
@@ -55,6 +55,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =======================================================
 
+* **Added a new driver for the Intel Dynamic Load Balancer v1.0 device.**
+
+  Added the new ``dlb`` eventdev driver for the Intel DLB V1.0 device. See the
+  :doc:`../eventdevs/dlb` eventdev guide for more details on this new driver.
+
 * **Added write combining store APIs.**
 
   Added ``rte_write32_wc`` and ``rte_write32_wc_relaxed`` APIs
drivers/event/dlb/meson.build new file +13 -0
new file mode 100644
index 0000000..5324043
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2019-2020 Intel Corporation
+
+if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
+        build = false
+        reason = 'only supported on ARCH_X86_64 Linux'
+        subdir_done()
+endif
+
+sources = files(
+)
+
+deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/version.map new file +3 -0
new file mode 100644
index 0000000..4a76d1d
@@ -0,0 +1,3 @@
+DPDK_21 {
+	local: *;
+};
drivers/event/meson.build +1 -1
index a7dac99..6601e62 100644
@@ -5,7 +5,7 @@ if is_windows
 	subdir_done()
 endif
 
-drivers = ['dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
+drivers = ['dlb', 'dpaa', 'dpaa2', 'octeontx2', 'opdl', 'skeleton', 'sw', 'dsw']
 if not (toolchain == 'gcc' and cc.version().version_compare('<4.8.6') and
 	dpdk_conf.has('RTE_ARCH_ARM64'))
 	drivers += 'octeontx'
Signature · 1 line
2.6.4
#162 lore raw

PATCH v13 2/23 event/dlb: add dynamic logging

In reply to Timothy McDaniel

This commit adds base support for dynamic logging. The default log level is NOTICE. Dynamic logging is used exclusively throughout this patchset.

---
 drivers/event/dlb/dlb.c       |  7 +++++++
 drivers/event/dlb/dlb_log.h   | 25 +++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  3 +--
 3 files changed, 33 insertions(+), 2 deletions(-)
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_log.h
drivers/event/dlb/dlb.c new file +7 -0
new file mode 100644
index 0000000..e03aa21
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <rte_log.h>
+
+RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
drivers/event/dlb/dlb_log.h new file +25 -0
new file mode 100644
index 0000000..c69c9e5
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_EVDEV_LOG_H_
+#define _DLB_EVDEV_LOG_H_
+
+extern int eventdev_dlb_log_level;
+
+/* Dynamic logging */
+#define DLB_LOG_IMPL(level, fmt, args...) \
+	rte_log(RTE_LOG_ ## level, eventdev_dlb_log_level, "%s" fmt "\n", \
+		__func__, ##args)
+
+#define DLB_LOG_INFO(fmt, args...) \
+	DLB_LOG_IMPL(INFO, fmt, ## args)
+
+#define DLB_LOG_ERR(fmt, args...) \
+	DLB_LOG_IMPL(ERR, fmt, ## args)
+
+/* remove debug logs at compile time unless actually debugging */
+#define DLB_LOG_DBG(fmt, args...) \
+	RTE_LOG_DP(DEBUG, PMD, fmt, ## args)
+
+#endif /* _DLB_EVDEV_LOG_H_ */
drivers/event/dlb/meson.build +1 -2
index 5324043..1e7d5ad 100644
@@ -7,7 +7,6 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files(
-)
+sources = files('dlb.c')
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
Signature · 1 line
2.6.4
#163 lore raw

PATCH v13 3/23 event/dlb: add private data structures and constants

In reply to Timothy McDaniel

Add headers used internally by the PMD. They include constants, macros for device resources, structure definitions for hardware interfaces and software state, and various forward-declarations.

---
 drivers/event/dlb/dlb_priv.h | 508 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 508 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_priv.h
drivers/event/dlb/dlb_priv.h new file +508 -0
new file mode 100644
index 0000000..f9ff0a5
@@ -0,0 +1,508 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_PRIV_H_
+#define _DLB_PRIV_H_
+
+#include <emmintrin.h>
+#include <stdbool.h>
+
+#include <rte_bus_pci.h>
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+#include <rte_eventdev_pmd_pci.h>
+#include <rte_pci.h>
+
+#include "dlb_user.h"
+#include "dlb_log.h"
+
+#ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
+#define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
+#else
+#define DLB_INC_STAT(_stat, _incr_val)
+#endif
+
+#define EVDEV_DLB_NAME_PMD_STR "dlb_event"
+
+/* command line arg strings */
+#define NUMA_NODE_ARG "numa_node"
+#define DLB_MAX_NUM_EVENTS "max_num_events"
+#define DLB_NUM_DIR_CREDITS "num_dir_credits"
+#define DEV_ID_ARG "dev_id"
+#define DLB_DEFER_SCHED_ARG "defer_sched"
+#define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
+
+/* Begin HW related defines and structs */
+
+#define DLB_MAX_NUM_DOMAINS 32
+#define DLB_MAX_NUM_VFS 16
+#define DLB_MAX_NUM_LDB_QUEUES 128
+#define DLB_MAX_NUM_LDB_PORTS 64
+#define DLB_MAX_NUM_DIR_PORTS 128
+#define DLB_MAX_NUM_DIR_QUEUES 128
+#define DLB_MAX_NUM_FLOWS (64 * 1024)
+#define DLB_MAX_NUM_LDB_CREDITS 16384
+#define DLB_MAX_NUM_DIR_CREDITS 4096
+#define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
+#define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
+#define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
+#define DLB_MAX_NUM_ATM_INFLIGHTS 2048
+#define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
+#define DLB_QID_PRIORITIES 8
+#define DLB_MAX_DEVICE_PATH 32
+#define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
+#define DLB_NUM_SN_GROUPS 4
+#define DLB_MAX_LDB_SN_ALLOC 1024
+/* Note: "- 1" here to support the timeout range check in eventdev_autotest */
+#define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
+#define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
+
+/* 5120 total hist list entries and 64 total ldb ports, which
+ * makes for 5120/64 == 80 hist list entries per port. However, CQ
+ * depth must be a power of 2 and must also be >= HIST LIST entries.
+ * As a result we just limit the maximum dequeue depth to 64.
+ */
+#define DLB_MIN_LDB_CQ_DEPTH 1
+#define DLB_MIN_DIR_CQ_DEPTH 8
+#define DLB_MIN_HARDWARE_CQ_DEPTH 8
+#define DLB_MAX_CQ_DEPTH 64
+#define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
+	DLB_MAX_CQ_DEPTH
+
+/* Static per queue/port provisioning values */
+#define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
+
+#define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
+
+#define PAGE_SIZE (sysconf(_SC_PAGESIZE))
+
+#define DLB_NUM_QES_PER_CACHE_LINE 4
+
+#define DLB_MAX_ENQUEUE_DEPTH 64
+#define DLB_MIN_ENQUEUE_DEPTH 4
+
+#define DLB_NAME_SIZE 64
+
+/* Use the upper 3 bits of the event priority to select the DLB priority */
+#define EV_TO_DLB_PRIO(x) ((x) >> 5)
+#define DLB_TO_EV_PRIO(x) ((x) << 5)
+
+enum dlb_hw_port_type {
+	DLB_LDB,
+	DLB_DIR,
+
+	/* NUM_DLB_PORT_TYPES must be last */
+	NUM_DLB_PORT_TYPES
+};
+
+#define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
+
+/* Do not change - must match hardware! */
+enum dlb_hw_sched_type {
+	DLB_SCHED_ATOMIC = 0,
+	DLB_SCHED_UNORDERED,
+	DLB_SCHED_ORDERED,
+	DLB_SCHED_DIRECTED,
+
+	/* DLB_NUM_HW_SCHED_TYPES must be last */
+	DLB_NUM_HW_SCHED_TYPES
+};
+
+struct dlb_devargs {
+	int socket_id;
+	int max_num_events;
+	int num_dir_credits_override;
+	int dev_id;
+	int defer_sched;
+	int num_atm_inflights;
+};
+
+struct dlb_hw_rsrcs {
+	int32_t nb_events_limit;
+	uint32_t num_queues;		/* Total queues (ldb + dir) */
+	uint32_t num_ldb_queues;	/* Number of available ldb queues */
+	uint32_t num_ldb_ports;         /* Number of load balanced ports */
+	uint32_t num_dir_ports;         /* Number of directed ports */
+	uint32_t num_ldb_credits;       /* Number of load balanced credits */
+	uint32_t num_dir_credits;       /* Number of directed credits */
+	uint32_t reorder_window_size;   /* Size of reorder window */
+};
+
+struct dlb_hw_resource_info {
+	/**> Max resources that can be provided */
+	struct dlb_hw_rsrcs hw_rsrc_max;
+	int num_sched_domains;
+	uint32_t socket_id;
+	/**> EAL flags passed to this DLB instance, allowing the application to
+	 * identify the pmd backend indicating hardware or software.
+	 */
+	const char *eal_flags;
+};
+
+/* hw-specific format - do not change */
+
+struct dlb_event_type {
+	uint8_t major:4;
+	uint8_t unused:4;
+	uint8_t sub;
+};
+
+union dlb_opaque_data {
+	uint16_t opaque_data;
+	struct dlb_event_type event_type;
+};
+
+struct dlb_msg_info {
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+};
+
+#define DLB_NEW_CMD_BYTE 0x08
+#define DLB_FWD_CMD_BYTE 0x0A
+#define DLB_COMP_CMD_BYTE 0x02
+#define DLB_NOOP_CMD_BYTE 0x00
+#define DLB_POP_CMD_BYTE 0x01
+
+/* hw-specific format - do not change */
+struct dlb_enqueue_qe {
+	uint64_t data;
+	/* Word 3 */
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	/* Word 4 */
+	uint16_t lock_id;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_cq_pop_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t tokens:10;
+	uint16_t rsvd2:6;
+	uint8_t meas_lat:1;
+	uint8_t rsvd1:2;
+	uint8_t no_dec:1;
+	uint8_t cmp_id:4;
+	union {
+		uint8_t cmd_byte;
+		struct {
+			uint8_t cq_token:1;
+			uint8_t qe_comp:1;
+			uint8_t qe_frag:1;
+			uint8_t qe_valid:1;
+			uint8_t int_arm:1;
+			uint8_t error:1;
+			uint8_t rsvd:2;
+		};
+	};
+};
+
+/* hw-specific format - do not change */
+struct dlb_dequeue_qe {
+	uint64_t data;
+	union dlb_opaque_data u;
+	uint8_t qid;
+	uint8_t sched_type:2;
+	uint8_t priority:3;
+	uint8_t msg_type:3;
+	uint16_t pp_id:10;
+	uint16_t rsvd0:6;
+	uint8_t debug;
+	uint8_t cq_gen:1;
+	uint8_t qid_depth:1;
+	uint8_t rsvd1:3;
+	uint8_t error:1;
+	uint8_t rsvd2:2;
+};
+
+enum dlb_port_state {
+	PORT_CLOSED,
+	PORT_STARTED,
+	PORT_STOPPED
+};
+
+enum dlb_configuration_state {
+	/* The resource has not been configured */
+	DLB_NOT_CONFIGURED,
+	/* The resource was configured, but the device was stopped */
+	DLB_PREV_CONFIGURED,
+	/* The resource is currently configured */
+	DLB_CONFIGURED
+};
+
+struct dlb_port {
+	uint32_t id;
+	bool is_directed;
+	bool gen_bit;
+	uint16_t dir_credits;
+	uint32_t dequeue_depth;
+	int pp_mmio_base;
+	uint16_t cached_ldb_credits;
+	uint16_t ldb_pushcount_at_credit_expiry;
+	uint16_t ldb_credits;
+	uint16_t cached_dir_credits;
+	uint16_t dir_pushcount_at_credit_expiry;
+	bool int_armed;
+	bool use_rsvd_token_scheme;
+	uint8_t cq_rsvd_token_deficit;
+	uint16_t owed_tokens;
+	int16_t issued_releases;
+	int cq_depth;
+	uint16_t cq_idx;
+	uint16_t cq_idx_unmasked;
+	uint16_t cq_depth_mask;
+	uint16_t gen_bit_shift;
+	enum dlb_port_state state;
+	enum dlb_configuration_state config_state;
+	int num_mapped_qids;
+	uint8_t *qid_mappings;
+	struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
+	struct dlb_cq_pop_qe *consume_qe;
+	struct dlb_eventdev *dlb; /* back ptr */
+	struct dlb_eventdev_port *ev_port; /* back ptr */
+};
+
+/* Per-process per-port mmio and memory pointers */
+struct process_local_port_data {
+	uint64_t *pp_addr;
+	uint16_t *ldb_popcount;
+	uint16_t *dir_popcount;
+	struct dlb_dequeue_qe *cq_base;
+	const struct rte_memzone *mz;
+	bool mmaped;
+};
+
+struct dlb_config {
+	int configured;
+	int reserved;
+	uint32_t ldb_credit_pool_id;
+	uint32_t dir_credit_pool_id;
+	uint32_t num_ldb_credits;
+	uint32_t num_dir_credits;
+	struct dlb_create_sched_domain_args resources;
+};
+
+struct dlb_hw_dev {
+	struct dlb_config cfg;
+	struct dlb_hw_resource_info info;
+	void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
+	int device_id;
+	uint32_t domain_id;
+	int domain_id_valid;
+	rte_spinlock_t resource_lock; /* for MP support */
+} __rte_cache_aligned;
+
+/* End HW related defines and structs */
+
+/* Begin DLB PMD Eventdev related defines and structs */
+
+#define DLB_MAX_NUM_QUEUES \
+	(DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
+
+#define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
+#define DLB_MAX_INPUT_QUEUE_DEPTH 256
+
+/** Structure to hold the queue to port link establishment attributes */
+
+struct dlb_event_queue_link {
+	uint8_t queue_id;
+	uint8_t priority;
+	bool mapped;
+	bool valid;
+};
+
+struct dlb_traffic_stats {
+	uint64_t rx_ok;
+	uint64_t rx_drop;
+	uint64_t rx_interrupt_wait;
+	uint64_t rx_umonitor_umwait;
+	uint64_t tx_ok;
+	uint64_t total_polls;
+	uint64_t zero_polls;
+	uint64_t tx_nospc_ldb_hw_credits;
+	uint64_t tx_nospc_dir_hw_credits;
+	uint64_t tx_nospc_inflight_max;
+	uint64_t tx_nospc_new_event_limit;
+	uint64_t tx_nospc_inflight_credits;
+};
+
+struct dlb_port_stats {
+	struct dlb_traffic_stats traffic;
+	uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
+	uint64_t tx_implicit_rel;
+	uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t tx_invalid;
+	uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
+	uint64_t rx_sched_invalid;
+	uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
+};
+
+struct dlb_eventdev_port {
+	struct dlb_port qm_port; /* hw specific data structure */
+	struct rte_event_port_conf conf; /* user-supplied configuration */
+	uint16_t inflight_credits; /* num credits this port has right now */
+	uint16_t credit_update_quanta;
+	struct dlb_eventdev *dlb; /* backlink optimization */
+	struct dlb_port_stats stats __rte_cache_aligned;
+	struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
+	int num_links;
+	uint32_t id;
+	/* num releases yet to be completed on this port.
+	 * Only applies to load-balanced ports.
+	 */
+	uint16_t outstanding_releases;
+	uint16_t inflight_max; /* app requested max inflights for this port */
+	/* setup_done is set when the event port is setup */
+	bool setup_done;
+	/* enq_configured is set when the qm port is created */
+	bool enq_configured;
+	uint8_t implicit_release; /* release events before dequeueing */
+} __rte_cache_aligned;
+
+struct dlb_queue {
+	uint32_t num_qid_inflights; /* User config */
+	uint32_t num_atm_inflights; /* User config */
+	enum dlb_configuration_state config_state;
+	int sched_type; /* LB queue only */
+	uint32_t id;
+	bool is_directed;
+};
+
+struct dlb_eventdev_queue {
+	struct dlb_queue qm_queue;
+	struct rte_event_queue_conf conf; /* User config */
+	uint64_t enq_ok;
+	uint32_t id;
+	bool setup_done;
+	uint8_t num_links;
+};
+
+enum dlb_run_state {
+	DLB_RUN_STATE_STOPPED = 0,
+	DLB_RUN_STATE_STOPPING,
+	DLB_RUN_STATE_STARTING,
+	DLB_RUN_STATE_STARTED
+};
+
+struct dlb_eventdev {
+	struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
+	struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+	uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each queue */
+	uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
+	uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
+
+	/* store num stats and offset of the stats for each port */
+	uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
+	uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
+	struct dlb_get_num_resources_args hw_rsrc_query_results;
+	uint32_t xstats_count_mode_queue;
+	struct dlb_hw_dev qm_instance; /* strictly hw related */
+	uint64_t global_dequeue_wait_ticks;
+	struct dlb_xstats_entry *xstats;
+	struct rte_eventdev *event_dev; /* backlink to dev */
+	uint32_t xstats_count_mode_port;
+	uint32_t xstats_count_mode_dev;
+	uint32_t xstats_count;
+	uint32_t inflights; /* use __atomic builtins to access */
+	uint32_t new_event_limit;
+	int max_num_events_override;
+	int num_dir_credits_override;
+	volatile enum dlb_run_state run_state;
+	uint16_t num_dir_queues; /* total num of evdev dir queues requested */
+	uint16_t num_dir_credits;
+	uint16_t num_ldb_credits;
+	uint16_t num_queues; /* total queues */
+	uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
+	uint16_t num_ports; /* total num of evdev ports requested */
+	uint16_t num_ldb_ports; /* total num of ldb ports requested */
+	uint16_t num_dir_ports; /* total num of dir ports requested */
+	bool is_vdev;
+	bool umwait_allowed;
+	bool global_dequeue_wait; /* Not using per dequeue wait if true */
+	bool defer_sched;
+	unsigned int num_atm_inflights_per_queue;
+	enum dlb_cq_poll_modes poll_mode;
+	uint8_t revision;
+	bool configured;
+};
+
+/* End Eventdev related defines and structs */
+
+/* externs */
+
+extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
+
+/* Forwards for non-inlined functions */
+
+void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
+
+int dlb_xstats_init(struct dlb_eventdev *dlb);
+
+void dlb_xstats_uninit(struct dlb_eventdev *dlb);
+
+int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
+			    enum rte_event_dev_xstats_mode mode,
+			    uint8_t queue_port_id, const unsigned int ids[],
+			    uint64_t values[], unsigned int n);
+
+int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
+				  enum rte_event_dev_xstats_mode mode,
+				  uint8_t queue_port_id,
+				  struct rte_event_dev_xstats_name *xstat_names,
+				  unsigned int *ids, unsigned int size);
+
+uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
+					 const char *name, unsigned int *id);
+
+int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
+			      enum rte_event_dev_xstats_mode mode,
+			      int16_t queue_port_id,
+			      const uint32_t ids[],
+			      uint32_t nb_ids);
+
+int test_dlb_eventdev(void);
+
+int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			       const char *name,
+			       struct dlb_devargs *dlb_args);
+
+int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+				 const char *name);
+
+uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *queue);
+
+int dlb_parse_params(const char *params,
+		     const char *name,
+		     struct dlb_devargs *dlb_args);
+
+#endif	/* _DLB_PRIV_H_ */
Signature · 1 line
2.6.4
#164 lore raw

PATCH v13 4/23 event/dlb: add definitions shared with LKM or shared code

In reply to Timothy McDaniel

Add headers containing structs and constants shared between the PMD and the shared code. The term shared code refers to the code that implements the hardware interface. The shared code is introduced in the probe patch, and then is extended as additional eventdev PMD entry points are added to the patchset. In the case of the bifurcated PMD (to be introduced in the future), the shared code is contained in the Linux kernel module itself.

---
 drivers/event/dlb/dlb_user.h | 814 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 814 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_user.h
drivers/event/dlb/dlb_user.h new file +814 -0
new file mode 100644
index 0000000..2d9582b
@@ -0,0 +1,814 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_USER_H
+#define __DLB_USER_H
+
+#include <linux/types.h>
+
+#define DLB_MAX_NAME_LEN 64
+
+enum dlb_error {
+	DLB_ST_SUCCESS = 0,
+	DLB_ST_NAME_EXISTS,
+	DLB_ST_DOMAIN_UNAVAILABLE,
+	DLB_ST_LDB_PORTS_UNAVAILABLE,
+	DLB_ST_DIR_PORTS_UNAVAILABLE,
+	DLB_ST_LDB_QUEUES_UNAVAILABLE,
+	DLB_ST_LDB_CREDITS_UNAVAILABLE,
+	DLB_ST_DIR_CREDITS_UNAVAILABLE,
+	DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE,
+	DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE,
+	DLB_ST_INVALID_DOMAIN_ID,
+	DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION,
+	DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE,
+	DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_POOL_ID,
+	DLB_ST_INVALID_DIR_CREDIT_POOL_ID,
+	DLB_ST_INVALID_POP_COUNT_VIRT_ADDR,
+	DLB_ST_INVALID_LDB_QUEUE_ID,
+	DLB_ST_INVALID_CQ_DEPTH,
+	DLB_ST_INVALID_CQ_VIRT_ADDR,
+	DLB_ST_INVALID_PORT_ID,
+	DLB_ST_INVALID_QID,
+	DLB_ST_INVALID_PRIORITY,
+	DLB_ST_NO_QID_SLOTS_AVAILABLE,
+	DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE,
+	DLB_ST_INVALID_DIR_QUEUE_ID,
+	DLB_ST_DIR_QUEUES_UNAVAILABLE,
+	DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_LDB_CREDIT_QUANTUM,
+	DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK,
+	DLB_ST_INVALID_DIR_CREDIT_QUANTUM,
+	DLB_ST_DOMAIN_NOT_CONFIGURED,
+	DLB_ST_PID_ALREADY_ATTACHED,
+	DLB_ST_PID_NOT_ATTACHED,
+	DLB_ST_INTERNAL_ERROR,
+	DLB_ST_DOMAIN_IN_USE,
+	DLB_ST_IOMMU_MAPPING_ERROR,
+	DLB_ST_FAIL_TO_PIN_MEMORY_PAGE,
+	DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES,
+	DLB_ST_UNABLE_TO_PIN_CQ_PAGES,
+	DLB_ST_DISCONTIGUOUS_CQ_MEMORY,
+	DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY,
+	DLB_ST_DOMAIN_STARTED,
+	DLB_ST_LARGE_POOL_NOT_SPECIFIED,
+	DLB_ST_SMALL_POOL_NOT_SPECIFIED,
+	DLB_ST_NEITHER_POOL_SPECIFIED,
+	DLB_ST_DOMAIN_NOT_STARTED,
+	DLB_ST_INVALID_MEASUREMENT_DURATION,
+	DLB_ST_INVALID_PERF_METRIC_GROUP_ID,
+	DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES,
+	DLB_ST_DOMAIN_RESET_FAILED,
+	DLB_ST_MBOX_ERROR,
+	DLB_ST_INVALID_HIST_LIST_DEPTH,
+	DLB_ST_NO_MEMORY,
+};
+
+static const char dlb_error_strings[][128] = {
+	"DLB_ST_SUCCESS",
+	"DLB_ST_NAME_EXISTS",
+	"DLB_ST_DOMAIN_UNAVAILABLE",
+	"DLB_ST_LDB_PORTS_UNAVAILABLE",
+	"DLB_ST_DIR_PORTS_UNAVAILABLE",
+	"DLB_ST_LDB_QUEUES_UNAVAILABLE",
+	"DLB_ST_LDB_CREDITS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDITS_UNAVAILABLE",
+	"DLB_ST_LDB_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_DIR_CREDIT_POOLS_UNAVAILABLE",
+	"DLB_ST_SEQUENCE_NUMBERS_UNAVAILABLE",
+	"DLB_ST_INVALID_DOMAIN_ID",
+	"DLB_ST_INVALID_QID_INFLIGHT_ALLOCATION",
+	"DLB_ST_ATOMIC_INFLIGHTS_UNAVAILABLE",
+	"DLB_ST_HIST_LIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_DIR_CREDIT_POOL_ID",
+	"DLB_ST_INVALID_POP_COUNT_VIRT_ADDR",
+	"DLB_ST_INVALID_LDB_QUEUE_ID",
+	"DLB_ST_INVALID_CQ_DEPTH",
+	"DLB_ST_INVALID_CQ_VIRT_ADDR",
+	"DLB_ST_INVALID_PORT_ID",
+	"DLB_ST_INVALID_QID",
+	"DLB_ST_INVALID_PRIORITY",
+	"DLB_ST_NO_QID_SLOTS_AVAILABLE",
+	"DLB_ST_QED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_DQED_FREELIST_ENTRIES_UNAVAILABLE",
+	"DLB_ST_INVALID_DIR_QUEUE_ID",
+	"DLB_ST_DIR_QUEUES_UNAVAILABLE",
+	"DLB_ST_INVALID_LDB_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_LDB_CREDIT_QUANTUM",
+	"DLB_ST_INVALID_DIR_CREDIT_LOW_WATERMARK",
+	"DLB_ST_INVALID_DIR_CREDIT_QUANTUM",
+	"DLB_ST_DOMAIN_NOT_CONFIGURED",
+	"DLB_ST_PID_ALREADY_ATTACHED",
+	"DLB_ST_PID_NOT_ATTACHED",
+	"DLB_ST_INTERNAL_ERROR",
+	"DLB_ST_DOMAIN_IN_USE",
+	"DLB_ST_IOMMU_MAPPING_ERROR",
+	"DLB_ST_FAIL_TO_PIN_MEMORY_PAGE",
+	"DLB_ST_UNABLE_TO_PIN_POPCOUNT_PAGES",
+	"DLB_ST_UNABLE_TO_PIN_CQ_PAGES",
+	"DLB_ST_DISCONTIGUOUS_CQ_MEMORY",
+	"DLB_ST_DISCONTIGUOUS_POP_COUNT_MEMORY",
+	"DLB_ST_DOMAIN_STARTED",
+	"DLB_ST_LARGE_POOL_NOT_SPECIFIED",
+	"DLB_ST_SMALL_POOL_NOT_SPECIFIED",
+	"DLB_ST_NEITHER_POOL_SPECIFIED",
+	"DLB_ST_DOMAIN_NOT_STARTED",
+	"DLB_ST_INVALID_MEASUREMENT_DURATION",
+	"DLB_ST_INVALID_PERF_METRIC_GROUP_ID",
+	"DLB_ST_LDB_PORT_REQUIRED_FOR_LDB_QUEUES",
+	"DLB_ST_DOMAIN_RESET_FAILED",
+	"DLB_ST_MBOX_ERROR",
+	"DLB_ST_INVALID_HIST_LIST_DEPTH",
+	"DLB_ST_NO_MEMORY",
+};
+
+struct dlb_cmd_response {
+	__u32 status; /* Interpret using enum dlb_error */
+	__u32 id;
+};
+
+/******************************/
+/* 'dlb' commands	      */
+/******************************/
+
+#define DLB_DEVICE_VERSION(x) (((x) >> 8) & 0xFF)
+#define DLB_DEVICE_REVISION(x) ((x) & 0xFF)
+
+enum dlb_revisions {
+	DLB_REV_A0 = 0,
+	DLB_REV_A1 = 1,
+	DLB_REV_A2 = 2,
+	DLB_REV_A3 = 3,
+	DLB_REV_B0 = 4,
+};
+
+/*
+ * DLB_CMD_CREATE_SCHED_DOMAIN: Create a DLB scheduling domain and reserve the
+ *	resources (queues, ports, etc.) that it contains.
+ *
+ * Input parameters:
+ * - num_ldb_queues: Number of load-balanced queues.
+ * - num_ldb_ports: Number of load-balanced ports.
+ * - num_dir_ports: Number of directed ports. A directed port has one directed
+ *	queue, so no num_dir_queues argument is necessary.
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for the domain. This storage is divided among the domain's
+ *	load-balanced queues that are configured for atomic scheduling.
+ * - num_hist_list_entries: Amount of history list storage. This is divided
+ *	among the domain's CQs.
+ * - num_ldb_credits: Amount of load-balanced QE storage (QED). QEs occupy this
+ *	space until they are scheduled to a load-balanced CQ. One credit
+ *	represents the storage for one QE.
+ * - num_dir_credits: Amount of directed QE storage (DQED). QEs occupy this
+ *	space until they are scheduled to a directed CQ. One credit represents
+ *	the storage for one QE.
+ * - num_ldb_credit_pools: Number of pools into which the load-balanced credits
+ *	are placed.
+ * - num_dir_credit_pools: Number of pools into which the directed credits are
+ *	placed.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: domain ID.
+ */
+struct dlb_create_sched_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+};
+
+/*
+ * DLB_CMD_GET_NUM_RESOURCES: Return the number of available resources
+ *	(queues, ports, etc.) that this device owns.
+ *
+ * Output parameters:
+ * - num_domains: Number of available scheduling domains.
+ * - num_ldb_queues: Number of available load-balanced queues.
+ * - num_ldb_ports: Number of available load-balanced ports.
+ * - num_dir_ports: Number of available directed ports. There is one directed
+ *	queue for every directed port.
+ * - num_atomic_inflights: Amount of available temporary atomic QE storage.
+ * - max_contiguous_atomic_inflights: When a domain is created, the temporary
+ *	atomic QE storage is allocated in a contiguous chunk. This return value
+ *	is the longest available contiguous range of atomic QE storage.
+ * - num_hist_list_entries: Amount of history list storage.
+ * - max_contiguous_hist_list_entries: History list storage is allocated in
+ *	a contiguous chunk, and this return value is the longest available
+ *	contiguous range of history list entries.
+ * - num_ldb_credits: Amount of available load-balanced QE storage.
+ * - max_contiguous_ldb_credits: QED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of load-balanced credit storage.
+ * - num_dir_credits: Amount of available directed QE storage.
+ * - max_contiguous_dir_credits: DQED storage is allocated in a contiguous
+ *	chunk, and this return value is the longest available contiguous range
+ *	of directed credit storage.
+ * - num_ldb_credit_pools: Number of available load-balanced credit pools.
+ * - num_dir_credit_pools: Number of available directed credit pools.
+ * - padding0: Reserved for future use.
+ */
+struct dlb_get_num_resources_args {
+	/* Output parameters */
+	__u32 num_sched_domains;
+	__u32 num_ldb_queues;
+	__u32 num_ldb_ports;
+	__u32 num_dir_ports;
+	__u32 num_atomic_inflights;
+	__u32 max_contiguous_atomic_inflights;
+	__u32 num_hist_list_entries;
+	__u32 max_contiguous_hist_list_entries;
+	__u32 num_ldb_credits;
+	__u32 max_contiguous_ldb_credits;
+	__u32 num_dir_credits;
+	__u32 max_contiguous_dir_credits;
+	__u32 num_ldb_credit_pools;
+	__u32 num_dir_credit_pools;
+	__u32 padding0;
+};
+
+/*
+ * DLB_CMD_SET_SN_ALLOCATION: Configure a sequence number group
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - num: Number of sequence numbers per queue.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_set_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 num;
+};
+
+/*
+ * DLB_CMD_GET_SN_ALLOCATION: Get a sequence number group's configuration
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of sequence numbers per queue.
+ */
+struct dlb_get_sn_allocation_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+enum dlb_cq_poll_modes {
+	DLB_CQ_POLL_MODE_STD,
+	DLB_CQ_POLL_MODE_SPARSE,
+
+	/* NUM_DLB_CQ_POLL_MODE must be last */
+	NUM_DLB_CQ_POLL_MODE,
+};
+
+/*
+ * DLB_CMD_QUERY_CQ_POLL_MODE: Query the CQ poll mode the kernel driver is using
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: CQ poll mode (see enum dlb_cq_poll_modes).
+ */
+struct dlb_query_cq_poll_mode_args {
+	/* Output parameters */
+	__u64 response;
+};
+
+/*
+ * DLB_CMD_GET_SN_OCCUPANCY: Get a sequence number group's occupancy
+ *
+ * Each sequence number group has one or more slots, depending on its
+ * configuration. I.e.:
+ * - If configured for 1024 sequence numbers per queue, the group has 1 slot
+ * - If configured for 512 sequence numbers per queue, the group has 2 slots
+ *   ...
+ * - If configured for 32 sequence numbers per queue, the group has 32 slots
+ *
+ * This ioctl returns the group's number of in-use slots. If its occupancy is
+ * 0, the group's sequence number allocation can be reconfigured.
+ *
+ * Input parameters:
+ * - group: Sequence number group ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Specified group's number of used slots.
+ */
+struct dlb_get_sn_occupancy_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 group;
+	__u32 padding0;
+};
+
+/*********************************/
+/* 'scheduling domain' commands  */
+/*********************************/
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_POOL: Configure a load-balanced credit pool.
+ * Input parameters:
+ * - num_ldb_credits: Number of load-balanced credits (QED space) for this
+ *	pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: pool ID.
+ */
+struct dlb_create_ldb_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_ldb_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_POOL: Configure a directed credit pool.
+ * Input parameters:
+ * - num_dir_credits: Number of directed credits (DQED space) for this pool.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Pool ID.
+ */
+struct dlb_create_dir_pool_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_dir_credits;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_QUEUE: Configure a load-balanced queue.
+ * Input parameters:
+ * - num_atomic_inflights: This specifies the amount of temporary atomic QE
+ *	storage for this queue. If zero, the queue will not support atomic
+ *	scheduling.
+ * - num_sequence_numbers: This specifies the number of sequence numbers used
+ *	by this queue. If zero, the queue will not support ordered scheduling.
+ *	If non-zero, the queue will not support unordered scheduling.
+ * - num_qid_inflights: The maximum number of QEs that can be inflight
+ *	(scheduled to a CQ but not completed) at any time. If
+ *	num_sequence_numbers is non-zero, num_qid_inflights must be set equal
+ *	to num_sequence_numbers.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_ldb_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 num_sequence_numbers;
+	__u32 num_qid_inflights;
+	__u32 num_atomic_inflights;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_QUEUE: Configure a directed queue.
+ * Input parameters:
+ * - port_id: Port ID. If the corresponding directed port is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the queue is being created before the port.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Queue ID.
+ */
+struct dlb_create_dir_queue_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__s32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_LDB_PORT: Configure a load-balanced port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ *
+ *	If this port's scheduling domain does not have any directed queues,
+ *	this argument is ignored and the port is given no directed credits.
+ * - padding0: Reserved for future use.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - cq_history_list_size: Number of history list entries. This must be greater
+ *	than or equal to cq_depth.
+ * - padding1: Reserved for future use.
+ * - padding2: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: port ID.
+ */
+struct dlb_create_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 padding0;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__u16 cq_history_list_size;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_CREATE_DIR_PORT: Configure a directed port.
+ * Input parameters:
+ * - ldb_credit_pool_id: Load-balanced credit pool this port will belong to.
+ * - dir_credit_pool_id: Directed credit pool this port will belong to.
+ * - ldb_credit_high_watermark: Number of load-balanced credits from the pool
+ *	that this port will own.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_high_watermark: Number of directed credits from the pool that
+ *	this port will own.
+ * - ldb_credit_low_watermark: Load-balanced credit low watermark. When the
+ *	port's credits reach this watermark, they become eligible to be
+ *	refilled by the DLB as credits until the high watermark
+ *	(num_ldb_credits) is reached.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_low_watermark: Directed credit low watermark. When the port's
+ *	credits reach this watermark, they become eligible to be refilled by
+ *	the DLB as credits until the high watermark (num_dir_credits) is
+ *	reached.
+ * - ldb_credit_quantum: Number of load-balanced credits for the DLB to refill
+ *	per refill operation.
+ *
+ *	If this port's scheduling domain does not have any load-balanced queues,
+ *	this argument is ignored and the port is given no load-balanced
+ *	credits.
+ * - dir_credit_quantum: Number of directed credits for the DLB to refill per
+ *	refill operation.
+ * - cq_depth: Depth of the port's CQ. Must be a power-of-two between 8 and
+ *	1024, inclusive.
+ * - cq_depth_threshold: CQ depth interrupt threshold. A value of N means that
+ *	the CQ interrupt won't fire until there are N or more outstanding CQ
+ *	tokens.
+ * - qid: Queue ID. If the corresponding directed queue is already created,
+ *	specify its ID here. Else this argument must be 0xFFFFFFFF to indicate
+ *	that the port is being created before the queue.
+ * - padding1: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: Port ID.
+ */
+struct dlb_create_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 ldb_credit_pool_id;
+	__u32 dir_credit_pool_id;
+	__u16 ldb_credit_high_watermark;
+	__u16 ldb_credit_low_watermark;
+	__u16 ldb_credit_quantum;
+	__u16 dir_credit_high_watermark;
+	__u16 dir_credit_low_watermark;
+	__u16 dir_credit_quantum;
+	__u16 cq_depth;
+	__u16 cq_depth_threshold;
+	__s32 queue_id;
+	__u32 padding1;
+};
+
+/*
+ * DLB_DOMAIN_CMD_START_DOMAIN: Mark the end of the domain configuration. This
+ *	must be called before passing QEs into the device, and no configuration
+ *	ioctls can be issued once the domain has started. Sending QEs into the
+ *	device before calling this ioctl will result in undefined behavior.
+ * Input parameters:
+ * - (None)
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_start_domain_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+};
+
+/*
+ * DLB_DOMAIN_CMD_MAP_QID: Map a load-balanced queue to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ * - priority: Queue->port service priority.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_map_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+	__u32 priority;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_UNMAP_QID: Unmap a load-balanced queue to a load-balanced
+ *	port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - qid: Load-balanced queue ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_unmap_qid_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 qid;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_LDB_PORT: Enable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_ENABLE_DIR_PORT: Enable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_enable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_LDB_PORT: Disable scheduling to a load-balanced port.
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_ldb_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_DISABLE_DIR_PORT: Disable scheduling to a directed port.
+ * Input parameters:
+ * - port_id: Directed port ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ */
+struct dlb_disable_dir_port_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_LDB_QUEUE_DEPTH: Get a load-balanced queue's depth.
+ * Input parameters:
+ * - queue_id: The load-balanced queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_ldb_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_GET_DIR_QUEUE_DEPTH: Get a directed queue's depth.
+ * Input parameters:
+ * - queue_id: The directed queue ID.
+ * - padding0: Reserved for future use.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: queue depth.
+ */
+struct dlb_get_dir_queue_depth_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 queue_id;
+	__u32 padding0;
+};
+
+/*
+ * DLB_DOMAIN_CMD_PENDING_PORT_UNMAPS: Get number of queue unmap operations in
+ *	progress for a load-balanced port.
+ *
+ *	Note: This is a snapshot; the number of unmap operations in progress
+ *	is subject to change at any time.
+ *
+ * Input parameters:
+ * - port_id: Load-balanced port ID.
+ *
+ * Output parameters:
+ * - response: pointer to a struct dlb_cmd_response.
+ *	response.status: Detailed error code. In certain cases, such as if the
+ *		response pointer is invalid, the driver won't set status.
+ *	response.id: number of unmaps in progress.
+ */
+struct dlb_pending_port_unmaps_args {
+	/* Output parameters */
+	__u64 response;
+	/* Input parameters */
+	__u32 port_id;
+	__u32 padding0;
+};
+
+/*
+ * Base addresses for memory mapping the consumer queue (CQ) and popcount (PC)
+ * memory space, and producer port (PP) MMIO space. The CQ, PC, and PP
+ * addresses are per-port. Every address is page-separated (e.g. LDB PP 0 is at
+ * 0x2100000 and LDB PP 1 is at 0x2101000).
+ */
+#define DLB_LDB_CQ_BASE 0x3000000
+#define DLB_LDB_CQ_MAX_SIZE 65536
+#define DLB_LDB_CQ_OFFS(id) (DLB_LDB_CQ_BASE + (id) * DLB_LDB_CQ_MAX_SIZE)
+
+#define DLB_DIR_CQ_BASE 0x3800000
+#define DLB_DIR_CQ_MAX_SIZE 65536
+#define DLB_DIR_CQ_OFFS(id) (DLB_DIR_CQ_BASE + (id) * DLB_DIR_CQ_MAX_SIZE)
+
+#define DLB_LDB_PC_BASE 0x2300000
+#define DLB_LDB_PC_MAX_SIZE 4096
+#define DLB_LDB_PC_OFFS(id) (DLB_LDB_PC_BASE + (id) * DLB_LDB_PC_MAX_SIZE)
+
+#define DLB_DIR_PC_BASE 0x2200000
+#define DLB_DIR_PC_MAX_SIZE 4096
+#define DLB_DIR_PC_OFFS(id) (DLB_DIR_PC_BASE + (id) * DLB_DIR_PC_MAX_SIZE)
+
+#define DLB_LDB_PP_BASE 0x2100000
+#define DLB_LDB_PP_MAX_SIZE 4096
+#define DLB_LDB_PP_OFFS(id) (DLB_LDB_PP_BASE + (id) * DLB_LDB_PP_MAX_SIZE)
+
+#define DLB_DIR_PP_BASE 0x2000000
+#define DLB_DIR_PP_MAX_SIZE 4096
+#define DLB_DIR_PP_OFFS(id) (DLB_DIR_PP_BASE + (id) * DLB_DIR_PP_MAX_SIZE)
+
+#endif /* __DLB_USER_H */
Signature · 1 line
2.6.4
#165 lore raw

PATCH v13 5/23 event/dlb: add inline functions

In reply to Timothy McDaniel

Add miscellaneous inline functions that may be called from multiple files. These functions include inline assembly of new x86 instructions, such as movdir64b, since they are not available as builtin functions in the minimum supported GCC version.

---
 drivers/event/dlb/dlb_inline_fns.h | 40 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
drivers/event/dlb/dlb_inline_fns.h new file +40 -0
new file mode 100644
index 0000000..1ecddb7
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include "rte_memcpy.h"
+#include "rte_io.h"
+
+/* Inline functions required in more than one source file. */
+
+static inline struct dlb_eventdev *
+dlb_pmd_priv(const struct rte_eventdev *eventdev)
+{
+	return eventdev->data->dev_private;
+}
+
+static inline void
+dlb_movntdq_single(void *dest, void *src)
+{
+	long long *_src  = (long long *)src;
+	__m128i src_data0 = (__m128i){_src[0], _src[1]};
+
+	_mm_stream_si128(dest, src_data0);
+}
+
+static inline void
+dlb_cldemote(void *addr)
+{
+	/* Load addr into RSI, then demote the cache line of the address
+	 * contained in that register.
+	 */
+	asm volatile(".byte 0x0f, 0x1c, 0x06" :: "S" (addr));
+}
+
+static inline void
+dlb_movdir64b(void *dest, void *src)
+{
+	asm volatile(".byte 0x66, 0x0f, 0x38, 0xf8, 0x02"
+		     :
+		     : "a" (dest), "d" (src));
+}
Signature · 1 line
2.6.4
#166 lore raw

PATCH v13 6/23 event/dlb: add eventdev probe

In reply to Timothy McDaniel

Add the eventdev portion of probe, and parse command line options, but do not initialize hardware.

Changes since v5 patch-set probe:

Primary and secondary probe-time init has been removed, and will be introduced in subsequent patches contained in this patch-set.

Hardware init has been moved to a subsequent patch in order to minimize the patch size.

Initialization of the flexible interface layer has been moved to a subsequent patch in order to minimize patch size.

---
 drivers/event/dlb/dlb.c                      |  327 ++++
 drivers/event/dlb/dlb_priv.h                 |    2 +
 drivers/event/dlb/meson.build                |    5 +-
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 +++++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 ++
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++++++++
 drivers/event/dlb/pf/dlb_main.c              |  568 ++++++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  147 ++
 13 files changed, 5586 insertions(+), 1 deletion(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
drivers/event/dlb/dlb.c +327 -0
index e03aa21..1659f93 100644
@@ -2,6 +2,333 @@
  * Copyright(c) 2016-2020 Intel Corporation
  */
 
+#include <assert.h>
+#include <errno.h>
+#include <nmmintrin.h>
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cycles.h>
+#include <rte_debug.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_io.h>
+#include <rte_kvargs.h>
 #include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_mbuf.h>
+#include <rte_prefetch.h>
+#include <rte_ring.h>
+#include <rte_string_fns.h>
+
+#include <rte_eventdev.h>
+#include <rte_eventdev_pmd.h>
+
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+/*
+ * Resources exposed to eventdev.
+ */
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
+#error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
+#endif
+
+struct process_local_port_data
+dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
+
+/* Wrapper for string to int conversion. Substituted for atoi(...), which is
+ * unsafe.
+ */
+#define DLB_BASE_10 10
+
+static int
+dlb_string_to_int(int *result, const char *str)
+{
+	long ret;
+	char *endstr;
+
+	if (str == NULL || result == NULL)
+		return -EINVAL;
+
+	errno = 0;
+	ret = strtol(str, &endstr, DLB_BASE_10);
+	if (errno)
+		return -errno;
+
+	/* long int and int may be different width for some architectures */
+	if (ret < INT_MIN || ret > INT_MAX || endstr == str)
+		return -EINVAL;
+
+	*result = ret;
+	return 0;
+}
+
+static int
+set_numa_node(const char *key __rte_unused, const char *value, void *opaque)
+{
+	int *socket_id = opaque;
+	int ret;
+
+	ret = dlb_string_to_int(socket_id, value);
+	if (ret < 0)
+		return ret;
+
+	if (*socket_id > RTE_MAX_NUMA_NODES)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+set_max_num_events(const char *key __rte_unused,
+		   const char *value,
+		   void *opaque)
+{
+	int *max_num_events = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(max_num_events, value);
+	if (ret < 0)
+		return ret;
+
+	if (*max_num_events < 0 || *max_num_events > DLB_MAX_NUM_LDB_CREDITS) {
+		DLB_LOG_ERR("dlb: max_num_events must be between 0 and %d\n",
+			    DLB_MAX_NUM_LDB_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_num_dir_credits(const char *key __rte_unused,
+		    const char *value,
+		    void *opaque)
+{
+	int *num_dir_credits = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_dir_credits, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_dir_credits < 0 ||
+	    *num_dir_credits > DLB_MAX_NUM_DIR_CREDITS) {
+		DLB_LOG_ERR("dlb: num_dir_credits must be between 0 and %d\n",
+			    DLB_MAX_NUM_DIR_CREDITS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+set_dev_id(const char *key __rte_unused,
+	   const char *value,
+	   void *opaque)
+{
+	int *dev_id = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(dev_id, value);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int
+set_defer_sched(const char *key __rte_unused,
+		const char *value,
+		void *opaque)
+{
+	int *defer_sched = opaque;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	if (strncmp(value, "on", 2) != 0) {
+		DLB_LOG_ERR("Invalid defer_sched argument \"%s\" (expected \"on\")\n",
+			    value);
+		return -EINVAL;
+	}
+
+	*defer_sched = 1;
+
+	return 0;
+}
+
+static int
+set_num_atm_inflights(const char *key __rte_unused,
+		      const char *value,
+		      void *opaque)
+{
+	int *num_atm_inflights = opaque;
+	int ret;
+
+	if (value == NULL || opaque == NULL) {
+		DLB_LOG_ERR("NULL pointer\n");
+		return -EINVAL;
+	}
+
+	ret = dlb_string_to_int(num_atm_inflights, value);
+	if (ret < 0)
+		return ret;
+
+	if (*num_atm_inflights < 0 ||
+	    *num_atm_inflights > DLB_MAX_NUM_ATM_INFLIGHTS) {
+		DLB_LOG_ERR("dlb: atm_inflights must be between 0 and %d\n",
+			    DLB_MAX_NUM_ATM_INFLIGHTS);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void
+dlb_entry_points_init(struct rte_eventdev *dev)
+{
+	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+	};
+
+	/* Expose PMD's eventdev interface */
+	dev->dev_ops = &dlb_eventdev_entry_ops;
+}
+
+int
+dlb_primary_eventdev_probe(struct rte_eventdev *dev,
+			   const char *name,
+			   struct dlb_devargs *dlb_args)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+	RTE_SET_USED(dlb_args);
+
+	return 0;
+}
+
+int
+dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
+			     const char *name)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(name);
+
+	return 0;
+}
+
+int
+dlb_parse_params(const char *params,
+		 const char *name,
+		 struct dlb_devargs *dlb_args)
+{
+	int ret = 0;
+	static const char * const args[] = { NUMA_NODE_ARG,
+					     DLB_MAX_NUM_EVENTS,
+					     DLB_NUM_DIR_CREDITS,
+					     DEV_ID_ARG,
+					     DLB_DEFER_SCHED_ARG,
+					     DLB_NUM_ATM_INFLIGHTS_ARG,
+					     NULL };
+
+	if (params && params[0] != '\0') {
+		struct rte_kvargs *kvlist = rte_kvargs_parse(params, args);
+
+		if (kvlist == NULL) {
+			DLB_LOG_INFO("Ignoring unsupported parameters when creating device '%s'\n",
+				     name);
+		} else {
+			int ret = rte_kvargs_process(kvlist, NUMA_NODE_ARG,
+						     set_numa_node,
+						     &dlb_args->socket_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing numa node parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_MAX_NUM_EVENTS,
+						 set_max_num_events,
+						 &dlb_args->max_num_events);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing max_num_events parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+					DLB_NUM_DIR_CREDITS,
+					set_num_dir_credits,
+					&dlb_args->num_dir_credits_override);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing num_dir_credits parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DEV_ID_ARG,
+						 set_dev_id,
+						 &dlb_args->dev_id);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing dev_id parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist, DLB_DEFER_SCHED_ARG,
+						 set_defer_sched,
+						 &dlb_args->defer_sched);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing defer_sched parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
+
+			ret = rte_kvargs_process(kvlist,
+						 DLB_NUM_ATM_INFLIGHTS_ARG,
+						 set_num_atm_inflights,
+						 &dlb_args->num_atm_inflights);
+			if (ret != 0) {
+				DLB_LOG_ERR("%s: Error parsing atm_inflights parameter",
+					    name);
+				rte_kvargs_free(kvlist);
+				return ret;
+			}
 
+			rte_kvargs_free(kvlist);
+		}
+	}
+	return ret;
+}
 RTE_LOG_REGISTER(eventdev_dlb_log_level, pmd.event.dlb, NOTICE);
drivers/event/dlb/dlb_priv.h +2 -0
index f9ff0a5..adb1f7a 100644
@@ -505,4 +505,6 @@ int dlb_parse_params(const char *params,
 		     const char *name,
 		     struct dlb_devargs *dlb_args);
 
+void dlb_entry_points_init(struct rte_eventdev *dev);
+
 #endif	/* _DLB_PRIV_H_ */
drivers/event/dlb/meson.build +4 -1
index 1e7d5ad..b4bdc8b 100644
@@ -7,6 +7,9 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
         subdir_done()
 endif
 
-sources = files('dlb.c')
+sources = files('dlb.c',
+		'pf/dlb_main.c',
+		'pf/dlb_pf.c'
+)
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/pf/base/dlb_hw_types.h new file +334 -0

334 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep.h new file +310 -0

310 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep_bitmap.h new file +441 -0

441 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep_list.h new file +131 -0

131 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_osdep_types.h new file +31 -0
new file mode 100644
index 0000000..2e9d7d8
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef __DLB_OSDEP_TYPES_H
+#define __DLB_OSDEP_TYPES_H
+
+#include <linux/types.h>
+
+#include <inttypes.h>
+#include <ctype.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+/* Types for user mode PF PMD */
+typedef uint8_t         u8;
+typedef int8_t          s8;
+typedef uint16_t        u16;
+typedef int16_t         s16;
+typedef uint32_t        u32;
+typedef int32_t         s32;
+typedef uint64_t        u64;
+
+#define __iomem
+
+/* END types for user mode PF PMD */
+
+#endif /* __DLB_OSDEP_TYPES_H */
drivers/event/dlb/pf/base/dlb_regs.h new file +2368 -0

1,404 lines not on this page · raw message

drivers/event/dlb/pf/base/dlb_resource.h new file +876 -0

876 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.c new file +568 -0

568 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.h new file +47 -0

47 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c new file +147 -0

147 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.6.4

Part of the diff is not shown here. View the raw message.

#167 lore raw

PATCH v13 7/23 event/dlb: add flexible interface

In reply to Timothy McDaniel

This commit introduces the flexible interface. This interface allows the core code to operate in PF mode (direct hardware access) or bifurcated mode (hardware configured via kernel driver). This driver currently only supports PF modei, but bifurcated mode will be added in a future patch-set. Note that the flexible interface is not used for data path operations, and thus there are no performance concerns related to the use of function pointers.

---
 drivers/event/dlb/dlb.c       |  1 +
 drivers/event/dlb/dlb_iface.c | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.h | 27 +++++++++++++++++++++++++++
 drivers/event/dlb/meson.build |  1 +
 drivers/event/dlb/pf/dlb_pf.c |  1 +
 5 files changed, 57 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
drivers/event/dlb/dlb.c +1 -0
index 1659f93..8008a50 100644
@@ -33,6 +33,7 @@
 #include <rte_eventdev_pmd.h>
 
 #include "dlb_priv.h"
+#include "dlb_iface.h"
 #include "dlb_inline_fns.h"
 
 /*
drivers/event/dlb/dlb_iface.c new file +27 -0
new file mode 100644
index 0000000..dd72120
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#include <stdint.h>
+
+#include "dlb_priv.h"
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD),  these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD),  these will be implemented locally in user mode.
+ */
+
+void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+				    uint8_t *revision);
+
+int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+				  enum dlb_cq_poll_modes *mode);
+
drivers/event/dlb/dlb_iface.h new file +27 -0
new file mode 100644
index 0000000..416d1b3
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2016-2020 Intel Corporation
+ */
+
+#ifndef _DLB_IFACE_H
+#define _DLB_IFACE_H
+
+/* DLB PMD Internal interface function pointers.
+ * If VDEV (bifurcated PMD), these will resolve to functions that issue ioctls
+ * serviced by DLB kernel module.
+ * If PCI (PF PMD), these will be implemented locally in user mode.
+ */
+
+extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
+
+extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
+
+extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
+					   uint8_t *revision);
+
+extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
+				   struct dlb_get_num_resources_args *rsrcs);
+
+extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
+					 enum dlb_cq_poll_modes *mode);
+
+#endif /* _DLB_IFACE_H */
drivers/event/dlb/meson.build +1 -0
index b4bdc8b..8707d3d 100644
@@ -8,6 +8,7 @@ if not is_linux or not dpdk_conf.has('RTE_ARCH_X86_64')
 endif
 
 sources = files('dlb.c',
+		'dlb_iface.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c'
 )
drivers/event/dlb/pf/dlb_pf.c +1 -0
index 3f836f3..05fd76c 100644
@@ -27,6 +27,7 @@
 #include <rte_string_fns.h>
 
 #include "../dlb_priv.h"
+#include "../dlb_iface.h"
 #include "../dlb_inline_fns.h"
 #include "dlb_main.h"
 #include "base/dlb_hw_types.h"
Signature · 1 line
2.6.4
#168 lore raw

PATCH v13 8/23 event/dlb: add probe-time hardware init

In reply to Timothy McDaniel

This commit adds probe-time low level hardware initialization. It also adds probe-time init for both primary and secondary DPDK processes.

---
 drivers/event/dlb/dlb.c                  | 158 +++++++++++++++-
 drivers/event/dlb/meson.build            |   3 +-
 drivers/event/dlb/pf/base/dlb_resource.c | 302 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_main.c          |  20 +-
 drivers/event/dlb/pf/dlb_pf.c            |  86 ++++++++-
 5 files changed, 561 insertions(+), 8 deletions(-)
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
drivers/event/dlb/dlb.c +153 -5
index 8008a50..57b2837 100644
@@ -42,10 +42,92 @@
 #if (RTE_EVENT_MAX_QUEUES_PER_DEV > UINT8_MAX)
 #error "RTE_EVENT_MAX_QUEUES_PER_DEV cannot fit in member max_event_queues"
 #endif
+static struct rte_event_dev_info evdev_dlb_default_info = {
+	.driver_name = "", /* probe will set */
+	.min_dequeue_timeout_ns = DLB_MIN_DEQUEUE_TIMEOUT_NS,
+	.max_dequeue_timeout_ns = DLB_MAX_DEQUEUE_TIMEOUT_NS,
+#if (RTE_EVENT_MAX_QUEUES_PER_DEV < DLB_MAX_NUM_LDB_QUEUES)
+	.max_event_queues = RTE_EVENT_MAX_QUEUES_PER_DEV,
+#else
+	.max_event_queues = DLB_MAX_NUM_LDB_QUEUES,
+#endif
+	.max_event_queue_flows = DLB_MAX_NUM_FLOWS,
+	.max_event_queue_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_priority_levels = DLB_QID_PRIORITIES,
+	.max_event_ports = DLB_MAX_NUM_LDB_PORTS,
+	.max_event_port_dequeue_depth = DLB_MAX_CQ_DEPTH,
+	.max_event_port_enqueue_depth = DLB_MAX_ENQUEUE_DEPTH,
+	.max_event_port_links = DLB_MAX_NUM_QIDS_PER_LDB_CQ,
+	.max_num_events = DLB_MAX_NUM_LDB_CREDITS,
+	.max_single_link_event_port_queue_pairs = DLB_MAX_NUM_DIR_PORTS,
+	.event_dev_cap = (RTE_EVENT_DEV_CAP_QUEUE_QOS |
+			  RTE_EVENT_DEV_CAP_EVENT_QOS |
+			  RTE_EVENT_DEV_CAP_BURST_MODE |
+			  RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED |
+			  RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE |
+			  RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES),
+};
 
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+static int
+dlb_hw_query_resources(struct dlb_eventdev *dlb)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_hw_resource_info *dlb_info = &handle->info;
+	int ret;
+
+	ret = dlb_iface_get_num_resources(handle,
+					  &dlb->hw_rsrc_query_results);
+	if (ret) {
+		DLB_LOG_ERR("get dlb num resources, err=%d\n", ret);
+		return ret;
+	}
+
+	/* Complete filling in device resource info returned to evdev app,
+	 * overriding any default values.
+	 * The capabilities (CAPs) were set at compile time.
+	 */
+
+	evdev_dlb_default_info.max_event_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	evdev_dlb_default_info.max_event_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	evdev_dlb_default_info.max_num_events =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	/* Save off values used when creating the scheduling domain. */
+
+	handle->info.num_sched_domains =
+		dlb->hw_rsrc_query_results.num_sched_domains;
+
+	handle->info.hw_rsrc_max.nb_events_limit =
+		dlb->hw_rsrc_query_results.max_contiguous_ldb_credits;
+
+	handle->info.hw_rsrc_max.num_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues +
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.num_ldb_queues =
+		dlb->hw_rsrc_query_results.num_ldb_queues;
+
+	handle->info.hw_rsrc_max.num_ldb_ports =
+		dlb->hw_rsrc_query_results.num_ldb_ports;
+
+	handle->info.hw_rsrc_max.num_dir_ports =
+		dlb->hw_rsrc_query_results.num_dir_ports;
+
+	handle->info.hw_rsrc_max.reorder_window_size =
+		dlb->hw_rsrc_query_results.num_hist_list_entries;
+
+	rte_memcpy(dlb_info, &handle->info.hw_rsrc_max, sizeof(*dlb_info));
+
+	return 0;
+}
+
 /* Wrapper for string to int conversion. Substituted for atoi(...), which is
  * unsafe.
  */
@@ -227,9 +309,54 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 			   const char *name,
 			   struct dlb_devargs *dlb_args)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
-	RTE_SET_USED(dlb_args);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	dlb->event_dev = dev; /* backlink */
+
+	evdev_dlb_default_info.driver_name = name;
+
+	dlb->max_num_events_override = dlb_args->max_num_events;
+	dlb->num_dir_credits_override = dlb_args->num_dir_credits_override;
+	dlb->defer_sched = dlb_args->defer_sched;
+	dlb->num_atm_inflights_per_queue = dlb_args->num_atm_inflights;
+
+	/* Open the interface.
+	 * For vdev mode, this means open the dlb kernel module.
+	 */
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_iface_get_device_version(&dlb->qm_instance, &dlb->revision);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the device version, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	err = dlb_iface_get_cq_poll_mode(&dlb->qm_instance, &dlb->poll_mode);
+	if (err < 0) {
+		DLB_LOG_ERR("dlb: failed to get the poll mode, err=%d\n", err);
+		return err;
+	}
+
+	rte_spinlock_init(&dlb->qm_instance.resource_lock);
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
@@ -238,8 +365,29 @@ int
 dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
 			     const char *name)
 {
-	RTE_SET_USED(dev);
-	RTE_SET_USED(name);
+	struct dlb_eventdev *dlb;
+	int err;
+
+	dlb = dev->data->dev_private;
+
+	evdev_dlb_default_info.driver_name = name;
+
+	err = dlb_iface_open(&dlb->qm_instance, name);
+	if (err < 0) {
+		DLB_LOG_ERR("could not open event hardware device, err=%d\n",
+			    err);
+		return err;
+	}
+
+	err = dlb_hw_query_resources(dlb);
+	if (err) {
+		DLB_LOG_ERR("get resources err=%d for %s\n", err, name);
+		return err;
+	}
+
+	dlb_iface_low_level_io_init(dlb);
+
+	dlb_entry_points_init(dev);
 
 	return 0;
 }
drivers/event/dlb/meson.build +2 -1
index 8707d3d..9777178 100644
@@ -10,7 +10,8 @@ endif
 sources = files('dlb.c',
 		'dlb_iface.c',
 		'pf/dlb_main.c',
-		'pf/dlb_pf.c'
+		'pf/dlb_pf.c',
+		'pf/base/dlb_resource.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
drivers/event/dlb/pf/base/dlb_resource.c new file +302 -0

302 lines not on this page · raw message

drivers/event/dlb/pf/dlb_main.c +19 -1
index c10c36c..2f4a828 100644
@@ -223,12 +223,18 @@ dlb_probe(struct rte_pci_device *pdev)
 	if (ret)
 		goto init_driver_state_fail;
 
+	ret = dlb_resource_init(&dlb_dev->hw);
+	if (ret)
+		goto resource_init_fail;
+
 	dlb_dev->revision = os_get_dev_revision(&dlb_dev->hw);
 
 	dlb_pf_init_hardware(dlb_dev);
 
 	return dlb_dev;
 
+resource_init_fail:
+	dlb_resource_free(&dlb_dev->hw);
 init_driver_state_fail:
 mask_ur_err_fail:
 dlb_reset_fail:
@@ -564,5 +570,17 @@ dlb_pf_init_driver_state(struct dlb_dev *dlb_dev)
 void
 dlb_pf_init_hardware(struct dlb_dev *dlb_dev)
 {
-	RTE_SET_USED(dlb_dev);
+	dlb_disable_dp_vasr_feature(&dlb_dev->hw);
+
+	dlb_enable_excess_tokens_alarm(&dlb_dev->hw);
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_enable_sparse_ldb_cq_mode(&dlb_dev->hw);
+		dlb_hw_enable_sparse_dir_cq_mode(&dlb_dev->hw);
+	}
+
+	if (dlb_dev->revision >= DLB_REV_B0) {
+		dlb_hw_disable_pf_to_vf_isr_pend_err(&dlb_dev->hw);
+		dlb_hw_disable_vf_to_pf_isr_pend_err(&dlb_dev->hw);
+	}
 }
drivers/event/dlb/pf/dlb_pf.c +85 -1

94 lines not on this page · raw message

Signature · 1 line
2.6.4
#169 lore raw

PATCH v13 9/23 event/dlb: add xstats

In reply to Timothy McDaniel

Add support for DLB xstats. Perform initialization and add standard xstats entry points

---
 drivers/event/dlb/dlb.c        |   23 +
 drivers/event/dlb/dlb_xstats.c | 1222 ++++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build  |    1 +
 3 files changed, 1246 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_xstats.c
drivers/event/dlb/dlb.c +23 -0
index 57b2837..62b9695 100644
@@ -71,6 +71,17 @@ static struct rte_event_dev_info evdev_dlb_default_info = {
 struct process_local_port_data
 dlb_port[DLB_MAX_NUM_PORTS][NUM_DLB_PORT_TYPES];
 
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	/* DUMMY FOR NOW So "xstats" patch compiles */
+	RTE_SET_USED(dlb);
+	RTE_SET_USED(queue);
+
+	return 0;
+}
+
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -298,6 +309,11 @@ void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
+		.dump             = dlb_eventdev_dump,
+		.xstats_get       = dlb_eventdev_xstats_get,
+		.xstats_get_names = dlb_eventdev_xstats_get_names,
+		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
+		.xstats_reset	    = dlb_eventdev_xstats_reset,
 	};
 
 	/* Expose PMD's eventdev interface */
@@ -352,6 +368,13 @@ dlb_primary_eventdev_probe(struct rte_eventdev *dev,
 		return err;
 	}
 
+	/* Complete xtstats runtime initialization */
+	err = dlb_xstats_init(dlb);
+	if (err) {
+		DLB_LOG_ERR("dlb: failed to init xstats, err=%d\n", err);
+		return err;
+	}
+
 	rte_spinlock_init(&dlb->qm_instance.resource_lock);
 
 	dlb_iface_low_level_io_init(dlb);
drivers/event/dlb/dlb_xstats.c new file +1222 -0

1,222 lines not on this page · raw message

drivers/event/dlb/meson.build +1 -0
index 9777178..552ff9d 100644
@@ -9,6 +9,7 @@ endif
 
 sources = files('dlb.c',
 		'dlb_iface.c',
+		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c'
Signature · 1 line
2.6.4
#170 lore raw

PATCH v13 10/23 event/dlb: add infos get and configure

In reply to Timothy McDaniel

Add support for configuring the DLB hardware.
In particular, this patch configures the DLB hardware's scheduling domain, such that it is provisioned with the requested number of ports and queues, provided sufficient resources are available. Individual queues and ports are configured later in port setup and eventdev start.

---
 doc/guides/eventdevs/dlb.rst             |   48 +
 drivers/event/dlb/dlb.c                  |  397 +++
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   11 +
 drivers/event/dlb/pf/base/dlb_resource.c | 4100 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |   88 +
 6 files changed, 4562 insertions(+), 93 deletions(-)
doc/guides/eventdevs/dlb.rst +48 -0
index 92341c0..2d7999b 100644
@@ -34,3 +34,51 @@ detailed understanding of the hardware, but these details are important when
 writing high-performance code. This section describes the places where the
 eventdev API and DLB misalign.
 
+Scheduling Domain Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are 32 scheduling domainis the DLB.
+When one is configured, it allocates load-balanced and
+directed queues, ports, credits, and other hardware resources. Some
+resource allocations are user-controlled -- the number of queues, for example
+-- and others, like credit pools (one directed and one load-balanced pool per
+scheduling domain), are not.
+
+The DLB is a closed system eventdev, and as such the ``nb_events_limit`` device
+setup argument and the per-port ``new_event_threshold`` argument apply as
+defined in the eventdev header file. The limit is applied to all enqueues,
+regardless of whether it will consume a directed or load-balanced credit.
+
+Reconfiguration
+~~~~~~~~~~~~~~~
+
+The Eventdev API allows one to reconfigure a device, its ports, and its queues
+by first stopping the device, calling the configuration function(s), then
+restarting the device. The DLB does not support configuring an individual queue
+or port without first reconfiguring the entire device, however, so there are
+certain reconfiguration sequences that are valid in the eventdev API but not
+supported by the PMD.
+
+Specifically, the PMD supports the following configuration sequence:
+1. Configure and start the device
+2. Stop the device
+3. (Optional) Reconfigure the device
+4. (Optional) If step 3 is run:
+
+   a. Setup queue(s). The reconfigured queue(s) lose their previous port links.
+   b. The reconfigured port(s) lose their previous queue links.
+
+5. (Optional, only if steps 4a and 4b are run) Link port(s) to queue(s)
+6. Restart the device. If the device is reconfigured in step 3 but one or more
+   of its ports or queues are not, the PMD will apply their previous
+   configuration (including port->queue links) at this time.
+
+The PMD does not support the following configuration sequences:
+1. Configure and start the device
+2. Stop the device
+3. Setup queue or setup port
+4. Start the device
+
+This sequence is not supported because the event device must be reconfigured
+before its ports or queues can be.
+
drivers/event/dlb/dlb.c +397 -0

415 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +11 -0
index dd72120..f3e82f2 100644
@@ -16,12 +16,23 @@ void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 				    uint8_t *revision);
 
 int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +11 -0
index 416d1b3..d576232 100644
@@ -15,12 +15,23 @@ extern void (*dlb_iface_low_level_io_init)(struct dlb_eventdev *dlb);
 
 extern int (*dlb_iface_open)(struct dlb_hw_dev *handle, const char *name);
 
+extern void (*dlb_iface_domain_close)(struct dlb_eventdev *dlb);
+
 extern int (*dlb_iface_get_device_version)(struct dlb_hw_dev *handle,
 					   uint8_t *revision);
 
 extern int (*dlb_iface_get_num_resources)(struct dlb_hw_dev *handle,
 				   struct dlb_get_num_resources_args *rsrcs);
 
+extern int (*dlb_iface_sched_domain_create)(struct dlb_hw_dev *handle,
+				     struct dlb_create_sched_domain_args *args);
+
+extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_pool_args *cfg);
+
+extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_pool_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +4007 -93

2,488 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +88 -0

88 lines not on this page · raw message

The rest of this diff is not shown. View the raw message.

Signature · 1 line
2.6.4

Part of the diff is not shown here. View the raw message.

#171 lore raw

PATCH v13 11/23 event/dlb: add queue and port default conf

In reply to Timothy McDaniel

Add support for getting the queue and port default configuration.

---
 drivers/event/dlb/dlb.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
drivers/event/dlb/dlb.c +29 -0
index c038794..e98a438 100644
@@ -630,6 +630,33 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
+				   uint8_t port_id,
+				   struct rte_event_port_conf *port_conf)
+{
+	RTE_SET_USED(port_id);
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	port_conf->new_event_threshold = dlb->new_event_limit;
+	port_conf->dequeue_depth = 32;
+	port_conf->enqueue_depth = DLB_MAX_ENQUEUE_DEPTH;
+	port_conf->event_port_cfg = 0;
+}
+
+static void
+dlb_eventdev_queue_default_conf_get(struct rte_eventdev *dev,
+				    uint8_t queue_id,
+				    struct rte_event_queue_conf *queue_conf)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(queue_id);
+	queue_conf->nb_atomic_flows = 1024;
+	queue_conf->nb_atomic_order_sequences = 32;
+	queue_conf->event_queue_cfg = 0;
+	queue_conf->priority = 0;
+}
+
 static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
@@ -706,6 +733,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
+		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#172 lore raw

PATCH v13 12/23 event/dlb: add queue setup

In reply to Timothy McDaniel

Load balanced (ldb) queues are setup here. Directed queues are not set up until link time, at which point we know the directed port ID. Directed queue setup will only fail if this queue is already setup or there are no directed queues left to configure.

---
 doc/guides/eventdevs/dlb.rst             |  35 +++
 drivers/event/dlb/dlb.c                  | 293 +++++++++++++++++++++++
 drivers/event/dlb/dlb_iface.c            |  12 +
 drivers/event/dlb/dlb_iface.h            |  12 +
 drivers/event/dlb/pf/base/dlb_resource.c | 386 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  81 +++++++
 6 files changed, 819 insertions(+)
doc/guides/eventdevs/dlb.rst +35 -0
index 2d7999b..d8e936a 100644
@@ -82,3 +82,38 @@ The PMD does not support the following configuration sequences:
 This sequence is not supported because the event device must be reconfigured
 before its ports or queues can be.
 
+Load-Balanced Queues
+~~~~~~~~~~~~~~~~~~~~
+
+A load-balanced queue can support atomic and ordered scheduling, or atomic and
+unordered scheduling, but not atomic and unordered and ordered scheduling. A
+queue's scheduling types are controlled by the event queue configuration.
+
+If the user sets the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag, the
+``nb_atomic_order_sequences`` determines the supported scheduling types.
+With non-zero ``nb_atomic_order_sequences``, the queue is configured for atomic
+and ordered scheduling. In this case, ``RTE_SCHED_TYPE_PARALLEL`` scheduling is
+supported by scheduling those events as ordered events.  Note that when the
+event is dequeued, its sched_type will be ``RTE_SCHED_TYPE_ORDERED``. Else if
+``nb_atomic_order_sequences`` is zero, the queue is configured for atomic and
+unordered scheduling. In this case, ``RTE_SCHED_TYPE_ORDERED`` is unsupported.
+
+If the ``RTE_EVENT_QUEUE_CFG_ALL_TYPES`` flag is not set, schedule_type
+dictates the queue's scheduling type.
+
+The ``nb_atomic_order_sequences`` queue configuration field sets the ordered
+queue's reorder buffer size.  DLB has 4 groups of ordered queues, where each
+group is configured to contain either 1 queue with 1024 reorder entries, 2
+queues with 512 reorder entries, and so on down to 32 queues with 32 entries.
+
+When a load-balanced queue is created, the PMD will configure a new sequence
+number group on-demand if num_sequence_numbers does not match a pre-existing
+group with available reorder buffer entries. If all sequence number groups are
+in use, no new group will be created and queue configuration will fail. (Note
+that when the PMD is used with a virtual DLB device, it cannot change the
+sequence number configuration.)
+
+The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
+the DLB does not limit the number of flows a queue can track. In the DLB, all
+load-balanced queues can use the full 16-bit flow ID range.
+
drivers/event/dlb/dlb.c +293 -0

305 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +12 -0
index f3e82f2..219f79e 100644
@@ -33,6 +33,18 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
+int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_get_sn_allocation_args *args);
+
+int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				   struct dlb_set_sn_allocation_args *args);
+
+int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
drivers/event/dlb/dlb_iface.h +12 -0
index d576232..af1416d 100644
@@ -32,7 +32,19 @@ extern int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
+extern int (*dlb_iface_get_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_allocation_args *args);
+
+extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
+				  struct dlb_set_sn_allocation_args *args);
+
+extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
+				  struct dlb_get_sn_occupancy_args *args);
+
 #endif /* _DLB_IFACE_H */
drivers/event/dlb/pf/base/dlb_resource.c +386 -0

389 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +81 -0

94 lines not on this page · raw message

Signature · 1 line
2.6.4
#173 lore raw

PATCH v13 13/23 event/dlb: add port setup

In reply to Timothy McDaniel

Configure the load balanced (ldb) or directed (dir) port. The consumer queue (CQ) and producer port (PP) are also set up here.

---
 doc/guides/eventdevs/dlb.rst             |   40 +
 drivers/event/dlb/dlb.c                  |  516 ++++++++++-
 drivers/event/dlb/dlb_iface.c            |   11 +
 drivers/event/dlb/dlb_iface.h            |   14 +
 drivers/event/dlb/pf/base/dlb_resource.c | 1436 +++++++++++++++++++++++++++++-
 drivers/event/dlb/pf/dlb_pf.c            |  210 +++++
 6 files changed, 2223 insertions(+), 4 deletions(-)
doc/guides/eventdevs/dlb.rst +40 -0
index d8e936a..f106a07 100644
@@ -117,3 +117,43 @@ The queue's ``nb_atomic_flows`` parameter is ignored by the DLB PMD, because
 the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
+Load-balanced and Directed Ports
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+DLB ports come in two flavors: load-balanced and directed. The eventdev API
+does not have the same concept, but it has a similar one: ports and queues that
+are singly-linked (i.e. linked to a single queue or port, respectively).
+
+The ``rte_event_dev_info_get()`` function reports the number of available
+event ports and queues (among other things). For the DLB PMD, max_event_ports
+and max_event_queues report the number of available load-balanced ports and
+queues, and max_single_link_event_port_queue_pairs reports the number of
+available directed ports and queues.
+
+When a scheduling domain is created in ``rte_event_dev_configure()``, the user
+specifies ``nb_event_ports`` and ``nb_single_link_event_port_queues``, which
+control the total number of ports (load-balanced and directed) and the number
+of directed ports. Hence, the number of requested load-balanced ports is
+``nb_event_ports - nb_single_link_event_ports``. The ``nb_event_queues`` field
+specifies the total number of queues (load-balanced and directed). The number
+of directed queues comes from ``nb_single_link_event_port_queues``, since
+directed ports and queues come in pairs.
+
+When a port is setup, the ``RTE_EVENT_PORT_CFG_SINGLE_LINK`` flag determines
+whether it should be configured as a directed (the flag is set) or a
+load-balanced (the flag is unset) port. Similarly, the
+``RTE_EVENT_QUEUE_CFG_SINGLE_LINK`` queue configuration flag controls
+whether it is a directed or load-balanced queue.
+
+Load-balanced ports can only be linked to load-balanced queues, and directed
+ports can only be linked to directed queues. Furthermore, directed ports can
+only be linked to a single directed queue (and vice versa), and that link
+cannot change after the eventdev is started.
+
+The eventdev API does not have a directed scheduling type. To support directed
+traffic, the dlb PMD detects when an event is being sent to a directed queue
+and overrides its scheduling type. Note that the originally selected scheduling
+type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
+will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
+port.
+
drivers/event/dlb/dlb.c +515 -1

546 lines not on this page · raw message

drivers/event/dlb/dlb_iface.c +11 -0
index 219f79e..fbbf9d7 100644
@@ -33,9 +33,20 @@ int (*dlb_iface_ldb_credit_pool_create)(struct dlb_hw_dev *handle,
 int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 					struct dlb_create_dir_pool_args *cfg);
 
+int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
 int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_ldb_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
+int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+				 struct dlb_create_dir_port_args *cfg,
+				 enum dlb_cq_poll_modes poll_mode);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +14 -0
index af1416d..d578185 100644
@@ -35,6 +35,20 @@ extern int (*dlb_iface_dir_credit_pool_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_dir_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_dir_queue_args *cfg);
+
+extern int (*dlb_iface_ldb_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_ldb_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
+					struct dlb_create_dir_port_args *cfg,
+					enum dlb_cq_poll_modes poll_mode);
+
+extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
+				  struct dlb_create_ldb_queue_args *cfg);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +1433 -3

1,457 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +210 -0

222 lines not on this page · raw message

Signature · 1 line
2.6.4
#174 lore raw

PATCH v13 14/23 event/dlb: add port link

In reply to Timothy McDaniel

Add port link entry point. Directed queues are identified and created at this stage. Their setup deferred until link-time, at which point we know the directed port ID. Directed queue setup will only fail if this queue is already setup or there are no directed queues left to configure.

---
 drivers/event/dlb/dlb.c                  | 306 +++++++++++++++
 drivers/event/dlb/dlb_iface.c            |   9 +
 drivers/event/dlb/dlb_iface.h            |   9 +
 drivers/event/dlb/pf/base/dlb_resource.c | 641 +++++++++++++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  69 ++++
 5 files changed, 1034 insertions(+)
drivers/event/dlb/dlb.c +306 -0
index 4d91ddd..2ad195d 100644
@@ -1532,6 +1532,311 @@ set_num_atm_inflights(const char *key __rte_unused,
 	return 0;
 }
 
+static int
+dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
+		       uint8_t queue_id,
+		       bool link_exists,
+		       int index)
+{
+	struct dlb_eventdev *dlb = ev_port->dlb;
+	struct dlb_eventdev_queue *ev_queue;
+	bool port_is_dir, queue_is_dir;
+
+	if (queue_id > dlb->num_queues) {
+		DLB_LOG_ERR("queue_id %d > num queues %d\n",
+			    queue_id, dlb->num_queues);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	ev_queue = &dlb->ev_queues[queue_id];
+
+	if (!ev_queue->setup_done &&
+	    ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("setup not done and not previously configured\n");
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	port_is_dir = ev_port->qm_port.is_directed;
+	queue_is_dir = ev_queue->qm_queue.is_directed;
+
+	if (port_is_dir != queue_is_dir) {
+		DLB_LOG_ERR("%s queue %u can't link to %s port %u\n",
+			    queue_is_dir ? "DIR" : "LDB", ev_queue->id,
+			    port_is_dir ? "DIR" : "LDB", ev_port->id);
+
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if there is space for the requested link */
+	if (!link_exists && index == -1) {
+		DLB_LOG_ERR("no space for new link\n");
+		rte_errno = -ENOSPC;
+		return -1;
+	}
+
+	/* Check if the directed port is already linked */
+	if (ev_port->qm_port.is_directed && ev_port->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR port %d to >1 queues\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	/* Check if the directed queue is already linked */
+	if (ev_queue->qm_queue.is_directed && ev_queue->num_links > 0 &&
+	    !link_exists) {
+		DLB_LOG_ERR("Can't link DIR queue %d to >1 ports\n",
+			    ev_queue->id);
+		rte_errno = -EINVAL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int16_t
+dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
+			   uint32_t qm_port_id,
+			   uint16_t qm_qid,
+			   uint8_t priority)
+{
+	struct dlb_map_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	/* Build message */
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+	cfg.priority = EV_TO_DLB_PRIO(priority);
+
+	ret = dlb_iface_map_qid(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: map qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		DLB_LOG_ERR("dlb: device_id=%d grp=%d, qm_port=%d, qm_qid=%d prio=%d\n",
+			    handle->device_id,
+			    handle->domain_id, cfg.port_id,
+			    cfg.qid,
+			    cfg.priority);
+	} else {
+		DLB_LOG_DBG("dlb: mapped queue %d to qm_port %d\n",
+			    qm_qid, qm_port_id);
+	}
+
+	return ret;
+}
+
+static int
+dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
+			 struct dlb_eventdev_port *ev_port,
+			 struct dlb_eventdev_queue *ev_queue,
+			 uint8_t priority)
+{
+	int first_avail = -1;
+	int ret, i;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid) {
+			if (ev_port->link[i].queue_id == ev_queue->id &&
+			    ev_port->link[i].priority == priority) {
+				if (ev_port->link[i].mapped)
+					return 0; /* already mapped */
+				first_avail = i;
+			}
+		} else {
+			if (first_avail == -1)
+				first_avail = i;
+		}
+	}
+	if (first_avail == -1) {
+		DLB_LOG_ERR("dlb: qm_port %d has no available QID slots.\n",
+			    ev_port->qm_port.id);
+		return -EINVAL;
+	}
+
+	ret = dlb_hw_map_ldb_qid_to_port(&dlb->qm_instance,
+					 ev_port->qm_port.id,
+					 ev_queue->qm_queue.id,
+					 priority);
+
+	if (!ret)
+		ev_port->link[first_avail].mapped = true;
+
+	return ret;
+}
+
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
+static int
+dlb_do_port_link(struct rte_eventdev *dev,
+		 struct dlb_eventdev_queue *ev_queue,
+		 struct dlb_eventdev_port *ev_port,
+		 uint8_t prio)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int err;
+
+	/* Don't link until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	if (ev_queue->qm_queue.is_directed)
+		err = dlb_eventdev_dir_queue_setup(dlb, ev_queue, ev_port);
+	else
+		err = dlb_event_queue_join_ldb(dlb, ev_port, ev_queue, prio);
+
+	if (err) {
+		DLB_LOG_ERR("port link failure for %s ev_q %d, ev_port %d\n",
+			    ev_queue->qm_queue.is_directed ? "DIR" : "LDB",
+			    ev_queue->id, ev_port->id);
+
+		rte_errno = err;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
+		       const uint8_t queues[], const uint8_t priorities[],
+		       uint16_t nb_links)
+
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i, j;
+
+	RTE_SET_USED(dev);
+
+	if (ev_port == NULL) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (!ev_port->setup_done &&
+	    ev_port->qm_port.config_state != DLB_PREV_CONFIGURED) {
+		DLB_LOG_ERR("dlb: evport not setup\n");
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	/* Note: rte_event_port_link() ensures the PMD won't receive a NULL
+	 * queues pointer.
+	 */
+	if (nb_links == 0) {
+		DLB_LOG_DBG("dlb: nb_links is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	dlb = ev_port->dlb;
+
+	DLB_LOG_DBG("Linking %u queues to %s port %d\n",
+		    nb_links,
+		    ev_port->qm_port.is_directed ? "DIR" : "LDB",
+		    ev_port->id);
+
+	for (i = 0; i < nb_links; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		uint8_t queue_id, prio;
+		bool found = false;
+		int index = -1;
+
+		queue_id = queues[i];
+		prio = priorities[i];
+
+		/* Check if the link already exists. */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].valid) {
+				if (ev_port->link[j].queue_id == queue_id) {
+					found = true;
+					index = j;
+					break;
+				}
+			} else {
+				if (index == -1)
+					index = j;
+			}
+
+		/* could not link */
+		if (index == -1)
+			break;
+
+		/* Check if already linked at the requested priority */
+		if (found && ev_port->link[j].priority == prio)
+			continue;
+
+		if (dlb_validate_port_link(ev_port, queue_id, found, index))
+			break; /* return index of offending queue */
+
+		ev_queue = &dlb->ev_queues[queue_id];
+
+		if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+			break; /* return index of offending queue */
+
+		ev_queue->num_links++;
+
+		ev_port->link[index].queue_id = queue_id;
+		ev_port->link[index].priority = prio;
+		ev_port->link[index].valid = true;
+		/* Entry already exists?  If so, then must be prio change */
+		if (!found)
+			ev_port->num_links++;
+	}
+	return i;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -1542,6 +1847,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_link        = dlb_eventdev_port_link,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
drivers/event/dlb/dlb_iface.c +9 -0
index fbbf9d7..aaf4506 100644
@@ -47,6 +47,15 @@ int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 				 struct dlb_create_dir_port_args *cfg,
 				 enum dlb_cq_poll_modes poll_mode);
 
+int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+			   struct dlb_unmap_qid_args *cfg);
+
+int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				     struct dlb_pending_port_unmaps_args *args);
+
 int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 				  enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/dlb_iface.h +9 -0
index d578185..c0f5f2e 100644
@@ -49,6 +49,15 @@ extern int (*dlb_iface_dir_port_create)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_ldb_queue_create)(struct dlb_hw_dev *handle,
 				  struct dlb_create_ldb_queue_args *cfg);
 
+extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
+			 struct dlb_map_qid_args *cfg);
+
+extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
+				  struct dlb_unmap_qid_args *cfg);
+
+extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
+				struct dlb_pending_port_unmaps_args *args);
+
 extern int (*dlb_iface_get_cq_poll_mode)(struct dlb_hw_dev *handle,
 					 enum dlb_cq_poll_modes *mode);
 
drivers/event/dlb/pf/base/dlb_resource.c +641 -0

644 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +69 -0

81 lines not on this page · raw message

Signature · 1 line
2.6.4
#175 lore raw

PATCH v13 15/23 event/dlb: add port unlink and port unlinks in progress

In reply to Timothy McDaniel

Add supports for the port unlink(s) eventdev entry points. The unlink operation is an asynchronous operation executed by a control thread, and the unlinks-in-progress function reads a counter shared with the control thread. Port QE and memzone memory is freed here.

---
 drivers/event/dlb/dlb.c | 166 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 166 insertions(+)
drivers/event/dlb/dlb.c +166 -0
index 2ad195d..c64f559 100644
@@ -693,6 +693,169 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)
 	return 0;
 }
 
+static int16_t
+dlb_hw_unmap_ldb_qid_from_port(struct dlb_hw_dev *handle,
+			       uint32_t qm_port_id,
+			       uint16_t qm_qid)
+{
+	struct dlb_unmap_qid_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	if (handle == NULL)
+		return -EINVAL;
+
+	cfg.response = (uintptr_t)&response;
+	cfg.port_id = qm_port_id;
+	cfg.qid = qm_qid;
+
+	ret = dlb_iface_unmap_qid(handle, &cfg);
+	if (ret < 0)
+		DLB_LOG_ERR("dlb: unmap qid error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+
+	return ret;
+}
+
+static int
+dlb_event_queue_detach_ldb(struct dlb_eventdev *dlb,
+			   struct dlb_eventdev_port *ev_port,
+			   struct dlb_eventdev_queue *ev_queue)
+{
+	int ret, i;
+
+	/* Don't unlink until start time. */
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED)
+		return 0;
+
+	for (i = 0; i < DLB_MAX_NUM_QIDS_PER_LDB_CQ; i++) {
+		if (ev_port->link[i].valid &&
+		    ev_port->link[i].queue_id == ev_queue->id)
+			break; /* found */
+	}
+
+	/* This is expected with eventdev API!
+	 * It blindly attempts to unmap all queues.
+	 */
+	if (i == DLB_MAX_NUM_QIDS_PER_LDB_CQ) {
+		DLB_LOG_DBG("dlb: ignoring LB QID %d not mapped for qm_port %d.\n",
+			    ev_queue->qm_queue.id,
+			    ev_port->qm_port.id);
+		return 0;
+	}
+
+	ret = dlb_hw_unmap_ldb_qid_from_port(&dlb->qm_instance,
+					     ev_port->qm_port.id,
+					     ev_queue->qm_queue.id);
+	if (!ret)
+		ev_port->link[i].mapped = false;
+
+	return ret;
+}
+
+static int
+dlb_eventdev_port_unlink(struct rte_eventdev *dev, void *event_port,
+			 uint8_t queues[], uint16_t nb_unlinks)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	int i;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	if (queues == NULL || nb_unlinks == 0) {
+		DLB_LOG_DBG("dlb: queues is NULL or nb_unlinks is 0\n");
+		return 0; /* Ignore and return success */
+	}
+
+	if (ev_port->qm_port.is_directed) {
+		DLB_LOG_DBG("dlb: ignore unlink from dir port %d\n",
+			    ev_port->id);
+		rte_errno = 0;
+		return nb_unlinks; /* as if success */
+	}
+
+	dlb = ev_port->dlb;
+
+	for (i = 0; i < nb_unlinks; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+		int ret, j;
+
+		if (queues[i] >= dlb->num_queues) {
+			DLB_LOG_ERR("dlb: invalid queue id %d\n", queues[i]);
+			rte_errno = -EINVAL;
+			return i; /* return index of offending queue */
+		}
+
+		ev_queue = &dlb->ev_queues[queues[i]];
+
+		/* Does a link exist? */
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++)
+			if (ev_port->link[j].queue_id == queues[i] &&
+			    ev_port->link[j].valid)
+				break;
+
+		if (j == DLB_MAX_NUM_QIDS_PER_LDB_CQ)
+			continue;
+
+		ret = dlb_event_queue_detach_ldb(dlb, ev_port, ev_queue);
+		if (ret) {
+			DLB_LOG_ERR("unlink err=%d for port %d queue %d\n",
+				    ret, ev_port->id, queues[i]);
+			rte_errno = -ENOENT;
+			return i; /* return index of offending queue */
+		}
+
+		ev_port->link[j].valid = false;
+		ev_port->num_links--;
+		ev_queue->num_links--;
+	}
+
+	return nb_unlinks;
+}
+
+static int
+dlb_eventdev_port_unlinks_in_progress(struct rte_eventdev *dev,
+				      void *event_port)
+{
+	struct dlb_eventdev_port *ev_port = event_port;
+	struct dlb_eventdev *dlb;
+	struct dlb_hw_dev *handle;
+	struct dlb_pending_port_unmaps_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	RTE_SET_USED(dev);
+
+	if (!ev_port->setup_done) {
+		DLB_LOG_ERR("dlb: evport %d is not configured\n",
+			    ev_port->id);
+		rte_errno = -EINVAL;
+		return 0;
+	}
+
+	cfg.port_id = ev_port->qm_port.id;
+	cfg.response = (uintptr_t)&response;
+	dlb = ev_port->dlb;
+	handle = &dlb->qm_instance;
+	ret = dlb_iface_pending_port_unmaps(handle, &cfg);
+
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: num_unlinks_in_progress ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
 static void
 dlb_eventdev_port_default_conf_get(struct rte_eventdev *dev,
 				   uint8_t port_id,
@@ -1848,6 +2011,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_setup      = dlb_eventdev_queue_setup,
 		.port_setup       = dlb_eventdev_port_setup,
 		.port_link        = dlb_eventdev_port_link,
+		.port_unlink      = dlb_eventdev_port_unlink,
+		.port_unlinks_in_progress =
+				    dlb_eventdev_port_unlinks_in_progress,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#176 lore raw

PATCH v13 16/23 event/dlb: add eventdev start

In reply to Timothy McDaniel

Add support for the eventdev start entry point. DLB delays setting up single link resources until eventdev start, because it is only then that it can ascertain which ports have just one linked queue.

---
 drivers/event/dlb/dlb.c                  | 224 +++++++++++++++++++++++++------
 drivers/event/dlb/dlb_iface.c            |   3 +
 drivers/event/dlb/dlb_iface.h            |   3 +
 drivers/event/dlb/pf/base/dlb_resource.c | 142 ++++++++++++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  23 ++++
 5 files changed, 351 insertions(+), 44 deletions(-)
drivers/event/dlb/dlb.c +180 -44
index c64f559..780ff7d 100644
@@ -1626,6 +1626,47 @@ dlb_eventdev_port_setup(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_reapply_configuration(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int ret, i;
+
+	/* If an event queue or port was previously configured, but hasn't been
+	 * reconfigured, reapply its original configuration.
+	 */
+	for (i = 0; i < dlb->num_queues; i++) {
+		struct dlb_eventdev_queue *ev_queue;
+
+		ev_queue = &dlb->ev_queues[i];
+
+		if (ev_queue->qm_queue.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_queue_setup(dev, i, &ev_queue->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure queue %d", i);
+			return ret;
+		}
+	}
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+
+		if (ev_port->qm_port.config_state != DLB_PREV_CONFIGURED)
+			continue;
+
+		ret = dlb_eventdev_port_setup(dev, i, &ev_port->conf);
+		if (ret < 0) {
+			DLB_LOG_ERR("dlb: failed to reconfigure ev_port %d",
+				    i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int
 set_dev_id(const char *key __rte_unused,
 	   const char *value,
 	   void *opaque)
@@ -1761,6 +1802,50 @@ dlb_validate_port_link(struct dlb_eventdev_port *ev_port,
 	return 0;
 }
 
+static int32_t
+dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_create_dir_queue_args cfg;
+	struct dlb_cmd_response response;
+	int32_t ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	/* The directed port is always configured before its queue */
+	cfg.port_id = qm_port_id;
+
+	ret = dlb_iface_dir_queue_create(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return -EINVAL;
+	}
+
+	return response.id;
+}
+
+static int
+dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
+			     struct dlb_eventdev_queue *ev_queue,
+			     struct dlb_eventdev_port *ev_port)
+{
+	int32_t qm_qid;
+
+	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
+
+	if (qm_qid < 0) {
+		DLB_LOG_ERR("Failed to create the DIR queue\n");
+		return qm_qid;
+	}
+
+	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
+
+	ev_queue->qm_queue.id = qm_qid;
+
+	return 0;
+}
+
 static int16_t
 dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle,
 			   uint32_t qm_port_id,
@@ -1836,50 +1921,6 @@ dlb_event_queue_join_ldb(struct dlb_eventdev *dlb,
 	return ret;
 }
 
-static int32_t
-dlb_hw_create_dir_queue(struct dlb_eventdev *dlb, int32_t qm_port_id)
-{
-	struct dlb_hw_dev *handle = &dlb->qm_instance;
-	struct dlb_create_dir_queue_args cfg;
-	struct dlb_cmd_response response;
-	int32_t ret;
-
-	cfg.response = (uintptr_t)&response;
-
-	/* The directed port is always configured before its queue */
-	cfg.port_id = qm_port_id;
-
-	ret = dlb_iface_dir_queue_create(handle, &cfg);
-	if (ret < 0) {
-		DLB_LOG_ERR("dlb: create DIR event queue error, ret=%d (driver status: %s)\n",
-			    ret, dlb_error_strings[response.status]);
-		return -EINVAL;
-	}
-
-	return response.id;
-}
-
-static int
-dlb_eventdev_dir_queue_setup(struct dlb_eventdev *dlb,
-			     struct dlb_eventdev_queue *ev_queue,
-			     struct dlb_eventdev_port *ev_port)
-{
-	int32_t qm_qid;
-
-	qm_qid = dlb_hw_create_dir_queue(dlb, ev_port->qm_port.id);
-
-	if (qm_qid < 0) {
-		DLB_LOG_ERR("Failed to create the DIR queue\n");
-		return qm_qid;
-	}
-
-	dlb->qm_dir_to_ev_queue_id[qm_qid] = ev_queue->id;
-
-	ev_queue->qm_queue.id = qm_qid;
-
-	return 0;
-}
-
 static int
 dlb_do_port_link(struct rte_eventdev *dev,
 		 struct dlb_eventdev_queue *ev_queue,
@@ -1911,6 +1952,40 @@ dlb_do_port_link(struct rte_eventdev *dev,
 }
 
 static int
+dlb_eventdev_apply_port_links(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	int i;
+
+	/* Perform requested port->queue links */
+	for (i = 0; i < dlb->num_ports; i++) {
+		struct dlb_eventdev_port *ev_port = &dlb->ev_ports[i];
+		int j;
+
+		for (j = 0; j < DLB_MAX_NUM_QIDS_PER_LDB_CQ; j++) {
+			struct dlb_eventdev_queue *ev_queue;
+			uint8_t prio, queue_id;
+
+			if (!ev_port->link[j].valid)
+				continue;
+
+			prio = ev_port->link[j].priority;
+			queue_id = ev_port->link[j].queue_id;
+
+			if (dlb_validate_port_link(ev_port, queue_id, true, j))
+				return -EINVAL;
+
+			ev_queue = &dlb->ev_queues[queue_id];
+
+			if (dlb_do_port_link(dev, ev_queue, ev_port, prio))
+				return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int
 dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 		       const uint8_t queues[], const uint8_t priorities[],
 		       uint16_t nb_links)
@@ -2000,12 +2075,73 @@ dlb_eventdev_port_link(struct rte_eventdev *dev, void *event_port,
 	return i;
 }
 
+static int
+dlb_eventdev_start(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_start_domain_args cfg;
+	struct dlb_cmd_response response;
+	int ret, i;
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+	if (dlb->run_state != DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_ERR("bad state %d for dev_start\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return -EINVAL;
+	}
+	dlb->run_state	= DLB_RUN_STATE_STARTING;
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	/* If the device was configured more than once, some event ports and/or
+	 * queues may need to be reconfigured.
+	 */
+	ret = dlb_eventdev_reapply_configuration(dev);
+	if (ret)
+		return ret;
+
+	/* The DLB PMD delays port links until the device is started. */
+	ret = dlb_eventdev_apply_port_links(dev);
+	if (ret)
+		return ret;
+
+	cfg.response = (uintptr_t)&response;
+
+	for (i = 0; i < dlb->num_ports; i++) {
+		if (!dlb->ev_ports[i].setup_done) {
+			DLB_LOG_ERR("dlb: port %d not setup", i);
+			return -ESTALE;
+		}
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0) {
+			DLB_LOG_ERR("dlb: queue %d is not linked", i);
+			return -ENOLINK;
+		}
+	}
+
+	ret = dlb_iface_sched_domain_start(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: sched_domain_start ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STARTED;
+	DLB_LOG_DBG("dlb: sched_domain_start completed OK\n");
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
 	static struct rte_eventdev_ops dlb_eventdev_entry_ops = {
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
+		.dev_start        = dlb_eventdev_start,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
drivers/event/dlb/dlb_iface.c +3 -0
index aaf4506..22d524b 100644
@@ -53,6 +53,9 @@ int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 			   struct dlb_unmap_qid_args *cfg);
 
+int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				     struct dlb_pending_port_unmaps_args *args);
 
drivers/event/dlb/dlb_iface.h +3 -0
index c0f5f2e..8c905ab 100644
@@ -55,6 +55,9 @@ extern int (*dlb_iface_map_qid)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_unmap_qid)(struct dlb_hw_dev *handle,
 				  struct dlb_unmap_qid_args *cfg);
 
+extern int (*dlb_iface_sched_domain_start)(struct dlb_hw_dev *handle,
+				    struct dlb_start_domain_args *cfg);
+
 extern int (*dlb_iface_pending_port_unmaps)(struct dlb_hw_dev *handle,
 				struct dlb_pending_port_unmaps_args *args);
 
drivers/event/dlb/pf/base/dlb_resource.c +142 -0

151 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +23 -0
index fed6719..1d2e133 100644
@@ -483,6 +483,28 @@ dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 }
 
 static int
+dlb_pf_sched_domain_start(struct dlb_hw_dev *handle,
+			  struct dlb_start_domain_args *cfg)
+{
+	struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
+	struct dlb_cmd_response response = {0};
+	int ret;
+
+	DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
+
+	ret = dlb_hw_start_domain(&dlb_dev->hw,
+				  handle->domain_id,
+				  cfg,
+				  &response);
+
+	*(struct dlb_cmd_response *)cfg->response = response;
+
+	DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
+
+	return ret;
+}
+
+static int
 dlb_pf_pending_port_unmaps(struct dlb_hw_dev *handle,
 			   struct dlb_pending_port_unmaps_args *args)
 {
@@ -565,6 +587,7 @@ dlb_pf_iface_fn_ptrs_init(void)
 	dlb_iface_dir_port_create = dlb_pf_dir_port_create;
 	dlb_iface_map_qid = dlb_pf_map_qid;
 	dlb_iface_unmap_qid = dlb_pf_unmap_qid;
+	dlb_iface_sched_domain_start = dlb_pf_sched_domain_start;
 	dlb_iface_pending_port_unmaps = dlb_pf_pending_port_unmaps;
 	dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 	dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
Signature · 1 line
2.6.4
#177 lore raw

PATCH v13 17/23 event/dlb: add enqueue and its burst variants

In reply to Timothy McDaniel

Add support for enqueue and its variants.

---
 doc/guides/eventdevs/dlb.rst | 163 ++++++++++-
 drivers/event/dlb/dlb.c      | 682 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 844 insertions(+), 1 deletion(-)
doc/guides/eventdevs/dlb.rst +162 -1
index f106a07..ae126c4 100644
@@ -118,7 +118,7 @@ the DLB does not limit the number of flows a queue can track. In the DLB, all
 load-balanced queues can use the full 16-bit flow ID range.
 
 Load-balanced and Directed Ports
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 DLB ports come in two flavors: load-balanced and directed. The eventdev API
 does not have the same concept, but it has a similar one: ports and queues that
@@ -157,3 +157,164 @@ type (atomic, ordered, or parallel) is not preserved, and an event's sched_type
 will be set to ``RTE_SCHED_TYPE_ATOMIC`` when it is dequeued from a directed
 port.
 
+Flow ID
+~~~~~~~
+
+The flow ID field is not preserved in the event when it is scheduled in the
+DLB, because the DLB hardware control word format does not have sufficient
+space to preserve every event field. As a result, the flow ID specified with
+the enqueued event will not be in the dequeued event. If this field is
+required, the application should pass it through an out-of-band path (for
+example in the mbuf's udata64 field, if the event points to an mbuf) or
+reconstruct the flow ID after receiving the event.
+
+Also, the DLB hardware control word supports a 16-bit flow ID. Since struct
+rte_event's flow_id field is 20 bits, the DLB PMD drops the most significant
+four bits from the event's flow ID.
+
+Hardware Credits
+~~~~~~~~~~~~~~~~
+
+DLB uses a hardware credit scheme to prevent software from overflowing hardware
+event storage, with each unit of storage represented by a credit. A port spends
+a credit to enqueue an event, and hardware refills the ports with credits as the
+events are scheduled to ports. Refills come from credit pools, and each port is
+a member of a load-balanced credit pool and a directed credit pool. The
+load-balanced credits are used to enqueue to load-balanced queues, and directed
+credits are used for directed queues.
+
+A DLB eventdev contains one load-balanced and one directed credit pool. These
+pools' sizes are controlled by the nb_events_limit field in struct
+rte_event_dev_config. The load-balanced pool is sized to contain
+nb_events_limit credits, and the directed pool is sized to contain
+nb_events_limit/4 credits. The directed pool size can be overridden with the
+num_dir_credits vdev argument, like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,num_dir_credits=<value>
+
+This can be used if the default allocation is too low or too high for the
+specific application needs. The PMD also supports a vdev arg that limits the
+max_num_events reported by rte_event_dev_info_get():
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,max_num_events=<value>
+
+By default, max_num_events is reported as the total available load-balanced
+credits. If multiple DLB-based applications are being used, it may be desirable
+to control how many load-balanced credits each application uses, particularly
+when application(s) are written to configure nb_events_limit equal to the
+reported max_num_events.
+
+Each port is a member of both credit pools. A port's credit allocation is
+defined by its low watermark, high watermark, and refill quanta. These three
+parameters are calculated by the dlb PMD like so:
+
+- The load-balanced high watermark is set to the port's enqueue_depth.
+  The directed high watermark is set to the minimum of the enqueue_depth and
+  the directed pool size divided by the total number of ports.
+- The refill quanta is set to half the high watermark.
+- The low watermark is set to the minimum of 16 and the refill quanta.
+
+When the eventdev is started, each port is pre-allocated a high watermark's
+worth of credits. For example, if an eventdev contains four ports with enqueue
+depths of 32 and a load-balanced credit pool size of 4096, each port will start
+with 32 load-balanced credits, and there will be 3968 credits available to
+replenish the ports. Thus, a single port is not capable of enqueueing up to the
+nb_events_limit (without any events being dequeued), since the other ports are
+retaining their initial credit allocation; in short, all ports must enqueue in
+order to reach the limit.
+
+If a port attempts to enqueue and has no credits available, the enqueue
+operation will fail and the application must retry the enqueue. Credits are
+replenished asynchronously by the DLB hardware.
+
+Software Credits
+~~~~~~~~~~~~~~~~
+
+The DLB is a "closed system" event dev, and the DLB PMD layers a software
+credit scheme on top of the hardware credit scheme in order to comply with
+the per-port backpressure described in the eventdev API.
+
+The DLB's hardware scheme is local to a queue/pipeline stage: a port spends a
+credit when it enqueues to a queue, and credits are later replenished after the
+events are dequeued and released.
+
+In the software credit scheme, a credit is consumed when a new (.op =
+RTE_EVENT_OP_NEW) event is injected into the system, and the credit is
+replenished when the event is released from the system (either explicitly with
+RTE_EVENT_OP_RELEASE or implicitly in dequeue_burst()).
+
+In this model, an event is "in the system" from its first enqueue into eventdev
+until it is last dequeued. If the event goes through multiple event queues, it
+is still considered "in the system" while a worker thread is processing it.
+
+A port will fail to enqueue if the number of events in the system exceeds its
+``new_event_threshold`` (specified at port setup time). A port will also fail
+to enqueue if it lacks enough hardware credits to enqueue; load-balanced
+credits are used to enqueue to a load-balanced queue, and directed credits are
+used to enqueue to a directed queue.
+
+The out-of-credit situations are typically transient, and an eventdev
+application using the DLB ought to retry its enqueues if they fail.
+If enqueue fails, DLB PMD sets rte_errno as follows:
+
+- -ENOSPC: Credit exhaustion (either hardware or software)
+- -EINVAL: Invalid argument, such as port ID, queue ID, or sched_type.
+
+Depending on the pipeline the application has constructed, it's possible to
+enter a credit deadlock scenario wherein the worker thread lacks the credit
+to enqueue an event, and it must dequeue an event before it can recover the
+credit. If the worker thread retries its enqueue indefinitely, it will not
+make forward progress. Such deadlock is possible if the application has event
+"loops", in which an event in dequeued from queue A and later enqueued back to
+queue A.
+
+Due to this, workers should stop retrying after a time, release the events it
+is attempting to enqueue, and dequeue more events. It is important that the
+worker release the events and don't simply set them aside to retry the enqueue
+again later, because the port has limited history list size (by default, twice
+the port's dequeue_depth).
+
+Priority
+~~~~~~~~
+
+The DLB supports event priority and per-port queue service priority, as
+described in the eventdev header file. The DLB does not support 'global' event
+queue priority established at queue creation time.
+
+DLB supports 8 event and queue service priority levels. For both priority
+types, the PMD uses the upper three bits of the priority field to determine the
+DLB priority, discarding the 5 least significant bits. The 5 least significant
+event priority bits are not preserved when an event is enqueued.
+
+Atomic Inflights Allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the last stage prior to scheduling an atomic event to a CQ, DLB holds the
+inflight event in a temporary buffer that is divided among load-balanced
+queues. If a queue's atomic buffer storage fills up, this can result in
+head-of-line-blocking. For example:
+
+- An LDB queue allocated N atomic buffer entries
+- All N entries are filled with events from flow X, which is pinned to CQ 0.
+
+Until CQ 0 releases 1+ events, no other atomic flows for that LDB queue can be
+scheduled. The likelihood of this case depends on the eventdev configuration,
+traffic behavior, event processing latency, potential for a worker to be
+interrupted or otherwise delayed, etc.
+
+By default, the PMD allocates 16 buffer entries for each load-balanced queue,
+which provides an even division across all 128 queues but potentially wastes
+buffer space (e.g. if not all queues are used, or aren't used for atomic
+scheduling).
+
+The PMD provides a dev arg to override the default per-queue allocation. To
+increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,atm_inflights=64
+
drivers/event/dlb/dlb.c +682 -0

700 lines not on this page · raw message

Signature · 1 line
2.6.4
#178 lore raw

PATCH v13 18/23 event/dlb: add dequeue and its burst variants

In reply to Timothy McDaniel

Add support for dequeue, dequeue_burst, ...

DLB does not currently support interrupts, but instead uses umonitor/umwait if supported by the processor. This allows the software to monitor and wait on writes to a cache-line.

DLB supports normal and sparse cq mode. In normal mode the hardware will pack 4 QEs into each cache line. In sparse cq mode, the hardware will only populate one QE per cache line. Software must be aware of the cq mode, and take the appropriate actions, based on the mode.

---
 doc/guides/eventdevs/dlb.rst |  21 ++
 drivers/event/dlb/dlb.c      | 679 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 700 insertions(+)
doc/guides/eventdevs/dlb.rst +21 -0
index ae126c4..4c4f56b 100644
@@ -318,3 +318,24 @@ increase a vdev's per-queue atomic-inflight allocation to (for example) 64:
 
        --vdev=dlb1_event,atm_inflights=64
 
+Deferred Scheduling
+~~~~~~~~~~~~~~~~~~~
+
+The DLB PMD's default behavior for managing a CQ is to "pop" the CQ once per
+dequeued event before returning from rte_event_dequeue_burst(). This frees the
+corresponding entries in the CQ, which enables the DLB to schedule more events
+to it.
+
+To support applications seeking finer-grained scheduling control -- for example
+deferring scheduling to get the best possible priority scheduling and
+load-balancing -- the PMD supports a deferred scheduling mode. In this mode,
+the CQ entry is not popped until the *subsequent* rte_event_dequeue_burst()
+call. This mode only applies to load-balanced event ports with dequeue depth of
+1.
+
+To enable deferred scheduling, use the defer_sched vdev argument like so:
+
+    .. code-block:: console
+
+       --vdev=dlb1_event,defer_sched=on
+
drivers/event/dlb/dlb.c +679 -0

700 lines not on this page · raw message

Signature · 1 line
2.6.4
#179 lore raw

PATCH v13 19/23 event/dlb: add eventdev stop and close

In reply to Timothy McDaniel

Add support for eventdev stop and close entry points.

---
 drivers/event/dlb/dlb.c                  | 256 +++++++++++++++++++++++++++++--
 drivers/event/dlb/dlb_iface.c            |   6 +
 drivers/event/dlb/dlb_iface.h            |   6 +
 drivers/event/dlb/pf/base/dlb_resource.c |  89 +++++++++++
 drivers/event/dlb/pf/dlb_pf.c            |  47 ++++++
 5 files changed, 393 insertions(+), 11 deletions(-)
drivers/event/dlb/dlb.c +245 -11
index e2f769c..a8efcc1 100644
@@ -91,17 +91,6 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,
 					const struct rte_event events[],
 					uint16_t num);
 
-uint32_t
-dlb_get_queue_depth(struct dlb_eventdev *dlb,
-		    struct dlb_eventdev_queue *queue)
-{
-	/* DUMMY FOR NOW So "xstats" patch compiles */
-	RTE_SET_USED(dlb);
-	RTE_SET_USED(queue);
-
-	return 0;
-}
-
 static int
 dlb_hw_query_resources(struct dlb_eventdev *dlb)
 {
@@ -3480,6 +3469,249 @@ dlb_event_dequeue_sparse(void *event_port, struct rte_event *ev, uint64_t wait)
 	return dlb_event_dequeue_burst_sparse(event_port, ev, 1, wait);
 }
 
+static uint32_t
+dlb_get_ldb_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_ldb_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_ldb_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+static uint32_t
+dlb_get_dir_queue_depth(struct dlb_eventdev *dlb,
+			struct dlb_eventdev_queue *queue)
+{
+	struct dlb_hw_dev *handle = &dlb->qm_instance;
+	struct dlb_get_dir_queue_depth_args cfg;
+	struct dlb_cmd_response response;
+	int ret;
+
+	cfg.queue_id = queue->qm_queue.id;
+	cfg.response = (uintptr_t)&response;
+
+	ret = dlb_iface_get_dir_queue_depth(handle, &cfg);
+	if (ret < 0) {
+		DLB_LOG_ERR("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n",
+			    ret, dlb_error_strings[response.status]);
+		return ret;
+	}
+
+	return response.id;
+}
+
+uint32_t
+dlb_get_queue_depth(struct dlb_eventdev *dlb,
+		    struct dlb_eventdev_queue *queue)
+{
+	if (queue->qm_queue.is_directed)
+		return dlb_get_dir_queue_depth(dlb, queue);
+	else
+		return dlb_get_ldb_queue_depth(dlb, queue);
+}
+
+static bool
+dlb_queue_is_empty(struct dlb_eventdev *dlb,
+		   struct dlb_eventdev_queue *queue)
+{
+	return dlb_get_queue_depth(dlb, queue) == 0;
+}
+
+static bool
+dlb_linked_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (dlb->ev_queues[i].num_links == 0)
+			continue;
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static bool
+dlb_queues_empty(struct dlb_eventdev *dlb)
+{
+	int i;
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		if (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			return false;
+	}
+
+	return true;
+}
+
+static void
+dlb_flush_port(struct rte_eventdev *dev, int port_id)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	eventdev_stop_flush_t flush;
+	struct rte_event ev;
+	uint8_t dev_id;
+	void *arg;
+	int i;
+
+	flush = dev->dev_ops->dev_stop_flush;
+	dev_id = dev->data->dev_id;
+	arg = dev->data->dev_stop_flush_arg;
+
+	while (rte_event_dequeue_burst(dev_id, port_id, &ev, 1, 0)) {
+		if (flush)
+			flush(dev_id, ev, arg);
+
+		if (dlb->ev_ports[port_id].qm_port.is_directed)
+			continue;
+
+		ev.op = RTE_EVENT_OP_RELEASE;
+
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+	}
+
+	/* Enqueue any additional outstanding releases */
+	ev.op = RTE_EVENT_OP_RELEASE;
+
+	for (i = dlb->ev_ports[port_id].outstanding_releases; i > 0; i--)
+		rte_event_enqueue_burst(dev_id, port_id, &ev, 1);
+}
+
+static void
+dlb_drain(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+	struct dlb_eventdev_port *ev_port = NULL;
+	uint8_t dev_id;
+	int i;
+
+	dev_id = dev->data->dev_id;
+
+	while (!dlb_linked_queues_empty(dlb)) {
+		/* Flush all the ev_ports, which will drain all their connected
+		 * queues.
+		 */
+		for (i = 0; i < dlb->num_ports; i++)
+			dlb_flush_port(dev, i);
+	}
+
+	/* The queues are empty, but there may be events left in the ports. */
+	for (i = 0; i < dlb->num_ports; i++)
+		dlb_flush_port(dev, i);
+
+	/* If the domain's queues are empty, we're done. */
+	if (dlb_queues_empty(dlb))
+		return;
+
+	/* Else, there must be at least one unlinked load-balanced queue.
+	 * Select a load-balanced port with which to drain the unlinked
+	 * queue(s).
+	 */
+	for (i = 0; i < dlb->num_ports; i++) {
+		ev_port = &dlb->ev_ports[i];
+
+		if (!ev_port->qm_port.is_directed)
+			break;
+	}
+
+	if (i == dlb->num_ports) {
+		DLB_LOG_ERR("internal error: no LDB ev_ports\n");
+		return;
+	}
+
+	rte_errno = 0;
+	rte_event_port_unlink(dev_id, ev_port->id, NULL, 0);
+
+	if (rte_errno) {
+		DLB_LOG_ERR("internal error: failed to unlink ev_port %d\n",
+			    ev_port->id);
+		return;
+	}
+
+	for (i = 0; i < dlb->num_queues; i++) {
+		uint8_t qid, prio;
+		int ret;
+
+		if (dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			continue;
+
+		qid = i;
+		prio = 0;
+
+		/* Link the ev_port to the queue */
+		ret = rte_event_port_link(dev_id, ev_port->id, &qid, &prio, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to link ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+
+		/* Flush the queue */
+		while (!dlb_queue_is_empty(dlb, &dlb->ev_queues[i]))
+			dlb_flush_port(dev, ev_port->id);
+
+		/* Drain any extant events in the ev_port. */
+		dlb_flush_port(dev, ev_port->id);
+
+		/* Unlink the ev_port from the queue */
+		ret = rte_event_port_unlink(dev_id, ev_port->id, &qid, 1);
+		if (ret != 1) {
+			DLB_LOG_ERR("internal error: failed to unlink ev_port %d to queue %d\n",
+				    ev_port->id, qid);
+			return;
+		}
+	}
+}
+
+static void
+dlb_eventdev_stop(struct rte_eventdev *dev)
+{
+	struct dlb_eventdev *dlb = dlb_pmd_priv(dev);
+
+	rte_spinlock_lock(&dlb->qm_instance.resource_lock);
+
+	if (dlb->run_state == DLB_RUN_STATE_STOPPED) {
+		DLB_LOG_DBG("Internal error: already stopped\n");
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	} else if (dlb->run_state != DLB_RUN_STATE_STARTED) {
+		DLB_LOG_ERR("Internal error: bad state %d for dev_stop\n",
+			    (int)dlb->run_state);
+		rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+		return;
+	}
+
+	dlb->run_state = DLB_RUN_STATE_STOPPING;
+
+	rte_spinlock_unlock(&dlb->qm_instance.resource_lock);
+
+	dlb_drain(dev);
+
+	dlb->run_state = DLB_RUN_STATE_STOPPED;
+}
+
+static int
+dlb_eventdev_close(struct rte_eventdev *dev)
+{
+	dlb_hw_reset_sched_domain(dev, false);
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3489,6 +3721,8 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.dev_infos_get    = dlb_eventdev_info_get,
 		.dev_configure    = dlb_eventdev_configure,
 		.dev_start        = dlb_eventdev_start,
+		.dev_stop         = dlb_eventdev_stop,
+		.dev_close        = dlb_eventdev_close,
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
drivers/event/dlb/dlb_iface.c +6 -0
index 22d524b..44f958f 100644
@@ -71,3 +71,9 @@ int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_ldb_queue_depth_args *args);
+
+int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				     struct dlb_get_dir_queue_depth_args *args);
+
drivers/event/dlb/dlb_iface.h +6 -0
index 8c905ab..9f61135 100644
@@ -73,4 +73,10 @@ extern int (*dlb_iface_set_sn_allocation)(struct dlb_hw_dev *handle,
 extern int (*dlb_iface_get_sn_occupancy)(struct dlb_hw_dev *handle,
 				  struct dlb_get_sn_occupancy_args *args);
 
+extern int (*dlb_iface_get_ldb_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_ldb_queue_depth_args *args);
+
+extern int (*dlb_iface_get_dir_queue_depth)(struct dlb_hw_dev *handle,
+				    struct dlb_get_dir_queue_depth_args *args);
+
 #endif /* _DLB_IFACE_H */
drivers/event/dlb/pf/base/dlb_resource.c +89 -0

92 lines not on this page · raw message

drivers/event/dlb/pf/dlb_pf.c +47 -0

63 lines not on this page · raw message

Signature · 1 line
2.6.4
#180 lore raw

PATCH v13 20/23 event/dlb: add PMD's token pop public interface

In reply to Timothy McDaniel

The PMD uses a public interface to allow applications to control the token pop mode. Supported token pop modes are as follows, and they impact core scheduling affinity for ldb ports.

AUTO_POP: Pop the CQ tokens immediately after dequeueing.
DELAYED_POP: Pop CQ tokens after (dequeue_depth - 1) events
are released. Supported on load-balanced ports only.
DEFERRED_POP: Pop the CQ tokens during next dequeue operation.

---
 doc/api/doxy-api-index.md       |   1 +
 drivers/event/dlb/dlb.c         | 121 ++++++++++++++++++++++++++++++++++++----
 drivers/event/dlb/dlb_priv.h    |   3 +
 drivers/event/dlb/meson.build   |   4 +-
 drivers/event/dlb/rte_pmd_dlb.c |  38 +++++++++++++
 drivers/event/dlb/rte_pmd_dlb.h |  77 +++++++++++++++++++++++++
 drivers/event/dlb/version.map   |   6 ++
 7 files changed, 237 insertions(+), 13 deletions(-)
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
doc/api/doxy-api-index.md +1 -0
index a9c12d1..1c83bf4 100644
@@ -52,6 +52,7 @@ The public API headers are grouped by topics:
   [dpaa2_cmdif]        (@ref rte_pmd_dpaa2_cmdif.h),
   [dpaa2_qdma]         (@ref rte_pmd_dpaa2_qdma.h),
   [crypto_scheduler]   (@ref rte_cryptodev_scheduler.h)
+  [dlb]		       (@ref rte_pmd_dlb.h),
 
 - **memory**:
   [memseg]             (@ref rte_memory.h),
drivers/event/dlb/dlb.c +109 -12

247 lines not on this page · raw message

drivers/event/dlb/dlb_priv.h +3 -0
index adb1f7a..58ff428 100644
@@ -16,6 +16,7 @@
 
 #include "dlb_user.h"
 #include "dlb_log.h"
+#include "rte_pmd_dlb.h"
 
 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
@@ -262,6 +263,7 @@ struct dlb_port {
 	bool gen_bit;
 	uint16_t dir_credits;
 	uint32_t dequeue_depth;
+	enum dlb_token_pop_mode token_pop_mode;
 	int pp_mmio_base;
 	uint16_t cached_ldb_credits;
 	uint16_t ldb_pushcount_at_credit_expiry;
@@ -273,6 +275,7 @@ struct dlb_port {
 	uint8_t cq_rsvd_token_deficit;
 	uint16_t owed_tokens;
 	int16_t issued_releases;
+	int16_t token_pop_thresh;
 	int cq_depth;
 	uint16_t cq_idx;
 	uint16_t cq_idx_unmasked;
drivers/event/dlb/meson.build +3 -1
index 552ff9d..7f38c30 100644
@@ -12,7 +12,9 @@ sources = files('dlb.c',
 		'dlb_xstats.c',
 		'pf/dlb_main.c',
 		'pf/dlb_pf.c',
-		'pf/base/dlb_resource.c'
+		'pf/base/dlb_resource.c',
+		'rte_pmd_dlb.c',
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
+install_headers('rte_pmd_dlb.h')
drivers/event/dlb/rte_pmd_dlb.c new file +38 -0
new file mode 100644
index 0000000..bc802d3
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Intel Corporation
+ */
+
+#include "rte_eventdev.h"
+#include "rte_eventdev_pmd.h"
+#include "rte_pmd_dlb.h"
+#include "dlb_priv.h"
+#include "dlb_inline_fns.h"
+
+int
+rte_pmd_dlb_set_token_pop_mode(uint8_t dev_id,
+			       uint8_t port_id,
+			       enum dlb_token_pop_mode mode)
+{
+	struct dlb_eventdev *dlb;
+	struct rte_eventdev *dev;
+
+	RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
+	dev = &rte_eventdevs[dev_id];
+
+	dlb = dlb_pmd_priv(dev);
+
+	if (mode >= NUM_TOKEN_POP_MODES)
+		return -EINVAL;
+
+	/* The event device must be configured, but not yet started */
+	if (!dlb->configured || dlb->run_state != DLB_RUN_STATE_STOPPED)
+		return -EINVAL;
+
+	/* The token pop mode must be set before configuring the port */
+	if (port_id >= dlb->num_ports || dlb->ev_ports[port_id].setup_done)
+		return -EINVAL;
+
+	dlb->ev_ports[port_id].qm_port.token_pop_mode = mode;
+
+	return 0;
+}
drivers/event/dlb/rte_pmd_dlb.h new file +77 -0

77 lines not on this page · raw message

drivers/event/dlb/version.map +6 -0
index 4a76d1d..3338a22 100644
@@ -1,3 +1,9 @@
 DPDK_21 {
 	local: *;
 };
+
+EXPERIMENTAL {
+	global:
+
+	rte_pmd_dlb_set_token_pop_mode;
+};
Signature · 1 line
2.6.4
#181 lore raw

PATCH v13 21/23 event/dlb: add PMD self-tests

In reply to Timothy McDaniel

Add a variety of self-tests for both ldb and directed ports/queues, as well as configure, start, stop, link, etc...

---
 app/test/test_eventdev.c         |    7 +
 drivers/event/dlb/dlb.c          |    1 +
 drivers/event/dlb/dlb_selftest.c | 1539 ++++++++++++++++++++++++++++++++++++++
 drivers/event/dlb/meson.build    |    1 +
 4 files changed, 1548 insertions(+)
 create mode 100644 drivers/event/dlb/dlb_selftest.c
app/test/test_eventdev.c +7 -0
index 62019c1..ba27bed 100644
@@ -1030,6 +1030,12 @@ test_eventdev_selftest_dpaa2(void)
 	return test_eventdev_selftest_impl("event_dpaa2", "");
 }
 
+static int
+test_eventdev_selftest_dlb(void)
+{
+	return test_eventdev_selftest_impl("dlb_event", "");
+}
+
 REGISTER_TEST_COMMAND(eventdev_common_autotest, test_eventdev_common);
 REGISTER_TEST_COMMAND(eventdev_selftest_sw, test_eventdev_selftest_sw);
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
@@ -1037,3 +1043,4 @@ REGISTER_TEST_COMMAND(eventdev_selftest_octeontx,
 REGISTER_TEST_COMMAND(eventdev_selftest_octeontx2,
 		test_eventdev_selftest_octeontx2);
 REGISTER_TEST_COMMAND(eventdev_selftest_dpaa2, test_eventdev_selftest_dpaa2);
+REGISTER_TEST_COMMAND(eventdev_selftest_dlb, test_eventdev_selftest_dlb);
drivers/event/dlb/dlb.c +1 -0
index 31e4d50..c37de85 100644
@@ -3829,6 +3829,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
 		.xstats_get_by_name = dlb_eventdev_xstats_get_by_name,
 		.xstats_reset	    = dlb_eventdev_xstats_reset,
+		.dev_selftest     = test_dlb_eventdev,
 	};
 
 	/* Expose PMD's eventdev interface */
drivers/event/dlb/dlb_selftest.c new file +1539 -0

1,539 lines not on this page · raw message

drivers/event/dlb/meson.build +1 -0
index 7f38c30..875cf89 100644
@@ -14,6 +14,7 @@ sources = files('dlb.c',
 		'pf/dlb_pf.c',
 		'pf/base/dlb_resource.c',
 		'rte_pmd_dlb.c',
+		'dlb_selftest.c'
 )
 
 deps += ['mbuf', 'mempool', 'ring', 'pci', 'bus_pci']
Signature · 1 line
2.6.4
#182 lore raw

PATCH v13 22/23 event/dlb: add queue and port release

In reply to Timothy McDaniel

These entry points are NO-OPS. DLB does not support reconfiguring individual queues or ports. The entire device must be reconfigured.

---
 drivers/event/dlb/dlb.c | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
drivers/event/dlb/dlb.c +27 -0
index c37de85..41ccb4a 100644
@@ -159,6 +159,9 @@ dlb_free_qe_mem(struct dlb_port *qm_port)
 
 	rte_free(qm_port->consume_qe);
 	qm_port->consume_qe = NULL;
+
+	rte_memzone_free(dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz);
+	dlb_port[qm_port->id][PORT_TYPE(qm_port)].mz = NULL;
 }
 
 static int
@@ -3805,6 +3808,28 @@ dlb_eventdev_close(struct rte_eventdev *dev)
 	return 0;
 }
 
+static void
+dlb_eventdev_port_release(void *port)
+{
+	struct dlb_eventdev_port *ev_port = port;
+
+	if (ev_port) {
+		struct dlb_port *qm_port = &ev_port->qm_port;
+
+		if (qm_port->config_state == DLB_CONFIGURED)
+			dlb_free_qe_mem(qm_port);
+	}
+}
+
+static void
+dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
+{
+	RTE_SET_USED(dev);
+	RTE_SET_USED(id);
+
+	/* This function intentionally left blank. */
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3819,7 +3844,9 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.queue_def_conf   = dlb_eventdev_queue_default_conf_get,
 		.port_def_conf    = dlb_eventdev_port_default_conf_get,
 		.queue_setup      = dlb_eventdev_queue_setup,
+		.queue_release    = dlb_eventdev_queue_release,
 		.port_setup       = dlb_eventdev_port_setup,
+		.port_release     = dlb_eventdev_port_release,
 		.port_link        = dlb_eventdev_port_link,
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
Signature · 1 line
2.6.4
#183 lore raw

PATCH v13 23/23 event/dlb: add timeout ticks entry point

In reply to Timothy McDaniel

Adds the timeout ticks conversion function.

---
 drivers/event/dlb/dlb.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)
drivers/event/dlb/dlb.c +13 -0
index 41ccb4a..4aab9f6 100644
@@ -3830,6 +3830,18 @@ dlb_eventdev_queue_release(struct rte_eventdev *dev, uint8_t id)
 	/* This function intentionally left blank. */
 }
 
+static int
+dlb_eventdev_timeout_ticks(struct rte_eventdev *dev, uint64_t ns,
+			   uint64_t *timeout_ticks)
+{
+	RTE_SET_USED(dev);
+	uint64_t cycles_per_ns = rte_get_timer_hz() / 1E9;
+
+	*timeout_ticks = ns * cycles_per_ns;
+
+	return 0;
+}
+
 void
 dlb_entry_points_init(struct rte_eventdev *dev)
 {
@@ -3851,6 +3863,7 @@ dlb_entry_points_init(struct rte_eventdev *dev)
 		.port_unlink      = dlb_eventdev_port_unlink,
 		.port_unlinks_in_progress =
 				    dlb_eventdev_port_unlinks_in_progress,
+		.timeout_ticks    = dlb_eventdev_timeout_ticks,
 		.dump             = dlb_eventdev_dump,
 		.xstats_get       = dlb_eventdev_xstats_get,
 		.xstats_get_names = dlb_eventdev_xstats_get_names,
Signature · 1 line
2.6.4
#184 lore raw

Re: [dpdk-dev] [PATCH v13 00/23] Add DLB PMD

In reply to Timothy McDaniel
Quoted 32 lines · Timothy McDaniel

The following patch series adds support for a new eventdev PMD. The DLB PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware. The DLB is a PCIe device that provides load-balanced, prioritized scheduling of core-to-core communication. The device consists of queues and arbiters that connect producer and consumer cores, and implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced distribution.

The DLB hardware supports both load balanced and directed ports and queues. Unlike other eventdev devices already in the repo, not all DLB ports and queues are equally capable. In particular, directed ports are limited to a single link, and must be connected to a directed queue.
Additionally, even though LDB ports may link multiple queues, the number of queues that may be linked is limited by hardware. Another difference is that DLB does not have a straightforward way of carrying the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full control over the DLB hardware. Intel will be extending the DLB PMD in the future (not as part of this first series) with a mode that we refer to as the bifurcated PMD. The bifurcated PMD communicates with a kernel driver to configure the device, ports, and queues, and memory maps device MMIO so datapath operations occur purely in user-space.

Quoted 6 lines

The framework to support both the PF PMD and bifurcated PMD exists in this patchset, and is why the iface.[ch] layer is present.

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

build error with clang at "event/dlb: add enqueue and its burst variants" patch. Please make sure each patch builds to avoid delay in merging the patch.
Also, address the David comment on the doc for the next version.

FAILED: drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o ccache clang -Idrivers/libtmp_rte_event_dlb.a.p -Idrivers -I../drivers -Idrivers/event/dlb -I../drivers/event/dlb -Ilib/librte_eventdev -I../lib/librte_eventdev -I. -I.. -Iconfig -I../config -Ilib/librte_eal/include -I../lib/librte_eal/incl
ude -Ilib/librte_eal/linux/include -I../lib/librte_eal/linux/include -Ilib/librte_eal/x86/include -I../lib/librte_eal/x86/include -Ilib/librte_eal/common -I../lib/librte_eal/common -Ilib/librte_eal -I../lib/librte_eal -Ilib/librte_kvargs -I ../lib/librte_kvargs -Ilib/librte_metrics -I../lib/librte_metrics -Ilib/librte_telemetry -I../lib/librte_telemetry -Ilib/librte_ring -I../lib/librte_ring -Ilib/librte_ethdev -I../lib/librte_ethdev -Ilib/librte_net -I../lib/librte_net -Ilib/
librte_mbuf -I../lib/librte_mbuf -Ilib/librte_mempool -I../lib/librte_mempool -Ilib/librte_meter -I../lib/librte_meter -Ilib/librte_hash -I../lib/librte_hash -Ilib/librte_rcu -I../lib/librte_rcu -Ilib/librte_timer -I../lib/librte_timer -Ili b/librte_cryptodev -I../lib/librte_cryptodev -Ilib/librte_pci -I../lib/librte_pci -Idrivers/bus/pci -I../drivers/bus/pci -I../drivers/bus/pci/linux -Xclang -fcolor-diagnostics -pipe -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g -include rte_config.h -Wextra -Wcast-qual -Wdeprecated -Wformat-nonliteral -Wformat-security -Wmissing-declarations -Wmissing-prototypes -Wnested-externs -Wold-style-definition -Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwr ite-strings -Wno-address-of-packed-member -Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native -DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -MD -MQ drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -MF drivers/libtmp_rte_even
t_dlb.a.p/event_dlb_dlb.c.o.d -o drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -c ../drivers/event/dlb/dlb.c ../drivers/event/dlb/dlb.c:2777:1: error: unused function 'dlb_event_enqueue_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2762:1: error: unused function 'dlb_event_enqueue_burst_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2792:1: error: unused function 'dlb_event_enqueue_new_burst_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_new_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2808:1: error: unused function 'dlb_event_enqueue_forward_burst_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_forward_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2605:1: error: unused function 'dlb_construct_token_pop_qe' [-Werror,-Wunused-function] dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx) ^
../drivers/event/dlb/dlb.c:2653:1: error: unused function 'dlb_consume_qe_immediate' [-Werror,-Wunused-function] dlb_consume_qe_immediate(struct dlb_port *qm_port, int num) ^
6 errors generated.

Quoted 186 lines

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of umonitor/umwait, rather than the new functions in the power patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The software is very similar in form so some DLB2 reviews comments were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ memory for depths < 8. Hardware requires minimum allocation to be at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias. I will address the remaining items identified by Mattias in the next patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
event/dlb: add documentation and meson infrastructure event/dlb: add dynamic logging
event/dlb: add private data structures and constants event/dlb: add definitions shared with LKM or shared code event/dlb: add inline functions
event/dlb: add eventdev probe
event/dlb: add flexible interface
event/dlb: add probe-time hardware init event/dlb: add xstats
event/dlb: add infos get and configure event/dlb: add queue and port default conf event/dlb: add queue setup
event/dlb: add port setup
event/dlb: add port link
event/dlb: add port unlink and port unlinks in progress event/dlb: add eventdev start
event/dlb: add enqueue and its burst variants event/dlb: add dequeue and its burst variants event/dlb: add eventdev stop and close event/dlb: add PMD's token pop public interface event/dlb: add PMD self-tests
event/dlb: add queue and port release
event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    1 +
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4080 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   40 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21684 insertions(+), 2 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map

--
2.6.4

#185 lore raw

Re: [dpdk-dev] [PATCH v13 00/23] Add DLB PMD

Quoted 109 lines

-----Original Message-----
From: Jerin Jacob <[email protected]>
Sent: Saturday, October 31, 2020 7:50 AM
To: McDaniel, Timothy <[email protected]>
Cc: dpdk-dev <[email protected]>; Carrillo, Erik G <[email protected]>; Eads, Gage <[email protected]>; Van Haaren, Harry <[email protected]>; Jerin Jacob <[email protected]>; Thomas Monjalon <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v13 00/23] Add DLB PMD

Quoted 32 lines · Timothy McDaniel

The following patch series adds support for a new eventdev PMD. The DLB PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware. The DLB is a PCIe device that provides load-balanced, prioritized scheduling of core-to-core communication. The device consists of queues and arbiters that connect producer and consumer cores, and implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced distribution.

The DLB hardware supports both load balanced and directed ports and queues. Unlike other eventdev devices already in the repo, not all DLB ports and queues are equally capable. In particular, directed ports are limited to a single link, and must be connected to a directed queue.
Additionally, even though LDB ports may link multiple queues, the number of queues that may be linked is limited by hardware. Another difference is that DLB does not have a straightforward way of carrying the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full control over the DLB hardware. Intel will be extending the DLB PMD in the future (not as part of this first series) with a mode that we refer to as the bifurcated PMD. The bifurcated PMD communicates with a kernel driver to configure the device, ports, and queues, and memory maps device MMIO so datapath operations occur purely in user-space.

Quoted 6 lines

The framework to support both the PF PMD and bifurcated PMD exists in this patchset, and is why the iface.[ch] layer is present.

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

build error with clang at "event/dlb: add enqueue and its burst variants" patch. Please make sure each patch builds to avoid delay in merging the patch.
Also, address the David comment on the doc for the next version.

FAILED: drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o ccache clang -Idrivers/libtmp_rte_event_dlb.a.p -Idrivers -I../drivers

(quoted diff) +0 -3
-Idrivers/event/dlb -I../drivers/event/dlb -Ilib/librte_eventdev
-I../lib/librte_eventdev -I. -I.. -Iconfig -I../config
-Ilib/librte_eal/include -I../lib/librte_eal/incl

ude -Ilib/librte_eal/linux/include -I../lib/librte_eal/linux/include

(quoted diff) +0 -3
-Ilib/librte_eal/x86/include -I../lib/librte_eal/x86/include
-Ilib/librte_eal/common -I../lib/librte_eal/common -Ilib/librte_eal
-I../lib/librte_eal -Ilib/librte_kvargs -I

../lib/librte_kvargs -Ilib/librte_metrics -I../lib/librte_metrics

(quoted diff) +0 -3
-Ilib/librte_telemetry -I../lib/librte_telemetry -Ilib/librte_ring
-I../lib/librte_ring -Ilib/librte_ethdev -I../lib/librte_ethdev
-Ilib/librte_net -I../lib/librte_net -Ilib/

librte_mbuf -I../lib/librte_mbuf -Ilib/librte_mempool

(quoted diff) +0 -3
-I../lib/librte_mempool -Ilib/librte_meter -I../lib/librte_meter
-Ilib/librte_hash -I../lib/librte_hash -Ilib/librte_rcu
-I../lib/librte_rcu -Ilib/librte_timer -I../lib/librte_timer -Ili

b/librte_cryptodev -I../lib/librte_cryptodev -Ilib/librte_pci

(quoted diff) +0 -7
-I../lib/librte_pci -Idrivers/bus/pci -I../drivers/bus/pci
-I../drivers/bus/pci/linux -Xclang -fcolor-diagnostics -pipe
-D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -Werror -O2 -g
-include rte_config.h -Wextra -Wcast-qual -Wdeprecated
-Wformat-nonliteral -Wformat-security -Wmissing-declarations
-Wmissing-prototypes -Wnested-externs -Wold-style-definition
-Wpointer-arith -Wsign-compare -Wstrict-prototypes -Wundef -Wwr

ite-strings -Wno-address-of-packed-member

(quoted diff) +0 -2
-Wno-missing-field-initializers -D_GNU_SOURCE -fPIC -march=native
-DALLOW_EXPERIMENTAL_API -DALLOW_INTERNAL_API -MD -MQ

drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -MF drivers/libtmp_rte_even
t_dlb.a.p/event_dlb_dlb.c.o.d -o drivers/libtmp_rte_event_dlb.a.p/event_dlb_dlb.c.o -c ../drivers/event/dlb/dlb.c ../drivers/event/dlb/dlb.c:2777:1: error: unused function 'dlb_event_enqueue_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2762:1: error: unused function 'dlb_event_enqueue_burst_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2792:1: error: unused function 'dlb_event_enqueue_new_burst_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_new_burst_delayed(void *event_port,
^
../drivers/event/dlb/dlb.c:2808:1: error: unused function 'dlb_event_enqueue_forward_burst_delayed' [-Werror,-Wunused-function] dlb_event_enqueue_forward_burst_delayed(void *event_port, ^
../drivers/event/dlb/dlb.c:2605:1: error: unused function

Quoted 9 lines

'dlb_construct_token_pop_qe' [-Werror,-Wunused-function] dlb_construct_token_pop_qe(struct dlb_port *qm_port, int idx) ^
../drivers/event/dlb/dlb.c:2653:1: error: unused function 'dlb_consume_qe_immediate' [-Werror,-Wunused-function] dlb_consume_qe_immediate(struct dlb_port *qm_port, int num) ^
6 errors generated.

Took care of these in V14. Hoping it’s the last of the errors.

#186 lore raw

v14 cover Add DLB PMD

In reply to McDaniel, Timothy

The following patch series adds support for a new eventdev PMD. The DLB PMD adds support for the Intel Dynamic Load Balancer (DLB) hardware. The DLB is a PCIe device that provides load-balanced, prioritized scheduling of core-to-core communication. The device consists of queues and arbiters that connect producer and consumer cores, and implements load-balanced queueing features including:
- Lock-free multi-producer/multi-consumer operation.
- Multiple priority levels for varying traffic types.
- 'Direct' traffic (i.e. multi-producer/single-consumer)
- Simple unordered load-balanced distribution.
- Atomic lock-free load balancing across multiple consumers.
- Queue element reordering feature allowing ordered load-balanced distribution.

The DLB hardware supports both load balanced and directed ports and queues. Unlike other eventdev devices already in the repo, not all DLB ports and queues are equally capable. In particular, directed ports are limited to a single link, and must be connected to a directed queue.
Additionally, even though LDB ports may link multiple queues, the number of queues that may be linked is limited by hardware. Another difference is that DLB does not have a straightforward way of carrying the flow_id in the queue elements (QE) that the hardware operates on.

While reviewing the code, please be aware that this PMD has full control over the DLB hardware. Intel will be extending the DLB PMD in the future (not as part of this first series) with a mode that we refer to as the bifurcated PMD. The bifurcated PMD communicates with a kernel driver to configure the device, ports, and queues, and memory maps device MMIO so datapath operations occur purely in user-space.

The framework to support both the PF PMD and bifurcated PMD exists in this patchset, and is why the iface.[ch] layer is present.

Major changes in V14
====================
- Fixed format errors in doc/api/doxy-api-index.md
- Delayed introduction of dlb2_consume_qe_immediate until add-dequeue-and-its-burst-variants.patch
- Delayed introduction of dlb2_construct_token_pop_qe until add-PMD-s-token-pop-public-interface.patch
- Delayed introduction of dlb_equeue_*_delayed until add dequeue and its burst variants.patch

Major changes in V13
====================
- removed now unused functions dlb_umwait and dlb_umonitor

Major changes in V12
====================
- Fix CENTOS build error: use __m128i instead of __v2di with _mm_stream_si128

Major changes in V11
====================
- removed unused function, fixing build error
- fixed typo in port_setup commit message
- this patch series is based on dpdk-next-eventdev

Major changes in v10
=====================
- convert to use rte_power_monitor patches
- replace __builtin_ia32_movntdq() with _mm_stream_si128()
- remove unused functions in dlb_selftest.c

Major changes in v9
=====================
- fixed a build error due to __rte_cache_aligned being placed after the ";" character, instead of before it.

Major changes in v8 after dpdk reviews
=====================
- moved introduction of dlb in relnotes_20_11 to first patch in series
- fixed underlines in dlb.rst that were too short
- note that the code still uses its private byte-encoded versions of umonitor/umwait, rather than the new functions in the power patch that are built on top of those intrinsics. This is intentional.

Major changes in v7 after dpdk reviews
=====================
- updated MAINTAINERS file to alphabetically insert DLB
- don't create RTE_ symbols in PMD
- converted to use version.map scheme
- converted to use .._master_lcore instead of .._main_lcore
- this patch set is based on dpdk-next-eventdev

Major changes in v6 after dpdk reviews:
=====================
- fixed meson conditional build. Moved test into driver’s meson.build file instead of event/meson.build
- documentation is populated as associated code is introduced
- add log_register in add dynamic logging patch
- rename RTE_xxx symbol(s) as DLB2_xxx
- replaced function ptr enqueue_four with direct call to movdir64b
- remove unused port_pages
- broke up probe patch into 3 smaller patches for easier review
- changed param order of movdir64b/movntdq to match intrinsics
- added self to MAINTAINERS files
- squashed announcement of availability into last patch in series
- correct spelling errors and delete repeated words
- DPDK_21.0 -> DPDK 21 in map file
- add experimental banner to public structs and APIs
- implemented other suggestions from code reviews of DLB2 PMD. The software is very similar in form so some DLB2 reviews comments were applicable to DLB as well

Major changes in v5 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- implement changes requested in code reviews by Gage Eads and Mike Chen
- fix a memzone leak
- convert to use eal rte-cpuflags patch from Liang Ma

Major changes in v4 after dpdk reviews and additional internal reviews by colleagues at Intel:
================
- Remove make infrastructure
- shared code (pf/base) is now added incrementally
- flexible interface (iface.[ch]) is now added incrementally
- removed calls to rte_panic
- do not call pthread_create directly
- remove unused internal API, os_time
- convert rte_atomic to __atomic builtins
- broke out eventdev ABI changes, test/api changes, and new internal PCI named probe API
- relocated enqueue logic to enqueue patch

Major Changes in V3:
================
- Fixed a memory corruption issue due to not allocating enough CQ memory for depths < 8. Hardware requires minimum allocation to be at least 8 entries.
- Address review comments from Gage and Mattias.
- Remove versioning
- minor formatting changes

Major changes in V2:
================
- Correct ABI break that was present in V1.
- Address some of the review comments received from Mattias. I will address the remaining items identified by Mattias in the next patch delivery.
- General code cleanup based on internal code reviews

Depends-on: patch-82202 ("eventdev: increase MAX QUEUES PER DEV to 255")

Timothy McDaniel (23):
event/dlb: add documentation and meson infrastructure event/dlb: add dynamic logging
event/dlb: add private data structures and constants event/dlb: add definitions shared with LKM or shared code event/dlb: add inline functions
event/dlb: add eventdev probe
event/dlb: add flexible interface
event/dlb: add probe-time hardware init event/dlb: add xstats
event/dlb: add infos get and configure event/dlb: add queue and port default conf event/dlb: add queue setup
event/dlb: add port setup
event/dlb: add port link
event/dlb: add port unlink and port unlinks in progress event/dlb: add eventdev start
event/dlb: add enqueue and its burst variants event/dlb: add dequeue and its burst variants event/dlb: add eventdev stop and close event/dlb: add PMD's token pop public interface event/dlb: add PMD self-tests
event/dlb: add queue and port release
event/dlb: add timeout ticks entry point

 MAINTAINERS                                  |    6 +-
 app/test/test_eventdev.c                     |    7 +
 config/rte_config.h                          |    6 +
 doc/api/doxy-api-index.md                    |    3 +-
 doc/guides/eventdevs/dlb.rst                 |  341 ++
 doc/guides/eventdevs/index.rst               |    1 +
 doc/guides/rel_notes/release_20_11.rst       |    5 +
 drivers/event/dlb/dlb.c                      | 4079 +++++++++++++++
 drivers/event/dlb/dlb_iface.c                |   79 +
 drivers/event/dlb/dlb_iface.h                |   82 +
 drivers/event/dlb/dlb_inline_fns.h           |   40 +
 drivers/event/dlb/dlb_log.h                  |   25 +
 drivers/event/dlb/dlb_priv.h                 |  513 ++
 drivers/event/dlb/dlb_selftest.c             | 1539 ++++++
 drivers/event/dlb/dlb_user.h                 |  814 +++
 drivers/event/dlb/dlb_xstats.c               | 1222 +++++
 drivers/event/dlb/meson.build                |   21 +
 drivers/event/dlb/pf/base/dlb_hw_types.h     |  334 ++
 drivers/event/dlb/pf/base/dlb_osdep.h        |  310 ++
 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h |  441 ++
 drivers/event/dlb/pf/base/dlb_osdep_list.h   |  131 +
 drivers/event/dlb/pf/base/dlb_osdep_types.h  |   31 +
 drivers/event/dlb/pf/base/dlb_regs.h         | 2368 +++++++++
 drivers/event/dlb/pf/base/dlb_resource.c     | 6904 ++++++++++++++++++++++++++
 drivers/event/dlb/pf/base/dlb_resource.h     |  876 ++++
 drivers/event/dlb/pf/dlb_main.c              |  586 +++
 drivers/event/dlb/pf/dlb_main.h              |   47 +
 drivers/event/dlb/pf/dlb_pf.c                |  750 +++
 drivers/event/dlb/rte_pmd_dlb.c              |   38 +
 drivers/event/dlb/rte_pmd_dlb.h              |   77 +
 drivers/event/dlb/version.map                |    9 +
 drivers/event/meson.build                    |    2 +-
 32 files changed, 21684 insertions(+), 3 deletions(-)
 create mode 100644 doc/guides/eventdevs/dlb.rst
 create mode 100644 drivers/event/dlb/dlb.c
 create mode 100644 drivers/event/dlb/dlb_iface.c
 create mode 100644 drivers/event/dlb/dlb_iface.h
 create mode 100644 drivers/event/dlb/dlb_inline_fns.h
 create mode 100644 drivers/event/dlb/dlb_log.h
 create mode 100644 drivers/event/dlb/dlb_priv.h
 create mode 100644 drivers/event/dlb/dlb_selftest.c
 create mode 100644 drivers/event/dlb/dlb_user.h
 create mode 100644 drivers/event/dlb/dlb_xstats.c
 create mode 100644 drivers/event/dlb/meson.build
 create mode 100644 drivers/event/dlb/pf/base/dlb_hw_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_bitmap.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_list.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_osdep_types.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_regs.h
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.c
 create mode 100644 drivers/event/dlb/pf/base/dlb_resource.h
 create mode 100644 drivers/event/dlb/pf/dlb_main.c
 create mode 100644 drivers/event/dlb/pf/dlb_main.h
 create mode 100644 drivers/event/dlb/pf/dlb_pf.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.c
 create mode 100644 drivers/event/dlb/rte_pmd_dlb.h
 create mode 100644 drivers/event/dlb/version.map
Signature · 1 line
2.6.4