Introduce Microsoft Azure Network Adatper (MANA) PMD

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB).
Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

drivers/net/mana/mana.c new file +188 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv __rte_unused,
+			      struct rte_pci_device *pci_dev,
+			      struct rte_ether_addr *mac_addr)
+{
+	struct ibv_device **ibv_list;
+	int ibv_idx;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex dev_attr;
+	int num_devices;
+	int ret = 0;
+	uint8_t port;
+	struct mana_priv *priv = NULL;
+	struct rte_eth_dev *eth_dev = NULL;
+	bool found_port;
+
+	ibv_list = ibv_get_device_list(&num_devices);
+	for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+		struct ibv_device *ibdev = ibv_list[ibv_idx];
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(INFO, "Probe device name %s dev_name %s ibdev_path %s",
+			ibdev->name, ibdev->dev_name, ibdev->ibdev_path);
+
+		if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+			continue;
+
+		/* Ignore if this IB device is not this PCI device */
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+
+		ctx = ibv_open_device(ibdev);
+		if (!ctx) {
+			DRV_LOG(ERR, "Failed to open IB device %s",
+				ibdev->name);
+			continue;
+		}
+
+		ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+		DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+			dev_attr.orig_attr.phys_port_cnt);
+		found_port = false;
+
+		for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+		     port++) {
+			struct ibv_parent_domain_init_attr attr = {};
+			struct rte_ether_addr addr;
+			char address[64];
+			char name[RTE_ETH_NAME_MAX_LEN];
+
+			ret = get_port_mac(ibdev, port, &addr);
+			if (ret)
+				continue;
+
+			if (mac_addr && !rte_is_same_ether_addr(&addr, mac_addr))
+				continue;
+
+			rte_ether_format_addr(address, sizeof(address), &addr);
+			DRV_LOG(INFO, "device located port %u address %s",
+				port, address);
+			found_port = true;
+
+			priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+						  RTE_CACHE_LINE_SIZE,
+						  SOCKET_ID_ANY);
+			if (!priv) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			snprintf(name, sizeof(name), "%s_port%d",
+				 pci_dev->device.name, port);
+
+			if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
+				int fd;
+
+				eth_dev = rte_eth_dev_attach_secondary(name);
+				if (!eth_dev) {
+					DRV_LOG(ERR, "Can't attach to dev %s",
+						name);
+					ret = -ENOMEM;
+					goto failed;
+				}
+
+				eth_dev->device = &pci_dev->device;
+				eth_dev->dev_ops = &mana_dev_sec_ops;
+				ret = mana_proc_priv_init(eth_dev);
+				if (ret)
+					goto failed;
+				priv->process_priv = eth_dev->process_private;
+
+				/* Get the IB FD from the primary process */
+				fd = mana_mp_req_verbs_cmd_fd(eth_dev);
+				if (fd < 0) {
+					DRV_LOG(ERR, "Failed to get FD %d", fd);
+					ret = -ENODEV;
+					goto failed;
+				}
+
+				ret = mana_map_doorbell_secondary(eth_dev, fd);
+				if (ret) {
+					DRV_LOG(ERR, "Failed secondary map %d",
+						fd);
+					goto failed;
+				}
+
+				/* fd is no not used after mapping doorbell */
+				close(fd);
+
+				rte_spinlock_lock(&mana_shared_data->lock);
+				mana_shared_data->secondary_cnt++;
+				mana_local_data.secondary_cnt++;
+				rte_spinlock_unlock(&mana_shared_data->lock);
+
+				rte_eth_copy_pci_info(eth_dev, pci_dev);
+				rte_eth_dev_probing_finish(eth_dev);
+
+				/* Impossible to have more than one port
+				 * matching a MAC address
+				 */
+				continue;
+			}
+
+			eth_dev = rte_eth_dev_allocate(name);
+			if (!eth_dev) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			eth_dev->data->mac_addrs =
+				rte_calloc("mana_mac", 1,
+					   sizeof(struct rte_ether_addr), 0);
+			if (!eth_dev->data->mac_addrs) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			rte_ether_addr_copy(&addr, eth_dev->data->mac_addrs);
+
+			priv->ib_pd = ibv_alloc_pd(ctx);
+			if (!priv->ib_pd) {
+				DRV_LOG(ERR, "ibv_alloc_pd failed port %d", port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			/* Create a parent domain with the port number */
+			attr.pd = priv->ib_pd;
+			attr.comp_mask = IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;
+			attr.pd_context = (void *)(uint64_t)port;
+			priv->ib_parent_pd = ibv_alloc_parent_domain(ctx, &attr);
+			if (!priv->ib_parent_pd) {
+				DRV_LOG(ERR,
+					"ibv_alloc_parent_domain failed port %d",
+					port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			priv->ib_ctx = ctx;
+			priv->port_id = eth_dev->data->port_id;
+			priv->dev_port = port;
+			eth_dev->data->dev_private = priv;
+			priv->dev_data = eth_dev->data;
+
+			priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+			priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+			priv->max_rx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+					dev_attr.orig_attr.max_cqe);
+			priv->max_tx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+					dev_attr.orig_attr.max_cqe);
+
+			priv->max_send_sge = dev_attr.orig_attr.max_sge;
+			priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+			priv->max_mr = dev_attr.orig_attr.max_mr;
+			priv->max_mr_size = dev_attr.orig_attr.max_mr_size;
+
+			DRV_LOG(INFO, "dev %s max queues %d desc %d sge %d",
+				name, priv->max_rx_queues, priv->max_rx_desc,
+				priv->max_send_sge);
+

drivers/net/mana/mana.c new file +43 -0

+			DRV_LOG(INFO, "device %s at port %u",
+				name, eth_dev->data->port_id);
+
+			eth_dev->rx_pkt_burst = mana_rx_burst_removed;
+			eth_dev->tx_pkt_burst = mana_tx_burst_removed;
+			eth_dev->dev_ops = &mana_dev_ops;
+
+			rte_eth_copy_pci_info(eth_dev, pci_dev);
+			rte_eth_dev_probing_finish(eth_dev);
+		}
+
+		/* Secondary process doesn't need an ibv_ctx. It maps the
+		 * doorbell pages using the IB cmd_fd passed from the primary
+		 * process and send messages to primary process for memory
+		 * registartions.
+		 */
+		if (!found_port || rte_eal_process_type() == RTE_PROC_SECONDARY)
+			ibv_close_device(ctx);
+	}
+
+	ibv_free_device_list(ibv_list);
+	return 0;
+
+failed:
+	/* Free the resource for the port failed */
+	if (priv) {
+		if (priv->ib_parent_pd)
+			ibv_dealloc_pd(priv->ib_parent_pd);
+
+		if (priv->ib_pd)
+			ibv_dealloc_pd(priv->ib_pd);
+	}
+
+	if (eth_dev)
+		rte_eth_dev_release_port(eth_dev);
+
+	rte_free(priv);
+
+	ibv_close_device(ctx);
+	ibv_free_device_list(ibv_list);
+
+	return ret;
+}
[snip]

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 23 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 9 lines

drivers/net/mana/mana.c new file +9 -0

+			      struct rte_pci_device *pci_dev,
+			      struct rte_ether_addr *mac_addr) {
+	struct ibv_device **ibv_list;
+	int ibv_idx;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex dev_attr;
+	int num_devices;
+	int ret = 0;
+	uint8_t port;

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+	struct mana_priv *priv = NULL;
+	struct rte_eth_dev *eth_dev = NULL;
+	bool found_port;
+
+	ibv_list = ibv_get_device_list(&num_devices);
+	for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+		struct ibv_device *ibdev = ibv_list[ibv_idx];
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(INFO, "Probe device name %s dev_name %s

ibdev_path %s",

Quoted 27 lines

(quoted diff) +1 -0

+			ibdev->name, ibdev->dev_name, ibdev-

ibdev_path);

drivers/net/mana/mana.c new file +25 -0

+
+		if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+			continue;
+
+		/* Ignore if this IB device is not this PCI device */
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+
+		ctx = ibv_open_device(ibdev);
+		if (!ctx) {
+			DRV_LOG(ERR, "Failed to open IB device %s",
+				ibdev->name);
+			continue;
+		}
+
+		ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+		DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+			dev_attr.orig_attr.phys_port_cnt);
+		found_port = false;
+
+		for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+		     port++) {

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+			struct ibv_parent_domain_init_attr attr = {};
+			struct rte_ether_addr addr;
+			char address[64];
+			char name[RTE_ETH_NAME_MAX_LEN];
+
+			ret = get_port_mac(ibdev, port, &addr);
+			if (ret)
+				continue;
+
+			if (mac_addr && !rte_is_same_ether_addr(&addr,

mac_addr))

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				continue;
+
+			rte_ether_format_addr(address, sizeof(address),

&addr);

Review

Quoted 1 line

(quoted diff) +1 -0

+			DRV_LOG(INFO, "device located port %u address

%s",

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+				port, address);
+			found_port = true;
+
+			priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+						  RTE_CACHE_LINE_SIZE,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+						  SOCKET_ID_ANY);
+			if (!priv) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			snprintf(name, sizeof(name), "%s_port%d",
+				 pci_dev->device.name, port);
+
+			if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				int fd;
+
+				eth_dev =

rte_eth_dev_attach_secondary(name);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (!eth_dev) {
+					DRV_LOG(ERR, "Can't attach to dev

%s",

Quoted 12 lines

drivers/net/mana/mana.c new file +11 -0

+						name);
+					ret = -ENOMEM;
+					goto failed;
+				}
+
+				eth_dev->device = &pci_dev->device;
+				eth_dev->dev_ops = &mana_dev_sec_ops;
+				ret = mana_proc_priv_init(eth_dev);
+				if (ret)
+					goto failed;
+				priv->process_priv = eth_dev-

process_private;

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+
+				/* Get the IB FD from the primary process */
+				fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (fd < 0) {
+					DRV_LOG(ERR, "Failed to get FD %d",

fd);

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+					ret = -ENODEV;
+					goto failed;
+				}
+
+				ret =

mana_map_doorbell_secondary(eth_dev, fd);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (ret) {
+					DRV_LOG(ERR, "Failed secondary

map %d",

Quoted 39 lines

drivers/net/mana/mana.c new file +8 -0

+						fd);
+					goto failed;
+				}
+
+				/* fd is no not used after mapping doorbell */
+				close(fd);
+
+				rte_spinlock_lock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +3 -0

+				mana_shared_data->secondary_cnt++;
+				mana_local_data.secondary_cnt++;
+				rte_spinlock_unlock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +25 -0

+
+				rte_eth_copy_pci_info(eth_dev, pci_dev);
+				rte_eth_dev_probing_finish(eth_dev);
+
+				/* Impossible to have more than one port
+				 * matching a MAC address
+				 */
+				continue;
+			}
+
+			eth_dev = rte_eth_dev_allocate(name);
+			if (!eth_dev) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			eth_dev->data->mac_addrs =
+				rte_calloc("mana_mac", 1,
+					   sizeof(struct rte_ether_addr), 0);
+			if (!eth_dev->data->mac_addrs) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			rte_ether_addr_copy(&addr, eth_dev->data-

mac_addrs);

drivers/net/mana/mana.c

Quoted 4 lines

drivers/net/mana/mana.c new file +4 -0

+
+			priv->ib_pd = ibv_alloc_pd(ctx);
+			if (!priv->ib_pd) {
+				DRV_LOG(ERR, "ibv_alloc_pd failed port %d",

port);

drivers/net/mana/mana.c

Quoted 7 lines

drivers/net/mana/mana.c new file +7 -0

+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			/* Create a parent domain with the port number */
+			attr.pd = priv->ib_pd;
+			attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Review

Quoted 2 lines

(quoted diff) +2 -0

+			attr.pd_context = (void *)(uint64_t)port;
+			priv->ib_parent_pd = ibv_alloc_parent_domain(ctx,

&attr);

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+			if (!priv->ib_parent_pd) {
+				DRV_LOG(ERR,
+					"ibv_alloc_parent_domain failed port

%d",

Quoted 16 lines

drivers/net/mana/mana.c new file +16 -0

+					port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			priv->ib_ctx = ctx;
+			priv->port_id = eth_dev->data->port_id;
+			priv->dev_port = port;
+			eth_dev->data->dev_private = priv;
+			priv->dev_data = eth_dev->data;
+
+			priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+			priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+			priv->max_rx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+					dev_attr.orig_attr.max_cqe);
+			priv->max_tx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+					dev_attr.orig_attr.max_cqe);
+
+			priv->max_send_sge = dev_attr.orig_attr.max_sge;
+			priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+			priv->max_mr = dev_attr.orig_attr.max_mr;
+			priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+			DRV_LOG(INFO, "dev %s max queues %d desc %d

sge %d",

drivers/net/mana/mana.c line 545

Quoted 9 lines

(quoted diff) +1 -0

+				name, priv->max_rx_queues, priv-

max_rx_desc,

drivers/net/mana/mana.c new file +7 -0

+				priv->max_send_sge);
+
+			rte_spinlock_lock(&mana_shared_data->lock);
+			mana_shared_data->primary_cnt++;
+			rte_spinlock_unlock(&mana_shared_data->lock);
+
+			eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

drivers/net/mana/mana.c line 547

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+
+			eth_dev->device = &pci_dev->device;
+			eth_dev->data->dev_flags |=
+				RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+

Please do not use the temporary macro. Please review this patch:

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Quoted 254 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 23 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 9 lines

drivers/net/mana/mana.c new file +9 -0

+			      struct rte_pci_device *pci_dev,
+			      struct rte_ether_addr *mac_addr) {
+	struct ibv_device **ibv_list;
+	int ibv_idx;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex dev_attr;
+	int num_devices;
+	int ret = 0;
+	uint8_t port;

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+	struct mana_priv *priv = NULL;
+	struct rte_eth_dev *eth_dev = NULL;
+	bool found_port;
+
+	ibv_list = ibv_get_device_list(&num_devices);
+	for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+		struct ibv_device *ibdev = ibv_list[ibv_idx];
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(INFO, "Probe device name %s dev_name %s

ibdev_path %s",

Quoted 27 lines

(quoted diff) +1 -0

+			ibdev->name, ibdev->dev_name, ibdev-

ibdev_path);

drivers/net/mana/mana.c new file +25 -0

+
+		if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+			continue;
+
+		/* Ignore if this IB device is not this PCI device */
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+
+		ctx = ibv_open_device(ibdev);
+		if (!ctx) {
+			DRV_LOG(ERR, "Failed to open IB device %s",
+				ibdev->name);
+			continue;
+		}
+
+		ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+		DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+			dev_attr.orig_attr.phys_port_cnt);
+		found_port = false;
+
+		for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+		     port++) {

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+			struct ibv_parent_domain_init_attr attr = {};
+			struct rte_ether_addr addr;
+			char address[64];
+			char name[RTE_ETH_NAME_MAX_LEN];
+
+			ret = get_port_mac(ibdev, port, &addr);
+			if (ret)
+				continue;
+
+			if (mac_addr && !rte_is_same_ether_addr(&addr,

mac_addr))

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				continue;
+
+			rte_ether_format_addr(address, sizeof(address),

&addr);

Review

Quoted 1 line

(quoted diff) +1 -0

+			DRV_LOG(INFO, "device located port %u address

%s",

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+				port, address);
+			found_port = true;
+
+			priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+						  RTE_CACHE_LINE_SIZE,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+						  SOCKET_ID_ANY);
+			if (!priv) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			snprintf(name, sizeof(name), "%s_port%d",
+				 pci_dev->device.name, port);
+
+			if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				int fd;
+
+				eth_dev =

rte_eth_dev_attach_secondary(name);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (!eth_dev) {
+					DRV_LOG(ERR, "Can't attach to dev

%s",

Quoted 12 lines

drivers/net/mana/mana.c new file +11 -0

+						name);
+					ret = -ENOMEM;
+					goto failed;
+				}
+
+				eth_dev->device = &pci_dev->device;
+				eth_dev->dev_ops = &mana_dev_sec_ops;
+				ret = mana_proc_priv_init(eth_dev);
+				if (ret)
+					goto failed;
+				priv->process_priv = eth_dev-

process_private;

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+
+				/* Get the IB FD from the primary process */
+				fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (fd < 0) {
+					DRV_LOG(ERR, "Failed to get FD %d",

fd);

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+					ret = -ENODEV;
+					goto failed;
+				}
+
+				ret =

mana_map_doorbell_secondary(eth_dev, fd);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (ret) {
+					DRV_LOG(ERR, "Failed secondary

map %d",

Quoted 39 lines

drivers/net/mana/mana.c new file +8 -0

+						fd);
+					goto failed;
+				}
+
+				/* fd is no not used after mapping doorbell */
+				close(fd);
+
+				rte_spinlock_lock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +3 -0

+				mana_shared_data->secondary_cnt++;
+				mana_local_data.secondary_cnt++;
+				rte_spinlock_unlock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +25 -0

+
+				rte_eth_copy_pci_info(eth_dev, pci_dev);
+				rte_eth_dev_probing_finish(eth_dev);
+
+				/* Impossible to have more than one port
+				 * matching a MAC address
+				 */
+				continue;
+			}
+
+			eth_dev = rte_eth_dev_allocate(name);
+			if (!eth_dev) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			eth_dev->data->mac_addrs =
+				rte_calloc("mana_mac", 1,
+					   sizeof(struct rte_ether_addr), 0);
+			if (!eth_dev->data->mac_addrs) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			rte_ether_addr_copy(&addr, eth_dev->data-

mac_addrs);

drivers/net/mana/mana.c

Quoted 4 lines

drivers/net/mana/mana.c new file +4 -0

+
+			priv->ib_pd = ibv_alloc_pd(ctx);
+			if (!priv->ib_pd) {
+				DRV_LOG(ERR, "ibv_alloc_pd failed port %d",

port);

drivers/net/mana/mana.c

Quoted 7 lines

drivers/net/mana/mana.c new file +7 -0

+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			/* Create a parent domain with the port number */
+			attr.pd = priv->ib_pd;
+			attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Review

Quoted 2 lines

(quoted diff) +2 -0

+			attr.pd_context = (void *)(uint64_t)port;
+			priv->ib_parent_pd = ibv_alloc_parent_domain(ctx,

&attr);

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+			if (!priv->ib_parent_pd) {
+				DRV_LOG(ERR,
+					"ibv_alloc_parent_domain failed port

%d",

Quoted 16 lines

drivers/net/mana/mana.c new file +16 -0

+					port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			priv->ib_ctx = ctx;
+			priv->port_id = eth_dev->data->port_id;
+			priv->dev_port = port;
+			eth_dev->data->dev_private = priv;
+			priv->dev_data = eth_dev->data;
+
+			priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+			priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+			priv->max_rx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+					dev_attr.orig_attr.max_cqe);
+			priv->max_tx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+					dev_attr.orig_attr.max_cqe);
+
+			priv->max_send_sge = dev_attr.orig_attr.max_sge;
+			priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+			priv->max_mr = dev_attr.orig_attr.max_mr;
+			priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+			DRV_LOG(INFO, "dev %s max queues %d desc %d

sge %d",

drivers/net/mana/mana.c line 545

Quoted 9 lines

(quoted diff) +1 -0

+				name, priv->max_rx_queues, priv-

max_rx_desc,

drivers/net/mana/mana.c new file +7 -0

+				priv->max_send_sge);
+
+			rte_spinlock_lock(&mana_shared_data->lock);
+			mana_shared_data->primary_cnt++;
+			rte_spinlock_unlock(&mana_shared_data->lock);
+
+			eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

drivers/net/mana/mana.c line 547

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+
+			eth_dev->device = &pci_dev->device;
+			eth_dev->data->dev_flags |=
+				RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+

Please do not use the temporary macro. Please review this patch:

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

Quoted 2 lines

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Thanks for pointing this out.

It seems some PMDs are still depending on this flag for xstats.

MANA doesn't implement xstats_get() currently, this flag is useful. Is it okay to keep using this flag before it's finally the time to remove it from all PMDs, or when MANA implements xstats?

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 261 lines · Long Li

Quoted 252 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 23 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 9 lines

drivers/net/mana/mana.c new file +9 -0

+			      struct rte_pci_device *pci_dev,
+			      struct rte_ether_addr *mac_addr) {
+	struct ibv_device **ibv_list;
+	int ibv_idx;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex dev_attr;
+	int num_devices;
+	int ret = 0;
+	uint8_t port;

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+	struct mana_priv *priv = NULL;
+	struct rte_eth_dev *eth_dev = NULL;
+	bool found_port;
+
+	ibv_list = ibv_get_device_list(&num_devices);
+	for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+		struct ibv_device *ibdev = ibv_list[ibv_idx];
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(INFO, "Probe device name %s dev_name %s

ibdev_path %s",

Quoted 27 lines

(quoted diff) +1 -0

+			ibdev->name, ibdev->dev_name, ibdev-

ibdev_path);

drivers/net/mana/mana.c new file +25 -0

+
+		if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+			continue;
+
+		/* Ignore if this IB device is not this PCI device */
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+
+		ctx = ibv_open_device(ibdev);
+		if (!ctx) {
+			DRV_LOG(ERR, "Failed to open IB device %s",
+				ibdev->name);
+			continue;
+		}
+
+		ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+		DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+			dev_attr.orig_attr.phys_port_cnt);
+		found_port = false;
+
+		for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+		     port++) {

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+			struct ibv_parent_domain_init_attr attr = {};
+			struct rte_ether_addr addr;
+			char address[64];
+			char name[RTE_ETH_NAME_MAX_LEN];
+
+			ret = get_port_mac(ibdev, port, &addr);
+			if (ret)
+				continue;
+
+			if (mac_addr && !rte_is_same_ether_addr(&addr,

mac_addr))

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				continue;
+
+			rte_ether_format_addr(address, sizeof(address),

&addr);

Review

Quoted 1 line

(quoted diff) +1 -0

+			DRV_LOG(INFO, "device located port %u address

%s",

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+				port, address);
+			found_port = true;
+
+			priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+						  RTE_CACHE_LINE_SIZE,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+						  SOCKET_ID_ANY);
+			if (!priv) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			snprintf(name, sizeof(name), "%s_port%d",
+				 pci_dev->device.name, port);
+
+			if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				int fd;
+
+				eth_dev =

rte_eth_dev_attach_secondary(name);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (!eth_dev) {
+					DRV_LOG(ERR, "Can't attach to dev

%s",

Quoted 12 lines

drivers/net/mana/mana.c new file +11 -0

+						name);
+					ret = -ENOMEM;
+					goto failed;
+				}
+
+				eth_dev->device = &pci_dev->device;
+				eth_dev->dev_ops = &mana_dev_sec_ops;
+				ret = mana_proc_priv_init(eth_dev);
+				if (ret)
+					goto failed;
+				priv->process_priv = eth_dev-

process_private;

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+
+				/* Get the IB FD from the primary process */
+				fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (fd < 0) {
+					DRV_LOG(ERR, "Failed to get FD %d",

fd);

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+					ret = -ENODEV;
+					goto failed;
+				}
+
+				ret =

mana_map_doorbell_secondary(eth_dev, fd);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (ret) {
+					DRV_LOG(ERR, "Failed secondary

map %d",

Quoted 39 lines

drivers/net/mana/mana.c new file +8 -0

+						fd);
+					goto failed;
+				}
+
+				/* fd is no not used after mapping doorbell */
+				close(fd);
+
+				rte_spinlock_lock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +3 -0

+				mana_shared_data->secondary_cnt++;
+				mana_local_data.secondary_cnt++;
+				rte_spinlock_unlock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +25 -0

+
+				rte_eth_copy_pci_info(eth_dev, pci_dev);
+				rte_eth_dev_probing_finish(eth_dev);
+
+				/* Impossible to have more than one port
+				 * matching a MAC address
+				 */
+				continue;
+			}
+
+			eth_dev = rte_eth_dev_allocate(name);
+			if (!eth_dev) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			eth_dev->data->mac_addrs =
+				rte_calloc("mana_mac", 1,
+					   sizeof(struct rte_ether_addr), 0);
+			if (!eth_dev->data->mac_addrs) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			rte_ether_addr_copy(&addr, eth_dev->data-

mac_addrs);

drivers/net/mana/mana.c

Quoted 4 lines

drivers/net/mana/mana.c new file +4 -0

+
+			priv->ib_pd = ibv_alloc_pd(ctx);
+			if (!priv->ib_pd) {
+				DRV_LOG(ERR, "ibv_alloc_pd failed port %d",

port);

drivers/net/mana/mana.c

Quoted 7 lines

drivers/net/mana/mana.c new file +7 -0

+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			/* Create a parent domain with the port number */
+			attr.pd = priv->ib_pd;
+			attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Review

Quoted 2 lines

(quoted diff) +2 -0

+			attr.pd_context = (void *)(uint64_t)port;
+			priv->ib_parent_pd = ibv_alloc_parent_domain(ctx,

&attr);

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+			if (!priv->ib_parent_pd) {
+				DRV_LOG(ERR,
+					"ibv_alloc_parent_domain failed port

%d",

Quoted 16 lines

drivers/net/mana/mana.c new file +16 -0

+					port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			priv->ib_ctx = ctx;
+			priv->port_id = eth_dev->data->port_id;
+			priv->dev_port = port;
+			eth_dev->data->dev_private = priv;
+			priv->dev_data = eth_dev->data;
+
+			priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+			priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+			priv->max_rx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+					dev_attr.orig_attr.max_cqe);
+			priv->max_tx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+					dev_attr.orig_attr.max_cqe);
+
+			priv->max_send_sge = dev_attr.orig_attr.max_sge;
+			priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+			priv->max_mr = dev_attr.orig_attr.max_mr;
+			priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+			DRV_LOG(INFO, "dev %s max queues %d desc %d

sge %d",

drivers/net/mana/mana.c line 545

Quoted 9 lines

(quoted diff) +1 -0

+				name, priv->max_rx_queues, priv-

max_rx_desc,

drivers/net/mana/mana.c new file +7 -0

+				priv->max_send_sge);
+
+			rte_spinlock_lock(&mana_shared_data->lock);
+			mana_shared_data->primary_cnt++;
+			rte_spinlock_unlock(&mana_shared_data->lock);
+
+			eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

drivers/net/mana/mana.c line 547

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+
+			eth_dev->device = &pci_dev->device;
+			eth_dev->data->dev_flags |=
+				RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+

Please do not use the temporary macro. Please review this patch:

Quoted 5 lines

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Thanks for pointing this out.

It seems some PMDs are still depending on this flag for xstats.

Quoted 1 line

MANA doesn't implement xstats_get() currently, this flag is useful. Is it

okay to keep using this flag before it's finally the time to remove it from all PMDs, or when MANA implements xstats?
Yes, your xstats doesn't implement now. Per queue stats should be filled in xstats API, and the stats API cannot see per queue stats, so stats API in driver shouldn't fill it(suggest that delete it from patch 17/18).

I guess this flag can be removed if PMD does not support xstats.

Quoted 2 lines

Quoted 273 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 255 lines · Long Li

Quoted 255 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 23 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 9 lines

drivers/net/mana/mana.c new file +9 -0

+			      struct rte_pci_device *pci_dev,
+			      struct rte_ether_addr *mac_addr) {
+	struct ibv_device **ibv_list;
+	int ibv_idx;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex dev_attr;
+	int num_devices;
+	int ret = 0;
+	uint8_t port;

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+	struct mana_priv *priv = NULL;
+	struct rte_eth_dev *eth_dev = NULL;
+	bool found_port;
+
+	ibv_list = ibv_get_device_list(&num_devices);
+	for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+		struct ibv_device *ibdev = ibv_list[ibv_idx];
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(INFO, "Probe device name %s dev_name %s

ibdev_path %s",

Quoted 27 lines

(quoted diff) +1 -0

+			ibdev->name, ibdev->dev_name, ibdev-

ibdev_path);

drivers/net/mana/mana.c new file +25 -0

+
+		if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+			continue;
+
+		/* Ignore if this IB device is not this PCI device */
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+
+		ctx = ibv_open_device(ibdev);
+		if (!ctx) {
+			DRV_LOG(ERR, "Failed to open IB device %s",
+				ibdev->name);
+			continue;
+		}
+
+		ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+		DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+			dev_attr.orig_attr.phys_port_cnt);
+		found_port = false;
+
+		for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+		     port++) {

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+			struct ibv_parent_domain_init_attr attr = {};
+			struct rte_ether_addr addr;
+			char address[64];
+			char name[RTE_ETH_NAME_MAX_LEN];
+
+			ret = get_port_mac(ibdev, port, &addr);
+			if (ret)
+				continue;
+
+			if (mac_addr && !rte_is_same_ether_addr(&addr,

mac_addr))

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				continue;
+
+			rte_ether_format_addr(address, sizeof(address),

&addr);

Review

Quoted 1 line

(quoted diff) +1 -0

+			DRV_LOG(INFO, "device located port %u address

%s",

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+				port, address);
+			found_port = true;
+
+			priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+						  RTE_CACHE_LINE_SIZE,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+						  SOCKET_ID_ANY);
+			if (!priv) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			snprintf(name, sizeof(name), "%s_port%d",
+				 pci_dev->device.name, port);
+
+			if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				int fd;
+
+				eth_dev =

rte_eth_dev_attach_secondary(name);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (!eth_dev) {
+					DRV_LOG(ERR, "Can't attach to dev

%s",

Quoted 12 lines

drivers/net/mana/mana.c new file +11 -0

+						name);
+					ret = -ENOMEM;
+					goto failed;
+				}
+
+				eth_dev->device = &pci_dev->device;
+				eth_dev->dev_ops = &mana_dev_sec_ops;
+				ret = mana_proc_priv_init(eth_dev);
+				if (ret)
+					goto failed;
+				priv->process_priv = eth_dev-

process_private;

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+
+				/* Get the IB FD from the primary process */
+				fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (fd < 0) {
+					DRV_LOG(ERR, "Failed to get FD %d",

fd);

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+					ret = -ENODEV;
+					goto failed;
+				}
+
+				ret =

mana_map_doorbell_secondary(eth_dev, fd);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (ret) {
+					DRV_LOG(ERR, "Failed secondary

map %d",

Quoted 39 lines

drivers/net/mana/mana.c new file +8 -0

+						fd);
+					goto failed;
+				}
+
+				/* fd is no not used after mapping doorbell */
+				close(fd);
+
+				rte_spinlock_lock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +3 -0

+				mana_shared_data->secondary_cnt++;
+				mana_local_data.secondary_cnt++;
+				rte_spinlock_unlock(&mana_shared_data-

lock);

drivers/net/mana/mana.c new file +25 -0

+
+				rte_eth_copy_pci_info(eth_dev, pci_dev);
+				rte_eth_dev_probing_finish(eth_dev);
+
+				/* Impossible to have more than one port
+				 * matching a MAC address
+				 */
+				continue;
+			}
+
+			eth_dev = rte_eth_dev_allocate(name);
+			if (!eth_dev) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			eth_dev->data->mac_addrs =
+				rte_calloc("mana_mac", 1,
+					   sizeof(struct rte_ether_addr), 0);
+			if (!eth_dev->data->mac_addrs) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			rte_ether_addr_copy(&addr, eth_dev->data-

mac_addrs);

drivers/net/mana/mana.c

Quoted 4 lines

drivers/net/mana/mana.c new file +4 -0

+
+			priv->ib_pd = ibv_alloc_pd(ctx);
+			if (!priv->ib_pd) {
+				DRV_LOG(ERR, "ibv_alloc_pd failed port %d",

port);

drivers/net/mana/mana.c

Quoted 7 lines

drivers/net/mana/mana.c new file +7 -0

+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			/* Create a parent domain with the port number */
+			attr.pd = priv->ib_pd;
+			attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Review

Quoted 2 lines

(quoted diff) +2 -0

+			attr.pd_context = (void *)(uint64_t)port;
+			priv->ib_parent_pd = ibv_alloc_parent_domain(ctx,

&attr);

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+			if (!priv->ib_parent_pd) {
+				DRV_LOG(ERR,
+					"ibv_alloc_parent_domain failed port

%d",

Quoted 16 lines

drivers/net/mana/mana.c new file +16 -0

+					port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			priv->ib_ctx = ctx;
+			priv->port_id = eth_dev->data->port_id;
+			priv->dev_port = port;
+			eth_dev->data->dev_private = priv;
+			priv->dev_data = eth_dev->data;
+
+			priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+			priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+			priv->max_rx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+					dev_attr.orig_attr.max_cqe);
+			priv->max_tx_desc =
+				RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+					dev_attr.orig_attr.max_cqe);
+
+			priv->max_send_sge = dev_attr.orig_attr.max_sge;
+			priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+			priv->max_mr = dev_attr.orig_attr.max_mr;
+			priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+			DRV_LOG(INFO, "dev %s max queues %d desc %d

sge %d",

drivers/net/mana/mana.c line 545

Quoted 9 lines

(quoted diff) +1 -0

+				name, priv->max_rx_queues, priv-

max_rx_desc,

drivers/net/mana/mana.c new file +7 -0

+				priv->max_send_sge);
+
+			rte_spinlock_lock(&mana_shared_data->lock);
+			mana_shared_data->primary_cnt++;
+			rte_spinlock_unlock(&mana_shared_data->lock);
+
+			eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

drivers/net/mana/mana.c line 547

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+
+			eth_dev->device = &pci_dev->device;
+			eth_dev->data->dev_flags |=
+				RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+

Please do not use the temporary macro. Please review this patch:

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

Quoted 7 lines

Quoted 2 lines

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Thanks for pointing this out.

It seems some PMDs are still depending on this flag for xstats.

MANA doesn't implement xstats_get() currently, this flag is useful. Is it

okay to keep using this flag before it's finally the time to remove it from all PMDs, or when MANA implements xstats?
Yes, your xstats doesn't implement now. Per queue stats should be filled in xstats API, and the stats API cannot see per queue stats, so stats API in driver shouldn't fill it(suggest that delete it from patch 17/18).

Quoted 2 lines

I guess this flag can be removed if PMD does not support xstats.

Quoted 1 line

I don't understand your suggestion. An application can call rte_eth_stats_get() to get port stats, and this will call into stats_get() in the driver, as implemented in patch 17/18.

When flag RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS is set, an application can also use rte_eth_xstats_get() to get port stats even the driver doesn't implement xstats_get().

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 47 lines · Long Li

Quoted 47 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 42 lines · Long Li

Quoted 42 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 23 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 11 lines

drivers/net/mana/mana.c new file +11 -0

+			      struct rte_pci_device *pci_dev,
+			      struct rte_ether_addr *mac_addr) {
+	struct ibv_device **ibv_list;
+	int ibv_idx;
+	struct ibv_context *ctx;
+	struct ibv_device_attr_ex dev_attr;
+	int num_devices;
+	int ret = 0;
+	uint8_t port;
+	struct mana_priv *priv = NULL;
+	struct rte_eth_dev *eth_dev = NULL;

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

drivers/net/mana/mana.c

Quoted 8 lines

drivers/net/mana/mana.c new file +8 -0

+	bool found_port;
+
+	ibv_list = ibv_get_device_list(&num_devices);
+	for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+		struct ibv_device *ibdev = ibv_list[ibv_idx];
+		struct rte_pci_addr pci_addr;
+
+		DRV_LOG(INFO, "Probe device name %s

dev_name %s

Quoted 15 lines

Quoted 15 lines

Quoted 15 lines

Quoted 15 lines

ibdev_path %s",

Quoted 14 lines

(quoted diff) +1 -0

+			ibdev->name, ibdev->dev_name, ibdev-

ibdev_path);

drivers/net/mana/mana.c new file +12 -0

+
+		if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+			continue;
+
+		/* Ignore if this IB device is not this PCI device */
+		if (pci_dev->addr.domain != pci_addr.domain ||
+		    pci_dev->addr.bus != pci_addr.bus ||
+		    pci_dev->addr.devid != pci_addr.devid ||
+		    pci_dev->addr.function != pci_addr.function)
+			continue;
+
+		ctx = ibv_open_device(ibdev);

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

drivers/net/mana/mana.c

Quoted 8 lines

drivers/net/mana/mana.c new file +8 -0

+		if (!ctx) {
+			DRV_LOG(ERR, "Failed to open IB device %s",
+				ibdev->name);
+			continue;
+		}
+
+		ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+		DRV_LOG(INFO,

"dev_attr.orig_attr.phys_port_cnt %u",

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

drivers/net/mana/mana.c

Quoted 4 lines

drivers/net/mana/mana.c new file +4 -0

+			dev_attr.orig_attr.phys_port_cnt);
+		found_port = false;
+
+		for (port = 1; port <=

dev_attr.orig_attr.phys_port_cnt;

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

drivers/net/mana/mana.c

Quoted 11 lines

drivers/net/mana/mana.c new file +11 -0

+		     port++) {
+			struct ibv_parent_domain_init_attr attr = {};
+			struct rte_ether_addr addr;
+			char address[64];
+			char name[RTE_ETH_NAME_MAX_LEN];
+
+			ret = get_port_mac(ibdev, port, &addr);
+			if (ret)
+				continue;
+
+			if (mac_addr

&& !rte_is_same_ether_addr(&addr,

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

mac_addr))

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				continue;
+
+			rte_ether_format_addr(address,

sizeof(address),

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

&addr);

Review

Quoted 1 line

(quoted diff) +1 -0

+			DRV_LOG(INFO, "device located port %u

address

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

%s",

drivers/net/mana/mana.c line 422

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+				port, address);
+			found_port = true;
+
+			priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+

RTE_CACHE_LINE_SIZE,

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

Quoted 11 lines

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+						  SOCKET_ID_ANY);
+			if (!priv) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			snprintf(name, sizeof(name), "%s_port%d",
+				 pci_dev->device.name, port);
+
+			if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				int fd;
+
+				eth_dev =

rte_eth_dev_attach_secondary(name);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (!eth_dev) {
+					DRV_LOG(ERR, "Can't attach

to dev

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

%s",

drivers/net/mana/mana.c

Quoted 7 lines

drivers/net/mana/mana.c new file +7 -0

+						name);
+					ret = -ENOMEM;
+					goto failed;
+				}
+
+				eth_dev->device = &pci_dev->device;
+				eth_dev->dev_ops =

&mana_dev_sec_ops;

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Review

Quoted 7 lines

drivers/net/mana/mana.c new file +4 -0

+				ret = mana_proc_priv_init(eth_dev);
+				if (ret)
+					goto failed;
+				priv->process_priv = eth_dev-

process_private;

(quoted diff) +2 -0

+
+				/* Get the IB FD from the primary

process */

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Review

Quoted 1 line

(quoted diff) +1 -0

+				fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (fd < 0) {
+					DRV_LOG(ERR, "Failed to get

FD %d",

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

Quoted 9 lines

fd);

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+					ret = -ENODEV;
+					goto failed;
+				}
+
+				ret =

mana_map_doorbell_secondary(eth_dev, fd);

Review

Quoted 2 lines

(quoted diff) +2 -0

+				if (ret) {
+					DRV_LOG(ERR, "Failed

secondary

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

map %d",

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+						fd);
+					goto failed;
+				}
+
+				/* fd is no not used after mapping

doorbell */

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

drivers/net/mana/mana.c line 466

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+				close(fd);
+
+

rte_spinlock_lock(&mana_shared_data-

Quoted 5 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Review

Quoted 2 lines

lock);

(quoted diff) +1 -0

+				mana_shared_data-

secondary_cnt++;

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

drivers/net/mana/mana.c line 466

Quoted 2 lines

drivers/net/mana/mana.c new file +2 -0

+				mana_local_data.secondary_cnt++;
+

rte_spinlock_unlock(&mana_shared_data-

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Review

Quoted 3 lines

lock);

(quoted diff) +2 -0

+
+				rte_eth_copy_pci_info(eth_dev,

pci_dev);

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

+

	rte_eth_dev_probing_finish(eth_dev);

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+				/* Impossible to have more than one

port

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

drivers/net/mana/mana.c new file +6 -0

+				 * matching a MAC address
+				 */
+				continue;
+			}
+
+			eth_dev = rte_eth_dev_allocate(name);

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

drivers/net/mana/mana.c

Quoted 8 lines

drivers/net/mana/mana.c new file +8 -0

+			if (!eth_dev) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			eth_dev->data->mac_addrs =
+				rte_calloc("mana_mac", 1,
+					   sizeof(struct

rte_ether_addr), 0);

Quoted 7 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

drivers/net/mana/mana.c

Quoted 6 lines

drivers/net/mana/mana.c new file +6 -0

+			if (!eth_dev->data->mac_addrs) {
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			rte_ether_addr_copy(&addr, eth_dev-

data-

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

drivers/net/mana/mana.c

Quoted 5 lines

mac_addrs);

drivers/net/mana/mana.c new file +4 -0

+
+			priv->ib_pd = ibv_alloc_pd(ctx);
+			if (!priv->ib_pd) {
+				DRV_LOG(ERR, "ibv_alloc_pd failed

port %d",

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

port);

drivers/net/mana/mana.c

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			/* Create a parent domain with the port

number */

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Quoted 5 lines

Review

Quoted 2 lines

(quoted diff) +2 -0

+			attr.pd = priv->ib_pd;
+			attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Review

Quoted 2 lines

(quoted diff) +2 -0

+			attr.pd_context = (void *)(uint64_t)port;
+			priv->ib_parent_pd =

ibv_alloc_parent_domain(ctx,

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

Quoted 4 lines

&attr);

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+			if (!priv->ib_parent_pd) {
+				DRV_LOG(ERR,
+					"ibv_alloc_parent_domain

failed port

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

Quoted 13 lines

%d",

drivers/net/mana/mana.c

Quoted 12 lines

drivers/net/mana/mana.c new file +12 -0

+					port);
+				ret = -ENOMEM;
+				goto failed;
+			}
+
+			priv->ib_ctx = ctx;
+			priv->port_id = eth_dev->data->port_id;
+			priv->dev_port = port;
+			eth_dev->data->dev_private = priv;
+			priv->dev_data = eth_dev->data;
+
+			priv->max_rx_queues =

dev_attr.orig_attr.max_qp;

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Review

Quoted 1 line

(quoted diff) +1 -0

+			priv->max_tx_queues =

dev_attr.orig_attr.max_qp;

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

drivers/net/mana/mana.c line 527

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+
+			priv->max_rx_desc =
+

RTE_MIN(dev_attr.orig_attr.max_qp_wr,

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

drivers/net/mana/mana.c line 527

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+					dev_attr.orig_attr.max_cqe);
+			priv->max_tx_desc =
+

RTE_MIN(dev_attr.orig_attr.max_qp_wr,

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

Quoted 3 lines

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+					dev_attr.orig_attr.max_cqe);
+
+			priv->max_send_sge =

dev_attr.orig_attr.max_sge;

Quoted 1 line

Quoted 1 line

Quoted 1 line

Quoted 1 line

Review

Quoted 1 line

(quoted diff) +1 -0

+			priv->max_recv_sge =

dev_attr.orig_attr.max_sge;

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

drivers/net/mana/mana.c

Quoted 3 lines

drivers/net/mana/mana.c new file +3 -0

+
+			priv->max_mr = dev_attr.orig_attr.max_mr;
+			priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+			DRV_LOG(INFO, "dev %s max queues %d

desc %d

Quoted 10 lines

Quoted 6 lines

Quoted 6 lines

Quoted 6 lines

sge %d",

drivers/net/mana/mana.c

Quoted 5 lines

(quoted diff) +1 -0

+				name, priv->max_rx_queues, priv-

max_rx_desc,

drivers/net/mana/mana.c new file +3 -0

+				priv->max_send_sge);
+
+			rte_spinlock_lock(&mana_shared_data-

lock);

Quoted 2 lines

Quoted 2 lines

Quoted 2 lines

Review

Quoted 2 lines

(quoted diff) +2 -0

+			mana_shared_data->primary_cnt++;
+			rte_spinlock_unlock(&mana_shared_data-

lock);

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

drivers/net/mana/mana.c line 545

Quoted 2 lines

drivers/net/mana/mana.c new file +2 -0

+
+			eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

drivers/net/mana/mana.c line 547

Quoted 4 lines

drivers/net/mana/mana.c new file +4 -0

+
+			eth_dev->device = &pci_dev->device;
+			eth_dev->data->dev_flags |=
+

	RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;

Quoted 12 lines

Quoted 12 lines

Quoted 12 lines

Quoted 8 lines

Quoted 1 line

+

Please do not use the temporary macro. Please review this patch:

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Thanks for pointing this out.

It seems some PMDs are still depending on this flag for xstats.

Quoted 7 lines

Quoted 7 lines

Quoted 2 lines

MANA doesn't implement xstats_get() currently, this flag is useful. Is it

okay to keep using this flag before it's finally the time to remove it from all PMDs, or when MANA implements xstats?
Yes, your xstats doesn't implement now. Per queue stats should be filled in xstats API, and the stats API cannot see per queue stats, so stats API in driver shouldn't fill it(suggest that delete it from patch

17/18).

Quoted 4 lines

Quoted 3 lines

I guess this flag can be removed if PMD does not support xstats.

Quoted 1 line

I don't understand your suggestion. An application can call

rte_eth_stats_get() to get port stats, and this will call into stats_get() in the driver, as implemented in patch 17/18.

Quoted 1 line

Quoted 1 line

When flag RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS is set, an application

can also use rte_eth_xstats_get() to get port stats even the driver doesn't implement xstats_get().

I think new PMD should follow the announced switch Ferruh mentioned, otherwise, the switch will never be completed.

Suggest that mana driver can implement a simple xstats_get() to fill per queue stats if you want to support per queue stats.

@Ferruh, what do you think?

CAUTION: This message has originated from an External Source. Please use proper judgment and caution when opening attachments, clicking links, or responding to this email.

Quoted 277 lines · Long Li

Quoted 277 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 259 lines · Long Li

Quoted 259 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 23 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux.
Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct verb header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

Review

Quoted 2 lines

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 12 lines

+                       struct rte_pci_device *pci_dev, +                       struct rte_ether_addr *mac_addr) {
+ struct ibv_device **ibv_list;
+ int ibv_idx;
+ struct ibv_context *ctx;
+ struct ibv_device_attr_ex dev_attr;
+ int num_devices;
+ int ret = 0;
+ uint8_t port;
+ struct mana_priv *priv = NULL;
+ struct rte_eth_dev *eth_dev = NULL;
+ bool found_port;

drivers/net/mana/mana.c

Quoted 7 lines

drivers/net/mana/mana.c new file +7 -0

+
+ ibv_list = ibv_get_device_list(&num_devices);
+ for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+         struct ibv_device *ibdev = ibv_list[ibv_idx];
+         struct rte_pci_addr pci_addr;
+
+         DRV_LOG(INFO, "Probe device name %s dev_name %s

ibdev_path %s",

Quoted 27 lines

+                 ibdev->name, ibdev->dev_name, ibdev- ibdev_path);

drivers/net/mana/mana.c new file +25 -0

+
+         if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+                 continue;
+
+         /* Ignore if this IB device is not this PCI device */
+         if (pci_dev->addr.domain != pci_addr.domain ||
+             pci_dev->addr.bus != pci_addr.bus ||
+             pci_dev->addr.devid != pci_addr.devid ||
+             pci_dev->addr.function != pci_addr.function)
+                 continue;
+
+         ctx = ibv_open_device(ibdev);
+         if (!ctx) {
+                 DRV_LOG(ERR, "Failed to open IB device %s",
+                         ibdev->name);
+                 continue;
+         }
+
+         ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+         DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+                 dev_attr.orig_attr.phys_port_cnt);
+         found_port = false;
+
+         for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+              port++) {

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+                 struct ibv_parent_domain_init_attr attr = {};
+                 struct rte_ether_addr addr;
+                 char address[64];
+                 char name[RTE_ETH_NAME_MAX_LEN];
+
+                 ret = get_port_mac(ibdev, port, &addr);
+                 if (ret)
+                         continue;
+
+                 if (mac_addr && !rte_is_same_ether_addr(&addr,

mac_addr))

Quoted 3 lines

+                         continue;
+
+                 rte_ether_format_addr(address, sizeof(address),

&addr);

Quoted 1 line

+                 DRV_LOG(INFO, "device located port %u address

%s",

Quoted 5 lines

+                         port, address);
+                 found_port = true;

drivers/net/mana/mana.c new file +3 -0

+
+                 priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+                                           RTE_CACHE_LINE_SIZE,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+                                           SOCKET_ID_ANY);
+                 if (!priv) {
+                         ret = -ENOMEM;
+                         goto failed;
+                 }
+
+                 snprintf(name, sizeof(name), "%s_port%d",
+                          pci_dev->device.name, port);
+
+                 if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

Quoted 3 lines

+                         int fd;
+
+                         eth_dev =

rte_eth_dev_attach_secondary(name);

Quoted 2 lines

+                         if (!eth_dev) {
+                                 DRV_LOG(ERR, "Can't attach to dev

%s",

Quoted 8 lines

+                                         name);
+                                 ret = -ENOMEM;
+                                 goto failed;
+                         }
+
+                         eth_dev->device = &pci_dev->device; +                         eth_dev->dev_ops = &mana_dev_sec_ops; +                         ret = mana_proc_priv_init(eth_dev);

Quoted 8 lines

+                         if (ret)
+                                 goto failed;
+                         priv->process_priv = eth_dev- process_private;
+
+                         /* Get the IB FD from the primary process */
+                         fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Quoted 2 lines

+                         if (fd < 0) {
+                                 DRV_LOG(ERR, "Failed to get FD %d",

fd);

Quoted 5 lines

+                                 ret = -ENODEV;
+                                 goto failed;
+                         }
+
+                         ret =

mana_map_doorbell_secondary(eth_dev, fd);

Quoted 2 lines

+                         if (ret) {
+                                 DRV_LOG(ERR, "Failed secondary

map %d",

Quoted 37 lines

+                                         fd);
+                                 goto failed;
+                         }
+
+                         /* fd is no not used after mapping doorbell */
+                         close(fd);
+
+                         rte_spinlock_lock(&mana_shared_data- lock);
+                         mana_shared_data->secondary_cnt++; +                         mana_local_data.secondary_cnt++; +                         rte_spinlock_unlock(&mana_shared_data- lock);

drivers/net/mana/mana.c new file +19 -0

+
+                         rte_eth_copy_pci_info(eth_dev, pci_dev);
+                         rte_eth_dev_probing_finish(eth_dev);
+
+                         /* Impossible to have more than one port
+                          * matching a MAC address
+                          */
+                         continue;
+                 }
+
+                 eth_dev = rte_eth_dev_allocate(name);
+                 if (!eth_dev) {
+                         ret = -ENOMEM;
+                         goto failed;
+                 }
+
+                 eth_dev->data->mac_addrs =
+                         rte_calloc("mana_mac", 1,
+                                    sizeof(struct 

rte_ether_addr), 0);
+                 if (!eth_dev->data->mac_addrs) { +                         ret = -ENOMEM;
+                         goto failed;

Quoted 8 lines

+                 }
+
+                 rte_ether_addr_copy(&addr, eth_dev->data- mac_addrs);
+
+                 priv->ib_pd = ibv_alloc_pd(ctx);
+                 if (!priv->ib_pd) {
+                         DRV_LOG(ERR, "ibv_alloc_pd failed port %d",

port);

Quoted 7 lines

+                         ret = -ENOMEM;
+                         goto failed;
+                 }
+
+                 /* Create a parent domain with the port number */ +                 attr.pd = priv->ib_pd;
+                 attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Quoted 2 lines

+                 attr.pd_context = (void *)(uint64_t)port; +                 priv->ib_parent_pd = ibv_alloc_parent_domain(ctx,

&attr);

Quoted 4 lines

+                 if (!priv->ib_parent_pd) {
+                         DRV_LOG(ERR,
+                                 "ibv_alloc_parent_domain failed port

%d",

Quoted 16 lines

+                                 port);
+                         ret = -ENOMEM;
+                         goto failed;
+                 }

drivers/net/mana/mana.c new file +12 -0

+
+                 priv->ib_ctx = ctx;
+                 priv->port_id = eth_dev->data->port_id;
+                 priv->dev_port = port;
+                 eth_dev->data->dev_private = priv;
+                 priv->dev_data = eth_dev->data;
+
+                 priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+                 priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+                 priv->max_rx_desc =
+                         RTE_MIN(dev_attr.orig_attr.max_qp_wr,

drivers/net/mana/mana.c

Quoted 10 lines

drivers/net/mana/mana.c new file +10 -0

+                                 dev_attr.orig_attr.max_cqe);
+                 priv->max_tx_desc =
+                         RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+                                 dev_attr.orig_attr.max_cqe);
+
+                 priv->max_send_sge = dev_attr.orig_attr.max_sge;
+                 priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+                 priv->max_mr = dev_attr.orig_attr.max_mr;
+                 priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Quoted 2 lines

+
+                 DRV_LOG(INFO, "dev %s max queues %d desc %d

sge %d",

drivers/net/mana/mana.c line 545

Quoted 9 lines

+                         name, priv->max_rx_queues, priv- max_rx_desc,
+                         priv->max_send_sge);

drivers/net/mana/mana.c new file +6 -0

+
+                 rte_spinlock_lock(&mana_shared_data->lock);
+                 mana_shared_data->primary_cnt++;
+                 rte_spinlock_unlock(&mana_shared_data->lock);
+
+                 eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

drivers/net/mana/mana.c line 547

Quoted 5 lines

drivers/net/mana/mana.c new file +5 -0

+
+                 eth_dev->device = &pci_dev->device;
+                 eth_dev->data->dev_flags |=
+                         RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+

Please do not use the temporary macro. Please review this patch:

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

Quoted 7 lines

Quoted 2 lines

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Thanks for pointing this out.

It seems some PMDs are still depending on this flag for xstats.

MANA doesn't implement xstats_get() currently, this flag is useful. Is it

okay to keep using this flag before it's finally the time to remove it from all PMDs, or when MANA implements xstats?
Yes, your xstats doesn't implement now. Per queue stats should be filled in xstats API, and the stats API cannot see per queue stats, so stats API in driver shouldn't fill it(suggest that delete it from patch 17/18).

Quoted 5 lines

Quoted 2 lines

I guess this flag can be removed if PMD does not support xstats.

Quoted 1 line

I don't understand your suggestion. An application can call rte_eth_stats_get() to get port stats, and this will call into stats_get() in the driver, as implemented in patch 17/18.

When flag RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS  is set, an application can also use rte_eth_xstats_get() to get port stats even the driver doesn't implement xstats_get().

I think new PMD should follow the announced switch Ferruh mentioned, otherwise, the switch will never be completed.

Suggest that mana driver can implement a simple xstats_get() to fill per queue stats if you want to support per queue stats.

@Ferruh, what do you think?

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 5 lines · lihuisong (C)

CAUTION: This message has originated from an External Source. Please use proper judgment and caution when opening attachments, clicking links, or responding to this email.

Quoted 28 lines

Quoted 27 lines · Long Li

Quoted 27 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 22 lines · Long Li

Quoted 22 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 17 lines · 在 2022/8/31 6:51, [email protected] 写道:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

---
Change log:
v2:
Fix typos.
Make the driver build only on x86-64 and Linux. Remove unused header files.
Change port definition to uint16_t or uint8_t (for IB). Use getline() in place of fgets() to read and truncate a line. v3:
Add meson build check for required functions from RDMA direct

verb

Quoted 264 lines

Quoted 256 lines

Quoted 254 lines

Quoted 237 lines

Quoted 237 lines

Review

Quoted 8 lines

header file
v4:
Remove extra "\n" in logging code.
Use "r" in place of "rb" in fopen() to read text files.

[snip]

(quoted diff) +2 -0

+
+static int mana_pci_probe_mac(struct rte_pci_driver *pci_drv

__rte_unused,

Quoted 12 lines

+                       struct rte_pci_device *pci_dev, +                       struct rte_ether_addr *mac_addr) {
+ struct ibv_device **ibv_list;
+ int ibv_idx;
+ struct ibv_context *ctx;
+ struct ibv_device_attr_ex dev_attr;
+ int num_devices;
+ int ret = 0;
+ uint8_t port;
+ struct mana_priv *priv = NULL;
+ struct rte_eth_dev *eth_dev = NULL;
+ bool found_port;

Review

Quoted 7 lines

(quoted diff) +7 -0

+
+ ibv_list = ibv_get_device_list(&num_devices);
+ for (ibv_idx = 0; ibv_idx < num_devices; ibv_idx++) {
+         struct ibv_device *ibdev = ibv_list[ibv_idx];
+         struct rte_pci_addr pci_addr;
+
+         DRV_LOG(INFO, "Probe device name %s dev_name %s

ibdev_path %s",

Quoted 27 lines

+                 ibdev->name, ibdev->dev_name, ibdev- ibdev_path);

(quoted diff) +25 -0

+
+         if (mana_ibv_device_to_pci_addr(ibdev, &pci_addr))
+                 continue;
+
+         /* Ignore if this IB device is not this PCI device */
+         if (pci_dev->addr.domain != pci_addr.domain ||
+             pci_dev->addr.bus != pci_addr.bus ||
+             pci_dev->addr.devid != pci_addr.devid ||
+             pci_dev->addr.function != pci_addr.function)
+                 continue;
+
+         ctx = ibv_open_device(ibdev);
+         if (!ctx) {
+                 DRV_LOG(ERR, "Failed to open IB device %s",
+                         ibdev->name);
+                 continue;
+         }
+
+         ret = ibv_query_device_ex(ctx, NULL, &dev_attr);
+         DRV_LOG(INFO, "dev_attr.orig_attr.phys_port_cnt %u",
+                 dev_attr.orig_attr.phys_port_cnt);
+         found_port = false;
+
+         for (port = 1; port <= dev_attr.orig_attr.phys_port_cnt;
+              port++) {

Review

Quoted 10 lines

(quoted diff) +10 -0

+                 struct ibv_parent_domain_init_attr attr = {};
+                 struct rte_ether_addr addr;
+                 char address[64];
+                 char name[RTE_ETH_NAME_MAX_LEN];
+
+                 ret = get_port_mac(ibdev, port, &addr);
+                 if (ret)
+                         continue;
+
+                 if (mac_addr && !rte_is_same_ether_addr(&addr,

mac_addr))

Quoted 3 lines

+                         continue;
+
+                 rte_ether_format_addr(address, sizeof(address),

&addr);

Quoted 1 line

+                 DRV_LOG(INFO, "device located port %u address

%s",

Quoted 5 lines

+                         port, address);
+                 found_port = true;

(quoted diff) +3 -0

+
+                 priv = rte_zmalloc_socket(NULL, sizeof(*priv),
+                                           RTE_CACHE_LINE_SIZE,

Review

Quoted 10 lines

(quoted diff) +10 -0

+                                           SOCKET_ID_ANY);
+                 if (!priv) {
+                         ret = -ENOMEM;
+                         goto failed;
+                 }
+
+                 snprintf(name, sizeof(name), "%s_port%d",
+                          pci_dev->device.name, port);
+
+                 if (rte_eal_process_type() ==

RTE_PROC_SECONDARY) {

Quoted 3 lines

+                         int fd;
+
+                         eth_dev =

rte_eth_dev_attach_secondary(name);

Quoted 2 lines

+                         if (!eth_dev) {
+                                 DRV_LOG(ERR, "Can't attach to dev

%s",

Quoted 8 lines

+                                         name);
+                                 ret = -ENOMEM;
+                                 goto failed;
+                         }
+
+                         eth_dev->device = &pci_dev->device; +                         eth_dev->dev_ops = &mana_dev_sec_ops; +                         ret = mana_proc_priv_init(eth_dev);

Quoted 8 lines

+                         if (ret)
+                                 goto failed;
+                         priv->process_priv = eth_dev- process_private;
+
+                         /* Get the IB FD from the primary process */
+                         fd =

mana_mp_req_verbs_cmd_fd(eth_dev);

Quoted 2 lines

+                         if (fd < 0) {
+                                 DRV_LOG(ERR, "Failed to get FD %d",

fd);

Quoted 5 lines

+                                 ret = -ENODEV;
+                                 goto failed;
+                         }
+
+                         ret =

mana_map_doorbell_secondary(eth_dev, fd);

Quoted 2 lines

+                         if (ret) {
+                                 DRV_LOG(ERR, "Failed secondary

map %d",

Quoted 37 lines

+                                         fd);
+                                 goto failed;
+                         }
+
+                         /* fd is no not used after mapping doorbell */
+                         close(fd);
+
+                         rte_spinlock_lock(&mana_shared_data- lock);
+                         mana_shared_data->secondary_cnt++; +                         mana_local_data.secondary_cnt++; +                         rte_spinlock_unlock(&mana_shared_data- lock);

(quoted diff) +19 -0

+
+                         rte_eth_copy_pci_info(eth_dev, pci_dev);
+                         rte_eth_dev_probing_finish(eth_dev);
+
+                         /* Impossible to have more than one port
+                          * matching a MAC address
+                          */
+                         continue;
+                 }
+
+                 eth_dev = rte_eth_dev_allocate(name);
+                 if (!eth_dev) {
+                         ret = -ENOMEM;
+                         goto failed;
+                 }
+
+                 eth_dev->data->mac_addrs =
+                         rte_calloc("mana_mac", 1,
+                                    sizeof(struct

rte_ether_addr), 0);
+                 if (!eth_dev->data->mac_addrs) { +                         ret = -ENOMEM;
+                         goto failed;

Quoted 8 lines

+                 }
+
+                 rte_ether_addr_copy(&addr, eth_dev->data- mac_addrs);
+
+                 priv->ib_pd = ibv_alloc_pd(ctx);
+                 if (!priv->ib_pd) {
+                         DRV_LOG(ERR, "ibv_alloc_pd failed port %d",

port);

Quoted 7 lines

+                         ret = -ENOMEM;
+                         goto failed;
+                 }
+
+                 /* Create a parent domain with the port number */ +                 attr.pd = priv->ib_pd;
+                 attr.comp_mask =

IBV_PARENT_DOMAIN_INIT_ATTR_PD_CONTEXT;

Quoted 2 lines

+                 attr.pd_context = (void *)(uint64_t)port; +                 priv->ib_parent_pd = ibv_alloc_parent_domain(ctx,

&attr);

Quoted 4 lines

+                 if (!priv->ib_parent_pd) {
+                         DRV_LOG(ERR,
+                                 "ibv_alloc_parent_domain failed port

%d",

Quoted 16 lines

+                                 port);
+                         ret = -ENOMEM;
+                         goto failed;
+                 }

(quoted diff) +12 -0

+
+                 priv->ib_ctx = ctx;
+                 priv->port_id = eth_dev->data->port_id;
+                 priv->dev_port = port;
+                 eth_dev->data->dev_private = priv;
+                 priv->dev_data = eth_dev->data;
+
+                 priv->max_rx_queues = dev_attr.orig_attr.max_qp;
+                 priv->max_tx_queues = dev_attr.orig_attr.max_qp;
+
+                 priv->max_rx_desc =
+                         RTE_MIN(dev_attr.orig_attr.max_qp_wr,

Review

Quoted 10 lines

(quoted diff) +10 -0

+                                 dev_attr.orig_attr.max_cqe);
+                 priv->max_tx_desc =
+                         RTE_MIN(dev_attr.orig_attr.max_qp_wr,
+                                 dev_attr.orig_attr.max_cqe);
+
+                 priv->max_send_sge = dev_attr.orig_attr.max_sge;
+                 priv->max_recv_sge = dev_attr.orig_attr.max_sge;
+
+                 priv->max_mr = dev_attr.orig_attr.max_mr;
+                 priv->max_mr_size =

dev_attr.orig_attr.max_mr_size;

Quoted 2 lines

+
+                 DRV_LOG(INFO, "dev %s max queues %d desc %d

sge %d",

Review

Quoted 9 lines

+                         name, priv->max_rx_queues, priv- max_rx_desc,
+                         priv->max_send_sge);

(quoted diff) +6 -0

+
+                 rte_spinlock_lock(&mana_shared_data->lock);
+                 mana_shared_data->primary_cnt++;
+                 rte_spinlock_unlock(&mana_shared_data->lock);
+
+                 eth_dev->data->dev_flags |=

RTE_ETH_DEV_INTR_RMV;

Review

Quoted 5 lines

(quoted diff) +5 -0

+
+                 eth_dev->device = &pci_dev->device;
+                 eth_dev->data->dev_flags |=
+                         RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
+

Please do not use the temporary macro. Please review this patch:

f30e69b41f94 ("ethdev: add device flag to bypass auto-filled queue xstats")

Quoted 8 lines

Quoted 2 lines

This patch requires that per queue statistics are filled in .xstats_get() by PMD.

Thanks for pointing this out.

It seems some PMDs are still depending on this flag for xstats.

MANA doesn't implement xstats_get() currently, this flag is useful. Is it

okay to keep using this flag before it's finally the time to remove it from all
PMDs, or when MANA implements xstats?
Yes, your xstats doesn't implement now. Per queue stats should be filled in
xstats API, and the stats API cannot see per queue stats, so stats API in driver
shouldn't fill it(suggest that delete it from patch 17/18).

Quoted 2 lines

I guess this flag can be removed if PMD does not support xstats.

Quoted 1 line

Quoted 7 lines

I don't understand your suggestion. An application can call rte_eth_stats_get() to get port stats, and this will call into stats_get() in the driver, as implemented in patch 17/18.

When flag RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS  is set, an application can also use rte_eth_xstats_get() to get port stats even the driver doesn't implement xstats_get().

Quoted 8 lines

I think new PMD should follow the announced switch Ferruh mentioned, otherwise, the switch will never be completed.

Suggest that mana driver can implement a simple xstats_get() to fill per queue stats if you want to support per queue stats.

@Ferruh, what do you think?

Hi Huisong,

Thanks for reminding it, yes it makes sense to implement new method in new drivers.

Long,

There is a long term plan to move queue stats from basic stats structure to xstats. The reason behind is increasing number of queues makes basis stats struct too big, on the other hand xstats is more flexible and no fixes size array is required.

You can remove the 'RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS' flag and driver
won't support queue stats in xstats by default. Instead to have that support you will need to implement xstats, later or in this set. When queue stats implemented in xstats, please remember to remove updating 'stats->q_*' in basic stats.

Also in 17/18, the feature 'Stats per queue' seems added, but that is not correct, feature name is misleading here. But it is about queue stats mapping, please check 'doc/guides/nics/features.rst'. So can you please drop that feature.

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

drivers/net/mana/mp.c new file +15 -0

+static int mana_mp_primary_handle(const struct rte_mp_msg *mp_msg,
+				  const void *peer)
+{
+	struct rte_eth_dev *dev;
+	const struct mana_mp_param *param =
+		(const struct mana_mp_param *)mp_msg->param;
+	struct rte_mp_msg mp_res = { 0 };
+	struct mana_mp_param *res = (struct mana_mp_param *)mp_res.param;
+	int ret;
+	struct mana_priv *priv;
+
+	if (!rte_eth_dev_is_valid_port(param->port_id)) {
+		DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id);
+		return -ENODEV;
+	}

drivers/net/mana/mp.c new file +13 -0

+		res->result = 0;
+		ret = rte_mp_reply(&mp_res, peer);
+		break;
+
+	default:
+		DRV_LOG(ERR, "Port %u unknown primary MP type %u",
+			param->port_id, param->type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 7 lines · On 2022/8/31 6:51, [email protected] wrote:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

...

drivers/net/mana/mp.c

Quoted 8 lines

drivers/net/mana/mp.c new file +8 -0

+static int mana_mp_primary_handle(const struct rte_mp_msg *mp_msg,
+				  const void *peer)
+{
+	struct rte_eth_dev *dev;
+	const struct mana_mp_param *param =
+		(const struct mana_mp_param *)mp_msg->param;
+	struct rte_mp_msg mp_res = { 0 };
+	struct mana_mp_param *res = (struct mana_mp_param

*)mp_res.param;

Quoted 7 lines

drivers/net/mana/mp.c new file +7 -0

+	int ret;
+	struct mana_priv *priv;
+
+	if (!rte_eth_dev_is_valid_port(param->port_id)) {
+		DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id);
+		return -ENODEV;
+	}

drivers/net/mana/mp.c line 52

Quoted 10 lines

drivers/net/mana/mp.c new file +10 -0

+
+	dev = &rte_eth_devices[param->port_id];
+	priv = dev->data->dev_private;
+
+	mp_init_msg(&mp_res, param->type, param->port_id);
+
+	switch (param->type) {
+	case MANA_MP_REQ_VERBS_CMD_FD:
+		mp_res.num_fds = 1;
+		mp_res.fds[0] = priv->ib_ctx->cmd_fd;

The cmd_fd is system level handler?

If it's process private handler, it should not used directly in another process.

Quoted 14 lines

drivers/net/mana/mp.c new file +13 -0

+		res->result = 0;
+		ret = rte_mp_reply(&mp_res, peer);
+		break;
+
+	default:
+		DRV_LOG(ERR, "Port %u unknown primary MP type %u",
+			param->port_id, param->type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+

Quoted 41 lines

Subject: Re: [Patch v6 01/18] net/mana: add basic driver, build environment and doc

Quoted 7 lines · On 2022/8/31 6:51, [email protected] wrote:

From: Long Li <[email protected]>

MANA is a PCI device. It uses IB verbs to access hardware through the kernel RDMA layer. This patch introduces build environment and basic device probe functions.

Signed-off-by Long Li <[email protected]>

...

drivers/net/mana/mp.c

Quoted 8 lines

drivers/net/mana/mp.c new file +8 -0

+static int mana_mp_primary_handle(const struct rte_mp_msg *mp_msg,
+				  const void *peer)
+{
+	struct rte_eth_dev *dev;
+	const struct mana_mp_param *param =
+		(const struct mana_mp_param *)mp_msg->param;
+	struct rte_mp_msg mp_res = { 0 };
+	struct mana_mp_param *res = (struct mana_mp_param

*)mp_res.param;

Quoted 7 lines

drivers/net/mana/mp.c new file +7 -0

+	int ret;
+	struct mana_priv *priv;
+
+	if (!rte_eth_dev_is_valid_port(param->port_id)) {
+		DRV_LOG(ERR, "MP handle port ID %u invalid", param->port_id);
+		return -ENODEV;
+	}

drivers/net/mana/mp.c line 52

Quoted 10 lines

drivers/net/mana/mp.c new file +10 -0

+
+	dev = &rte_eth_devices[param->port_id];
+	priv = dev->data->dev_private;
+
+	mp_init_msg(&mp_res, param->type, param->port_id);
+
+	switch (param->type) {
+	case MANA_MP_REQ_VERBS_CMD_FD:
+		mp_res.num_fds = 1;
+		mp_res.fds[0] = priv->ib_ctx->cmd_fd;

The cmd_fd is system level handler?

Quoted 1 line

If it's process private handler, it should not used directly in another process.

According to rte_mp_xxx semantics, the file handle is duplicated to another process. It's not directly used. It's required for secondary process to map to the same doorbell pages.

Quoted 14 lines

Quoted 14 lines

drivers/net/mana/mp.c new file +13 -0

+		res->result = 0;
+		ret = rte_mp_reply(&mp_res, peer);
+		break;
+
+	default:
+		DRV_LOG(ERR, "Port %u unknown primary MP type %u",
+			param->port_id, param->type);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+

From: Long Li <[email protected]>

Add the function to get device info.

Signed-off-by Long Li <[email protected]>

---
 doc/guides/nics/features/mana.ini |  1 +
 drivers/net/mana/mana.c           | 82 +++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)

doc/guides/nics/features/mana.ini +1 -0

index 8043e11f99..566b3e8770 100644
@@ -8,5 +8,6 @@ Link status          = P
 Linux                = Y
 Multiprocess aware   = Y
 Removal event        = Y
+Speed capabilities   = P
 Usage doc            = Y
 x86-64               = Y

drivers/net/mana/mana.c +6 -0

index c9591035ac..e1550b3c08 100644
@@ -116,6 +116,86 @@ mana_dev_close(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static int mana_dev_info_get(struct rte_eth_dev *dev,
+			     struct rte_eth_dev_info *dev_info)
+{
+	struct mana_priv *priv = dev->data->dev_private;
+
+	dev_info->max_mtu = RTE_ETHER_MTU;

drivers/net/mana/mana.c +14 -0

+
+	/* RX params */
+	dev_info->default_rxportconf.burst_size = 1;
+	dev_info->default_rxportconf.ring_size = MAX_RECEIVE_BUFFERS_PER_QUEUE;
+	dev_info->default_rxportconf.nb_queues = 1;
+
+	/* TX params */
+	dev_info->default_txportconf.burst_size = 1;
+	dev_info->default_txportconf.ring_size = MAX_SEND_BUFFERS_PER_QUEUE;
+	dev_info->default_txportconf.nb_queues = 1;
+
+	return 0;
+}
+

Subject: Re: [Patch v6 06/18] net/mana: add device info

Quoted 35 lines · On 2022/8/31 6:51, [email protected] wrote:

From: Long Li <[email protected]>

Add the function to get device info.

Signed-off-by Long Li <[email protected]>

---
 doc/guides/nics/features/mana.ini |  1 +
 drivers/net/mana/mana.c           | 82 +++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)

doc/guides/nics/features/mana.ini +1 -0

index 8043e11f99..566b3e8770 100644
@@ -8,5 +8,6 @@ Link status          = P
 Linux                = Y
 Multiprocess aware   = Y
 Removal event        = Y
+Speed capabilities   = P
 Usage doc            = Y
 x86-64               = Y

drivers/net/mana/mana.c index renamed from drivers/net/mana/mana.c +0 -0

c9591035ac..e1550b3c08 100644

drivers/net/mana/mana.c +5 -0

@@ -116,6 +116,86 @@ mana_dev_close(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static int mana_dev_info_get(struct rte_eth_dev *dev,
+			     struct rte_eth_dev_info *dev_info) {
+	struct mana_priv *priv = dev->data->dev_private;
+
+	dev_info->max_mtu = RTE_ETHER_MTU;

...