Skip to contentSkip to conversation

Intel IFC VF driver for vdpa

RFC In review Zhu Lingshan 3 participants 31 messages Oct 16, 2019 – Oct 23, 2019

Subsystems (5)
lore
Share & tools
lore
#0 lore raw

cover Intel IFC VF driver for vdpa

Hi all:

This series intends to introduce Intel IFC VF NIC driver for Vhost Data Plane Acceleration.

Here comes two main parts, one is ifcvf_base layer, which handles hardware operations. The other is ifcvf_main layer handles VF initialization, configuration and removal, which depends on and complys to vhost_mdev https://lkml.org/lkml/2019/9/26/15

This is a first RFC try, please help review.

Thanks!
BR
Zhu Lingshan

Zhu Lingshan (2):
vhost: IFC VF hardware operation layer
vhost: IFC VF vdpa layer

 drivers/vhost/ifcvf/ifcvf_base.c | 390 ++++++++++++++++++++++++++++
 drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++
 drivers/vhost/ifcvf/ifcvf_main.c | 541 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 1068 insertions(+)
 create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c
 create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h
 create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c
Signature · 1 line
2.16.4
#1 lore raw

RFC 1/2 vhost: IFC VF hardware operation layer

This commit introduced ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations.

---
 drivers/vhost/ifcvf/ifcvf_base.c | 390 +++++++++++++++++++++++++++++++++++++++
 drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++++++
 2 files changed, 527 insertions(+)
 create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c
 create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h
drivers/vhost/ifcvf/ifcvf_base.c new file +390 -0
new file mode 100644
index 000000000000..b85e14c9bdcf
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include "ifcvf_base.h"
+
+static void *get_cap_addr(struct ifcvf_hw *hw, struct virtio_pci_cap *cap)
+{
+	u8 bar = cap->bar;
+	u32 length = cap->length;
+	u32 offset = cap->offset;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	if (bar >= IFCVF_PCI_MAX_RESOURCE) {
+		IFC_ERR(ifcvf->dev,
+			"Invalid bar number %u to get capabilities.\n", bar);
+		return NULL;
+	}
+
+	if (offset + length < offset) {
+		IFC_ERR(ifcvf->dev, "offset(%u) + length(%u) overflows\n",
+			offset, length);
+		return NULL;
+	}
+
+	if (offset + length > hw->mem_resource[cap->bar].len) {
+		IFC_ERR(ifcvf->dev,
+			"offset(%u) + len(%u) overflows bar%u to get capabilities.\n",
+			offset, length, bar);
+		return NULL;
+	}
+
+	return hw->mem_resource[bar].addr + offset;
+}
+
+int ifcvf_read_config_range(struct pci_dev *dev,
+			uint32_t *val, int size, int where)
+{
+	int i;
+
+	for (i = 0; i < size; i += 4) {
+		if (pci_read_config_dword(dev, where + i, val + i / 4) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev)
+{
+	int ret;
+	u8 pos;
+	struct virtio_pci_cap cap;
+	u32 i;
+	u16 notify_off;
+
+	ret = pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &pos);
+
+	if (ret < 0) {
+		IFC_ERR(&dev->dev, "failed to read PCI capability list.\n");
+		return -EIO;
+	}
+
+	while (pos) {
+		ret = ifcvf_read_config_range(dev, (u32 *)&cap,
+				sizeof(cap), pos);
+
+		if (ret < 0) {
+			IFC_ERR(&dev->dev, "failed to get PCI capability at %x",
+					pos);
+			break;
+		}
+
+		if (cap.cap_vndr != PCI_CAP_ID_VNDR)
+			goto next;
+
+		IFC_INFO(&dev->dev, "read PCI config:\n"
+					"config type: %u.\n"
+					"PCI bar: %u.\n"
+					"PCI bar offset: %u.\n"
+					"PCI config len: %u.\n",
+					cap.cfg_type, cap.bar,
+					cap.offset, cap.length);
+
+		switch (cap.cfg_type) {
+		case VIRTIO_PCI_CAP_COMMON_CFG:
+			hw->common_cfg = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->common_cfg = %p.\n",
+					hw->common_cfg);
+			break;
+		case VIRTIO_PCI_CAP_NOTIFY_CFG:
+			pci_read_config_dword(dev, pos + sizeof(cap),
+				&hw->notify_off_multiplier);
+			hw->notify_bar = cap.bar;
+			hw->notify_base = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->notify_base = %p.\n",
+					hw->notify_base);
+			break;
+		case VIRTIO_PCI_CAP_ISR_CFG:
+			hw->isr = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->isr = %p.\n", hw->isr);
+			break;
+		case VIRTIO_PCI_CAP_DEVICE_CFG:
+			hw->dev_cfg = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->dev_cfg = %p.\n", hw->dev_cfg);
+			break;
+		}
+next:
+		pos = cap.cap_next;
+	}
+
+	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+		hw->isr == NULL || hw->dev_cfg == NULL) {
+		IFC_ERR(&dev->dev, "Incomplete PCI capabilities.\n");
+		return -1;
+	}
+
+	for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {
+		iowrite16(i, &hw->common_cfg->queue_select);
+		notify_off = ioread16(&hw->common_cfg->queue_notify_off);
+		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+				notify_off * hw->notify_off_multiplier);
+	}
+
+	hw->lm_cfg = hw->mem_resource[4].addr;
+
+	IFC_INFO(&dev->dev, "PCI capability mapping:\n"
+				"common cfg: %p\n"
+				"notify base: %p\n"
+				"isr cfg: %p\n"
+				"device cfg: %p\n"
+				"multiplier: %u\n",
+				hw->common_cfg,
+				hw->notify_base,
+				hw->isr,
+				hw->dev_cfg,
+				hw->notify_off_multiplier);
+
+	return 0;
+}
+
+static u8 ifcvf_get_status(struct ifcvf_hw *hw)
+{
+	return ioread8(&hw->common_cfg->device_status);
+}
+
+static void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
+{
+	iowrite8(status, &hw->common_cfg->device_status);
+}
+
+static void ifcvf_reset(struct ifcvf_hw *hw)
+{
+	ifcvf_set_status(hw, 0);
+
+	/* flush status write */
+	ifcvf_get_status(hw);
+	hw->generation++;
+}
+
+static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status)
+{
+	if (status != 0)
+		status |= ifcvf_get_status(hw);
+
+	ifcvf_set_status(hw, status);
+	ifcvf_get_status(hw);
+}
+
+u64 ifcvf_get_features(struct ifcvf_hw *hw)
+{
+	u32 features_lo, features_hi;
+	struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+	iowrite32(0, &cfg->device_feature_select);
+	features_lo = ioread32(&cfg->device_feature);
+
+	iowrite32(1, &cfg->device_feature_select);
+	features_hi = ioread32(&cfg->device_feature);
+
+	return ((u64)features_hi << 32) | features_lo;
+}
+static int ifcvf_with_feature(struct ifcvf_hw *hw, u64 bit)
+{
+	return (hw->req_features & (1ULL << bit)) != 0;
+}
+
+static void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+		       void *dst, int length)
+{
+	int i;
+	u8 *p;
+	u8 old_gen, new_gen;
+
+	do {
+		old_gen = ioread8(&hw->common_cfg->config_generation);
+
+		p = dst;
+		for (i = 0; i < length; i++)
+			*p++ = ioread8((u8 *)hw->dev_cfg + offset + i);
+
+		new_gen = ioread8(&hw->common_cfg->config_generation);
+	} while (old_gen != new_gen);
+}
+
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup)
+{
+	u16 status;
+	u64 host_features;
+
+	host_features = ifcvf_get_features(hw);
+	if (ifcvf_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+		ifcvf_read_dev_config(hw,
+				offsetof(struct ifcvf_net_config, status),
+				&status, sizeof(status));
+		if ((status & VIRTIO_NET_S_LINK_UP) == 0)
+			(*is_linkup) = 1;
+		else
+			(*is_linkup) = 0;
+	} else
+		(*is_linkup) = 0;
+}
+
+static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+{
+	struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+	iowrite32(0, &cfg->guest_feature_select);
+	iowrite32(features & ((1ULL << 32) - 1), &cfg->guest_feature);
+
+	iowrite32(1, &cfg->guest_feature_select);
+	iowrite32(features >> 32, &cfg->guest_feature);
+}
+
+static int ifcvf_config_features(struct ifcvf_hw *hw)
+{
+	u64 host_features;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	host_features = ifcvf_get_features(hw);
+	hw->req_features &= host_features;
+
+	ifcvf_set_features(hw, hw->req_features);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK);
+
+	if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+		IFC_ERR(ifcvf->dev, "Failed to set FEATURES_OK status\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void io_write64_twopart(u64 val, u32 *lo, u32 *hi)
+{
+	iowrite32(val & ((1ULL << 32) - 1), lo);
+	iowrite32(val >> 32, hi);
+}
+
+static int ifcvf_hw_enable(struct ifcvf_hw *hw)
+{
+	struct virtio_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u32 i;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	cfg = hw->common_cfg;
+	lm_cfg = hw->lm_cfg;
+
+	iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
+	if (ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
+		IFC_ERR(ifcvf->dev, "No msix vector for device config.\n");
+		return -1;
+	}
+
+	for (i = 0; i < hw->nr_vring; i++) {
+		iowrite16(i, &cfg->queue_select);
+		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+				&cfg->queue_desc_hi);
+		io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+				&cfg->queue_avail_hi);
+		io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+				&cfg->queue_used_hi);
+		iowrite16(hw->vring[i].size, &cfg->queue_size);
+
+		*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4) =
+			(u32)hw->vring[i].last_avail_idx |
+			((u32)hw->vring[i].last_used_idx << 16);
+
+		iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
+		if (ioread16(&cfg->queue_msix_vector) ==
+				VIRTIO_MSI_NO_VECTOR) {
+			IFC_ERR(ifcvf->dev,
+				"No msix vector for queue %u.\n", i);
+			return -1;
+		}
+
+		iowrite16(1, &cfg->queue_enable);
+	}
+
+	return 0;
+}
+
+static void ifcvf_hw_disable(struct ifcvf_hw *hw)
+{
+	u32 i;
+	struct virtio_pci_common_cfg *cfg;
+
+	cfg = hw->common_cfg;
+
+	iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
+	for (i = 0; i < hw->nr_vring; i++) {
+		iowrite16(i, &cfg->queue_select);
+		iowrite16(0, &cfg->queue_enable);
+		iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
+	}
+}
+
+int ifcvf_start_hw(struct ifcvf_hw *hw)
+{
+	ifcvf_reset(hw);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER);
+
+	if (ifcvf_config_features(hw) < 0)
+		return -1;
+
+	if (ifcvf_hw_enable(hw) < 0)
+		return -1;
+
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK);
+
+	return 0;
+}
+
+void ifcvf_stop_hw(struct ifcvf_hw *hw)
+{
+	ifcvf_hw_disable(hw);
+	ifcvf_reset(hw);
+}
+
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+		log_base & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =
+		(log_base >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+		(log_base + log_size) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+		((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}
+
+void ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
+{
+
+	iowrite16(qid, hw->notify_addr[qid]);
+}
+
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw)
+{
+	return hw->notify_bar;
+}
+
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid)
+{
+	return (u8 *)hw->notify_addr[qid] -
+		(u8 *)hw->mem_resource[hw->notify_bar].addr;
+}
drivers/vhost/ifcvf/ifcvf_base.h new file +137 -0

137 lines not on this page · raw message

Signature · 1 line
2.16.4
#2 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

Quoted 2 lines · Zhu Lingshan

This commit introduced ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations.

It's better to describe the difference between ifc vf and virtio in the commit log or is there a open doc for this?

Quoted 124 lines
---
  drivers/vhost/ifcvf/ifcvf_base.c | 390 +++++++++++++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++++++
  2 files changed, 527 insertions(+)

create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h

drivers/vhost/ifcvf/ifcvf_base.c new file +109 -0
new file mode 100644
index 000000000000..b85e14c9bdcf
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include "ifcvf_base.h"
+
+static void *get_cap_addr(struct ifcvf_hw *hw, struct virtio_pci_cap *cap)
+{
+	u8 bar = cap->bar;
+	u32 length = cap->length;
+	u32 offset = cap->offset;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	if (bar >= IFCVF_PCI_MAX_RESOURCE) {
+		IFC_ERR(ifcvf->dev,
+			"Invalid bar number %u to get capabilities.\n", bar);
+		return NULL;
+	}
+
+	if (offset + length < offset) {
+		IFC_ERR(ifcvf->dev, "offset(%u) + length(%u) overflows\n",
+			offset, length);
+		return NULL;
+	}
+
+	if (offset + length > hw->mem_resource[cap->bar].len) {
+		IFC_ERR(ifcvf->dev,
+			"offset(%u) + len(%u) overflows bar%u to get capabilities.\n",
+			offset, length, bar);
+		return NULL;
+	}
+
+	return hw->mem_resource[bar].addr + offset;
+}
+
+int ifcvf_read_config_range(struct pci_dev *dev,
+			uint32_t *val, int size, int where)
+{
+	int i;
+
+	for (i = 0; i < size; i += 4) {
+		if (pci_read_config_dword(dev, where + i, val + i / 4) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev)
+{
+	int ret;
+	u8 pos;
+	struct virtio_pci_cap cap;
+	u32 i;
+	u16 notify_off;
+
+	ret = pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &pos);
+
+	if (ret < 0) {
+		IFC_ERR(&dev->dev, "failed to read PCI capability list.\n");
+		return -EIO;
+	}
+
+	while (pos) {
+		ret = ifcvf_read_config_range(dev, (u32 *)&cap,
+				sizeof(cap), pos);
+
+		if (ret < 0) {
+			IFC_ERR(&dev->dev, "failed to get PCI capability at %x",
+					pos);
+			break;
+		}
+
+		if (cap.cap_vndr != PCI_CAP_ID_VNDR)
+			goto next;
+
+		IFC_INFO(&dev->dev, "read PCI config:\n"
+					"config type: %u.\n"
+					"PCI bar: %u.\n"
+					"PCI bar offset: %u.\n"
+					"PCI config len: %u.\n",
+					cap.cfg_type, cap.bar,
+					cap.offset, cap.length);
+
+		switch (cap.cfg_type) {
+		case VIRTIO_PCI_CAP_COMMON_CFG:
+			hw->common_cfg = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->common_cfg = %p.\n",
+					hw->common_cfg);
+			break;
+		case VIRTIO_PCI_CAP_NOTIFY_CFG:
+			pci_read_config_dword(dev, pos + sizeof(cap),
+				&hw->notify_off_multiplier);
+			hw->notify_bar = cap.bar;
+			hw->notify_base = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->notify_base = %p.\n",
+					hw->notify_base);
+			break;
+		case VIRTIO_PCI_CAP_ISR_CFG:
+			hw->isr = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->isr = %p.\n", hw->isr);
+			break;
+		case VIRTIO_PCI_CAP_DEVICE_CFG:
+			hw->dev_cfg = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->dev_cfg = %p.\n", hw->dev_cfg);
+			break;
+		}
+next:

drivers/vhost/ifcvf/ifcvf_base.c line 119

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+		pos = cap.cap_next;
+	}
+
+	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+		hw->isr == NULL || hw->dev_cfg == NULL) {
+		IFC_ERR(&dev->dev, "Incomplete PCI capabilities.\n");
+		return -1;
+	}
+
+	for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {

Any reason for using hard coded queue pairs limit other than the max_queue_pairs in the net config?

Quoted 29 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +29 -0
+		iowrite16(i, &hw->common_cfg->queue_select);
+		notify_off = ioread16(&hw->common_cfg->queue_notify_off);
+		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+				notify_off * hw->notify_off_multiplier);
+	}
+
+	hw->lm_cfg = hw->mem_resource[4].addr;
+
+	IFC_INFO(&dev->dev, "PCI capability mapping:\n"
+				"common cfg: %p\n"
+				"notify base: %p\n"
+				"isr cfg: %p\n"
+				"device cfg: %p\n"
+				"multiplier: %u\n",
+				hw->common_cfg,
+				hw->notify_base,
+				hw->isr,
+				hw->dev_cfg,
+				hw->notify_off_multiplier);
+
+	return 0;
+}
+
+static u8 ifcvf_get_status(struct ifcvf_hw *hw)
+{
+	return ioread8(&hw->common_cfg->device_status);
+}
+
+static void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)

drivers/vhost/ifcvf/ifcvf_base.c line 158

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+{
+	iowrite8(status, &hw->common_cfg->device_status);
+}
+
+static void ifcvf_reset(struct ifcvf_hw *hw)
+{
+	ifcvf_set_status(hw, 0);
+
+	/* flush status write */
+	ifcvf_get_status(hw);

Why this flush is needed?

Quoted 40 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +40 -0
+	hw->generation++;
+}
+
+static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status)
+{
+	if (status != 0)
+		status |= ifcvf_get_status(hw);
+
+	ifcvf_set_status(hw, status);
+	ifcvf_get_status(hw);
+}
+
+u64 ifcvf_get_features(struct ifcvf_hw *hw)
+{
+	u32 features_lo, features_hi;
+	struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+	iowrite32(0, &cfg->device_feature_select);
+	features_lo = ioread32(&cfg->device_feature);
+
+	iowrite32(1, &cfg->device_feature_select);
+	features_hi = ioread32(&cfg->device_feature);
+
+	return ((u64)features_hi << 32) | features_lo;
+}
+static int ifcvf_with_feature(struct ifcvf_hw *hw, u64 bit)
+{
+	return (hw->req_features & (1ULL << bit)) != 0;
+}
+
+static void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+		       void *dst, int length)
+{
+	int i;
+	u8 *p;
+	u8 old_gen, new_gen;
+
+	do {
+		old_gen = ioread8(&hw->common_cfg->config_generation);
+

drivers/vhost/ifcvf/ifcvf_base.c line 208

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+		p = dst;
+		for (i = 0; i < length; i++)
+			*p++ = ioread8((u8 *)hw->dev_cfg + offset + i);
+
+		new_gen = ioread8(&hw->common_cfg->config_generation);
+	} while (old_gen != new_gen);
+}
+
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup)
+{

Why not just return bollean?

Quoted 25 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+	u16 status;
+	u64 host_features;
+
+	host_features = ifcvf_get_features(hw);
+	if (ifcvf_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+		ifcvf_read_dev_config(hw,
+				offsetof(struct ifcvf_net_config, status),
+				&status, sizeof(status));
+		if ((status & VIRTIO_NET_S_LINK_UP) == 0)
+			(*is_linkup) = 1;
+		else
+			(*is_linkup) = 0;
+	} else
+		(*is_linkup) = 0;
+}
+
+static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+{
+	struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+	iowrite32(0, &cfg->guest_feature_select);
+	iowrite32(features & ((1ULL << 32) - 1), &cfg->guest_feature);
+
+	iowrite32(1, &cfg->guest_feature_select);
+	iowrite32(features >> 32, &cfg->guest_feature);

drivers/vhost/ifcvf/ifcvf_base.c line 243

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+}
+
+static int ifcvf_config_features(struct ifcvf_hw *hw)
+{
+	u64 host_features;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	host_features = ifcvf_get_features(hw);
+	hw->req_features &= host_features;

Is this a must, can't device deal with this?

Quoted 112 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +112 -0
+
+	ifcvf_set_features(hw, hw->req_features);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK);
+
+	if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+		IFC_ERR(ifcvf->dev, "Failed to set FEATURES_OK status\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void io_write64_twopart(u64 val, u32 *lo, u32 *hi)
+{
+	iowrite32(val & ((1ULL << 32) - 1), lo);
+	iowrite32(val >> 32, hi);
+}
+
+static int ifcvf_hw_enable(struct ifcvf_hw *hw)
+{
+	struct virtio_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u32 i;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	cfg = hw->common_cfg;
+	lm_cfg = hw->lm_cfg;
+
+	iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
+	if (ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
+		IFC_ERR(ifcvf->dev, "No msix vector for device config.\n");
+		return -1;
+	}
+
+	for (i = 0; i < hw->nr_vring; i++) {
+		iowrite16(i, &cfg->queue_select);
+		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+				&cfg->queue_desc_hi);
+		io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+				&cfg->queue_avail_hi);
+		io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+				&cfg->queue_used_hi);
+		iowrite16(hw->vring[i].size, &cfg->queue_size);
+
+		*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4) =
+			(u32)hw->vring[i].last_avail_idx |
+			((u32)hw->vring[i].last_used_idx << 16);
+
+		iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
+		if (ioread16(&cfg->queue_msix_vector) ==
+				VIRTIO_MSI_NO_VECTOR) {
+			IFC_ERR(ifcvf->dev,
+				"No msix vector for queue %u.\n", i);
+			return -1;
+		}
+
+		iowrite16(1, &cfg->queue_enable);
+	}
+
+	return 0;
+}
+
+static void ifcvf_hw_disable(struct ifcvf_hw *hw)
+{
+	u32 i;
+	struct virtio_pci_common_cfg *cfg;
+
+	cfg = hw->common_cfg;
+
+	iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
+	for (i = 0; i < hw->nr_vring; i++) {
+		iowrite16(i, &cfg->queue_select);
+		iowrite16(0, &cfg->queue_enable);
+		iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
+	}
+}
+
+int ifcvf_start_hw(struct ifcvf_hw *hw)
+{
+	ifcvf_reset(hw);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER);
+
+	if (ifcvf_config_features(hw) < 0)
+		return -1;
+
+	if (ifcvf_hw_enable(hw) < 0)
+		return -1;
+
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK);
+
+	return 0;
+}
+
+void ifcvf_stop_hw(struct ifcvf_hw *hw)
+{
+	ifcvf_hw_disable(hw);
+	ifcvf_reset(hw);
+}
+
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+		log_base & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =

drivers/vhost/ifcvf/ifcvf_base.c line 365

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+		(log_base >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+		(log_base + log_size) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+		((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}

Is the device using iova or gpa for the logging?

Quoted 120 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+
+void ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
+{
+
+	iowrite16(qid, hw->notify_addr[qid]);
+}
+
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw)
+{
+	return hw->notify_bar;
+}
+
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid)
+{
+	return (u8 *)hw->notify_addr[qid] -
+		(u8 *)hw->mem_resource[hw->notify_bar].addr;
+}
drivers/vhost/ifcvf/ifcvf_base.h new file +89 -0
new file mode 100644
index 000000000000..1ab1a1c40f24
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#ifndef _IFCVF_H_
+#define _IFCVF_H_
+
+#include <linux/virtio_mdev.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_pci.h>
+
+#define IFCVF_VENDOR_ID         0x1AF4
+#define IFCVF_DEVICE_ID         0x1041
+#define IFCVF_SUBSYS_VENDOR_ID  0x8086
+#define IFCVF_SUBSYS_DEVICE_ID  0x001A
+
+/*
+ * Some ifcvf feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. ifcvf_ring), the
+ * rest are per-device feature bits.
+ */
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+		((1ULL << VIRTIO_NET_F_MAC)			| \
+		 (1ULL << VIRTIO_F_ANY_LAYOUT)			| \
+		 (1ULL << VIRTIO_F_VERSION_1)			| \
+		 (1ULL << VHOST_F_LOG_ALL)			| \
+		 (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)		| \
+		 (1ULL << VIRTIO_NET_F_CTRL_VQ)			| \
+		 (1ULL << VIRTIO_NET_F_STATUS)			| \
+		 (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */
+
+#define IFCVF_MAX_QUEUE_PAIRS		1
+#define IFCVF_MAX_QUEUES		2
+
+#define IFCVF_QUEUE_ALIGNMENT		PAGE_SIZE
+
+#define IFCVF_MSI_CONFIG_OFF	0
+#define IFCVF_MSI_QUEUE_OFF	1
+#define IFCVF_PCI_MAX_RESOURCE	6
+
+/* 46 bit CPU physical address, avoid overlap */
+#define LM_IOVA 0x400000000000
+
+#define IFCVF_LM_CFG_SIZE		0x40
+#define IFCVF_LM_RING_STATE_OFFSET	0x20
+
+#define IFCVF_LM_LOGGING_CTRL		0x0
+
+#define IFCVF_LM_BASE_ADDR_LOW		0x10
+#define IFCVF_LM_BASE_ADDR_HIGH		0x14
+#define IFCVF_LM_END_ADDR_LOW		0x18
+#define IFCVF_LM_END_ADDR_HIGH		0x1c
+
+#define IFCVF_LM_DISABLE		0x0
+#define IFCVF_LM_ENABLE_VF		0x1
+#define IFCVF_LM_ENABLE_PF		0x3
+
+#define IFCVF_32_BIT_MASK		0xffffffff
+
+#define IFC_ERR(dev, fmt, ...)	dev_err(dev, fmt, ##__VA_ARGS__)
+#define IFC_INFO(dev, fmt, ...)	dev_info(dev, fmt, ##__VA_ARGS__)
+
+struct ifcvf_net_config {
+	u8    mac[6];
+	u16   status;
+	u16   max_virtqueue_pairs;
+} __packed;
+
+struct ifcvf_pci_mem_resource {
+	u64      phys_addr; /**< Physical address, 0 if not resource. */
+	u64      len;       /**< Length of the resource. */
+	u8       *addr;     /**< Virtual address, NULL when not mapped. */
+};
+
+struct vring_info {
+	u64 desc;
+	u64 avail;
+	u64 used;
+	u16 size;
+	u16 last_avail_idx;
+	u16 last_used_idx;
+	bool ready;

drivers/vhost/ifcvf/ifcvf_base.h line 99

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +10 -0
+	char msix_name[256];
+	struct virtio_mdev_callback cb;
+};
+
+struct ifcvf_hw {
+	u8	*isr;
+	u8	notify_bar;
+	u8	*lm_cfg;
+	u8	status;
+	u8	nr_vring;

Is the the number of queue currently used?

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+	u16	*notify_base;
+	u16	*notify_addr[IFCVF_MAX_QUEUE_PAIRS * 2];
+	u32	generation;
+	u32	notify_off_multiplier;
+	u64	req_features;

drivers/vhost/ifcvf/ifcvf_base.h line 114

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +10 -0
+	struct	virtio_pci_common_cfg *common_cfg;
+	struct	ifcvf_net_config *dev_cfg;
+	struct	vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
+	struct	ifcvf_pci_mem_resource mem_resource[IFCVF_PCI_MAX_RESOURCE];
+};
+
+#define IFC_PRIVATE_TO_VF(adapter) \
+	(&((struct ifcvf_adapter *)adapter)->vf)
+
+#define IFCVF_MAX_INTR (IFCVF_MAX_QUEUE_PAIRS * 2 + 1)

The extra one means the config interrupt?

drivers/vhost/ifcvf/ifcvf_base.h line 118

Quoted 4 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+
+struct ifcvf_adapter {
+	struct	device *dev;
+	struct	mutex mdev_lock;

Not used in the patch, move to next one?

drivers/vhost/ifcvf/ifcvf_base.h line 119

Quoted 1 line
drivers/vhost/ifcvf/ifcvf_base.h new file +1 -0
+	int	mdev_count;

Not used.

drivers/vhost/ifcvf/ifcvf_base.h line 120

Quoted 1 line
drivers/vhost/ifcvf/ifcvf_base.h new file +1 -0
+	struct	list_head dma_maps;

This is not used.

Thanks

Quoted 7 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+	int	vectors;
+	struct	ifcvf_hw vf;
+};
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
+u64 ifcvf_get_features(struct ifcvf_hw *hw);
+int ifcvf_start_hw(struct ifcvf_hw *hw);

drivers/vhost/ifcvf/ifcvf_base.h line 137

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +10 -0
+void ifcvf_stop_hw(struct ifcvf_hw *hw);
+void ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 log_size);
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size);
+void ifcvf_disable_logging(struct ifcvf_hw *hw);
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup);
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw);
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid);
+
+#endif /* _IFCVF_H_ */

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#3 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

In reply to Zhu Lingshan
Quoted 2 lines · Zhu Lingshan

This commit introduced ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations.

It's better to describe the difference between ifc vf and virtio in the commit log or is there a open doc for this?

Quoted 124 lines
---
  drivers/vhost/ifcvf/ifcvf_base.c | 390 +++++++++++++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++++++
  2 files changed, 527 insertions(+)

create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h

drivers/vhost/ifcvf/ifcvf_base.c new file +109 -0
new file mode 100644
index 000000000000..b85e14c9bdcf
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include "ifcvf_base.h"
+
+static void *get_cap_addr(struct ifcvf_hw *hw, struct virtio_pci_cap *cap)
+{
+	u8 bar = cap->bar;
+	u32 length = cap->length;
+	u32 offset = cap->offset;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	if (bar >= IFCVF_PCI_MAX_RESOURCE) {
+		IFC_ERR(ifcvf->dev,
+			"Invalid bar number %u to get capabilities.\n", bar);
+		return NULL;
+	}
+
+	if (offset + length < offset) {
+		IFC_ERR(ifcvf->dev, "offset(%u) + length(%u) overflows\n",
+			offset, length);
+		return NULL;
+	}
+
+	if (offset + length > hw->mem_resource[cap->bar].len) {
+		IFC_ERR(ifcvf->dev,
+			"offset(%u) + len(%u) overflows bar%u to get capabilities.\n",
+			offset, length, bar);
+		return NULL;
+	}
+
+	return hw->mem_resource[bar].addr + offset;
+}
+
+int ifcvf_read_config_range(struct pci_dev *dev,
+			uint32_t *val, int size, int where)
+{
+	int i;
+
+	for (i = 0; i < size; i += 4) {
+		if (pci_read_config_dword(dev, where + i, val + i / 4) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev)
+{
+	int ret;
+	u8 pos;
+	struct virtio_pci_cap cap;
+	u32 i;
+	u16 notify_off;
+
+	ret = pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &pos);
+
+	if (ret < 0) {
+		IFC_ERR(&dev->dev, "failed to read PCI capability list.\n");
+		return -EIO;
+	}
+
+	while (pos) {
+		ret = ifcvf_read_config_range(dev, (u32 *)&cap,
+				sizeof(cap), pos);
+
+		if (ret < 0) {
+			IFC_ERR(&dev->dev, "failed to get PCI capability at %x",
+					pos);
+			break;
+		}
+
+		if (cap.cap_vndr != PCI_CAP_ID_VNDR)
+			goto next;
+
+		IFC_INFO(&dev->dev, "read PCI config:\n"
+					"config type: %u.\n"
+					"PCI bar: %u.\n"
+					"PCI bar offset: %u.\n"
+					"PCI config len: %u.\n",
+					cap.cfg_type, cap.bar,
+					cap.offset, cap.length);
+
+		switch (cap.cfg_type) {
+		case VIRTIO_PCI_CAP_COMMON_CFG:
+			hw->common_cfg = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->common_cfg = %p.\n",
+					hw->common_cfg);
+			break;
+		case VIRTIO_PCI_CAP_NOTIFY_CFG:
+			pci_read_config_dword(dev, pos + sizeof(cap),
+				&hw->notify_off_multiplier);
+			hw->notify_bar = cap.bar;
+			hw->notify_base = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->notify_base = %p.\n",
+					hw->notify_base);
+			break;
+		case VIRTIO_PCI_CAP_ISR_CFG:
+			hw->isr = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->isr = %p.\n", hw->isr);
+			break;
+		case VIRTIO_PCI_CAP_DEVICE_CFG:
+			hw->dev_cfg = get_cap_addr(hw, &cap);
+			IFC_INFO(&dev->dev, "hw->dev_cfg = %p.\n", hw->dev_cfg);
+			break;
+		}
+next:

drivers/vhost/ifcvf/ifcvf_base.c line 119

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+		pos = cap.cap_next;
+	}
+
+	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+		hw->isr == NULL || hw->dev_cfg == NULL) {
+		IFC_ERR(&dev->dev, "Incomplete PCI capabilities.\n");
+		return -1;
+	}
+
+	for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {

Any reason for using hard coded queue pairs limit other than the max_queue_pairs in the net config?

Quoted 29 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +29 -0
+		iowrite16(i, &hw->common_cfg->queue_select);
+		notify_off = ioread16(&hw->common_cfg->queue_notify_off);
+		hw->notify_addr[i] = (void *)((u8 *)hw->notify_base +
+				notify_off * hw->notify_off_multiplier);
+	}
+
+	hw->lm_cfg = hw->mem_resource[4].addr;
+
+	IFC_INFO(&dev->dev, "PCI capability mapping:\n"
+				"common cfg: %p\n"
+				"notify base: %p\n"
+				"isr cfg: %p\n"
+				"device cfg: %p\n"
+				"multiplier: %u\n",
+				hw->common_cfg,
+				hw->notify_base,
+				hw->isr,
+				hw->dev_cfg,
+				hw->notify_off_multiplier);
+
+	return 0;
+}
+
+static u8 ifcvf_get_status(struct ifcvf_hw *hw)
+{
+	return ioread8(&hw->common_cfg->device_status);
+}
+
+static void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)

drivers/vhost/ifcvf/ifcvf_base.c line 158

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+{
+	iowrite8(status, &hw->common_cfg->device_status);
+}
+
+static void ifcvf_reset(struct ifcvf_hw *hw)
+{
+	ifcvf_set_status(hw, 0);
+
+	/* flush status write */
+	ifcvf_get_status(hw);

Why this flush is needed?

Quoted 40 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +40 -0
+	hw->generation++;
+}
+
+static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status)
+{
+	if (status != 0)
+		status |= ifcvf_get_status(hw);
+
+	ifcvf_set_status(hw, status);
+	ifcvf_get_status(hw);
+}
+
+u64 ifcvf_get_features(struct ifcvf_hw *hw)
+{
+	u32 features_lo, features_hi;
+	struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+	iowrite32(0, &cfg->device_feature_select);
+	features_lo = ioread32(&cfg->device_feature);
+
+	iowrite32(1, &cfg->device_feature_select);
+	features_hi = ioread32(&cfg->device_feature);
+
+	return ((u64)features_hi << 32) | features_lo;
+}
+static int ifcvf_with_feature(struct ifcvf_hw *hw, u64 bit)
+{
+	return (hw->req_features & (1ULL << bit)) != 0;
+}
+
+static void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+		       void *dst, int length)
+{
+	int i;
+	u8 *p;
+	u8 old_gen, new_gen;
+
+	do {
+		old_gen = ioread8(&hw->common_cfg->config_generation);
+

drivers/vhost/ifcvf/ifcvf_base.c line 208

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+		p = dst;
+		for (i = 0; i < length; i++)
+			*p++ = ioread8((u8 *)hw->dev_cfg + offset + i);
+
+		new_gen = ioread8(&hw->common_cfg->config_generation);
+	} while (old_gen != new_gen);
+}
+
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup)
+{

Why not just return bollean?

Quoted 25 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+	u16 status;
+	u64 host_features;
+
+	host_features = ifcvf_get_features(hw);
+	if (ifcvf_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+		ifcvf_read_dev_config(hw,
+				offsetof(struct ifcvf_net_config, status),
+				&status, sizeof(status));
+		if ((status & VIRTIO_NET_S_LINK_UP) == 0)
+			(*is_linkup) = 1;
+		else
+			(*is_linkup) = 0;
+	} else
+		(*is_linkup) = 0;
+}
+
+static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+{
+	struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+	iowrite32(0, &cfg->guest_feature_select);
+	iowrite32(features & ((1ULL << 32) - 1), &cfg->guest_feature);
+
+	iowrite32(1, &cfg->guest_feature_select);
+	iowrite32(features >> 32, &cfg->guest_feature);

drivers/vhost/ifcvf/ifcvf_base.c line 243

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+}
+
+static int ifcvf_config_features(struct ifcvf_hw *hw)
+{
+	u64 host_features;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	host_features = ifcvf_get_features(hw);
+	hw->req_features &= host_features;

Is this a must, can't device deal with this?

Quoted 112 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +112 -0
+
+	ifcvf_set_features(hw, hw->req_features);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK);
+
+	if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+		IFC_ERR(ifcvf->dev, "Failed to set FEATURES_OK status\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static void io_write64_twopart(u64 val, u32 *lo, u32 *hi)
+{
+	iowrite32(val & ((1ULL << 32) - 1), lo);
+	iowrite32(val >> 32, hi);
+}
+
+static int ifcvf_hw_enable(struct ifcvf_hw *hw)
+{
+	struct virtio_pci_common_cfg *cfg;
+	u8 *lm_cfg;
+	u32 i;
+	struct ifcvf_adapter *ifcvf =
+		container_of(hw, struct ifcvf_adapter, vf);
+
+	cfg = hw->common_cfg;
+	lm_cfg = hw->lm_cfg;
+
+	iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
+	if (ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
+		IFC_ERR(ifcvf->dev, "No msix vector for device config.\n");
+		return -1;
+	}
+
+	for (i = 0; i < hw->nr_vring; i++) {
+		iowrite16(i, &cfg->queue_select);
+		io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+				&cfg->queue_desc_hi);
+		io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+				&cfg->queue_avail_hi);
+		io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+				&cfg->queue_used_hi);
+		iowrite16(hw->vring[i].size, &cfg->queue_size);
+
+		*(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+				(i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4) =
+			(u32)hw->vring[i].last_avail_idx |
+			((u32)hw->vring[i].last_used_idx << 16);
+
+		iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
+		if (ioread16(&cfg->queue_msix_vector) ==
+				VIRTIO_MSI_NO_VECTOR) {
+			IFC_ERR(ifcvf->dev,
+				"No msix vector for queue %u.\n", i);
+			return -1;
+		}
+
+		iowrite16(1, &cfg->queue_enable);
+	}
+
+	return 0;
+}
+
+static void ifcvf_hw_disable(struct ifcvf_hw *hw)
+{
+	u32 i;
+	struct virtio_pci_common_cfg *cfg;
+
+	cfg = hw->common_cfg;
+
+	iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
+	for (i = 0; i < hw->nr_vring; i++) {
+		iowrite16(i, &cfg->queue_select);
+		iowrite16(0, &cfg->queue_enable);
+		iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
+	}
+}
+
+int ifcvf_start_hw(struct ifcvf_hw *hw)
+{
+	ifcvf_reset(hw);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER);
+
+	if (ifcvf_config_features(hw) < 0)
+		return -1;
+
+	if (ifcvf_hw_enable(hw) < 0)
+		return -1;
+
+	ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK);
+
+	return 0;
+}
+
+void ifcvf_stop_hw(struct ifcvf_hw *hw)
+{
+	ifcvf_hw_disable(hw);
+	ifcvf_reset(hw);
+}
+
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+		log_base & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =

drivers/vhost/ifcvf/ifcvf_base.c line 365

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +10 -0
+		(log_base >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+		(log_base + log_size) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+		((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;
+
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}

Is the device using iova or gpa for the logging?

Quoted 120 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+
+void ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+	u8 *lm_cfg;
+
+	lm_cfg = hw->lm_cfg;
+	*(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
+{
+
+	iowrite16(qid, hw->notify_addr[qid]);
+}
+
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw)
+{
+	return hw->notify_bar;
+}
+
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid)
+{
+	return (u8 *)hw->notify_addr[qid] -
+		(u8 *)hw->mem_resource[hw->notify_bar].addr;
+}
drivers/vhost/ifcvf/ifcvf_base.h new file +89 -0
new file mode 100644
index 000000000000..1ab1a1c40f24
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#ifndef _IFCVF_H_
+#define _IFCVF_H_
+
+#include <linux/virtio_mdev.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_pci.h>
+
+#define IFCVF_VENDOR_ID         0x1AF4
+#define IFCVF_DEVICE_ID         0x1041
+#define IFCVF_SUBSYS_VENDOR_ID  0x8086
+#define IFCVF_SUBSYS_DEVICE_ID  0x001A
+
+/*
+ * Some ifcvf feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. ifcvf_ring), the
+ * rest are per-device feature bits.
+ */
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+		((1ULL << VIRTIO_NET_F_MAC)			| \
+		 (1ULL << VIRTIO_F_ANY_LAYOUT)			| \
+		 (1ULL << VIRTIO_F_VERSION_1)			| \
+		 (1ULL << VHOST_F_LOG_ALL)			| \
+		 (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)		| \
+		 (1ULL << VIRTIO_NET_F_CTRL_VQ)			| \
+		 (1ULL << VIRTIO_NET_F_STATUS)			| \
+		 (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */
+
+#define IFCVF_MAX_QUEUE_PAIRS		1
+#define IFCVF_MAX_QUEUES		2
+
+#define IFCVF_QUEUE_ALIGNMENT		PAGE_SIZE
+
+#define IFCVF_MSI_CONFIG_OFF	0
+#define IFCVF_MSI_QUEUE_OFF	1
+#define IFCVF_PCI_MAX_RESOURCE	6
+
+/* 46 bit CPU physical address, avoid overlap */
+#define LM_IOVA 0x400000000000
+
+#define IFCVF_LM_CFG_SIZE		0x40
+#define IFCVF_LM_RING_STATE_OFFSET	0x20
+
+#define IFCVF_LM_LOGGING_CTRL		0x0
+
+#define IFCVF_LM_BASE_ADDR_LOW		0x10
+#define IFCVF_LM_BASE_ADDR_HIGH		0x14
+#define IFCVF_LM_END_ADDR_LOW		0x18
+#define IFCVF_LM_END_ADDR_HIGH		0x1c
+
+#define IFCVF_LM_DISABLE		0x0
+#define IFCVF_LM_ENABLE_VF		0x1
+#define IFCVF_LM_ENABLE_PF		0x3
+
+#define IFCVF_32_BIT_MASK		0xffffffff
+
+#define IFC_ERR(dev, fmt, ...)	dev_err(dev, fmt, ##__VA_ARGS__)
+#define IFC_INFO(dev, fmt, ...)	dev_info(dev, fmt, ##__VA_ARGS__)
+
+struct ifcvf_net_config {
+	u8    mac[6];
+	u16   status;
+	u16   max_virtqueue_pairs;
+} __packed;
+
+struct ifcvf_pci_mem_resource {
+	u64      phys_addr; /**< Physical address, 0 if not resource. */
+	u64      len;       /**< Length of the resource. */
+	u8       *addr;     /**< Virtual address, NULL when not mapped. */
+};
+
+struct vring_info {
+	u64 desc;
+	u64 avail;
+	u64 used;
+	u16 size;
+	u16 last_avail_idx;
+	u16 last_used_idx;
+	bool ready;

drivers/vhost/ifcvf/ifcvf_base.h line 99

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +10 -0
+	char msix_name[256];
+	struct virtio_mdev_callback cb;
+};
+
+struct ifcvf_hw {
+	u8	*isr;
+	u8	notify_bar;
+	u8	*lm_cfg;
+	u8	status;
+	u8	nr_vring;

Is the the number of queue currently used?

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+	u16	*notify_base;
+	u16	*notify_addr[IFCVF_MAX_QUEUE_PAIRS * 2];
+	u32	generation;
+	u32	notify_off_multiplier;
+	u64	req_features;

drivers/vhost/ifcvf/ifcvf_base.h line 114

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +10 -0
+	struct	virtio_pci_common_cfg *common_cfg;
+	struct	ifcvf_net_config *dev_cfg;
+	struct	vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
+	struct	ifcvf_pci_mem_resource mem_resource[IFCVF_PCI_MAX_RESOURCE];
+};
+
+#define IFC_PRIVATE_TO_VF(adapter) \
+	(&((struct ifcvf_adapter *)adapter)->vf)
+
+#define IFCVF_MAX_INTR (IFCVF_MAX_QUEUE_PAIRS * 2 + 1)

The extra one means the config interrupt?

drivers/vhost/ifcvf/ifcvf_base.h line 118

Quoted 4 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+
+struct ifcvf_adapter {
+	struct	device *dev;
+	struct	mutex mdev_lock;

Not used in the patch, move to next one?

drivers/vhost/ifcvf/ifcvf_base.h line 119

Quoted 1 line
drivers/vhost/ifcvf/ifcvf_base.h new file +1 -0
+	int	mdev_count;

Not used.

drivers/vhost/ifcvf/ifcvf_base.h line 120

Quoted 1 line
drivers/vhost/ifcvf/ifcvf_base.h new file +1 -0
+	struct	list_head dma_maps;

This is not used.

Thanks

Quoted 17 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +17 -0
+	int	vectors;
+	struct	ifcvf_hw vf;
+};
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
+u64 ifcvf_get_features(struct ifcvf_hw *hw);
+int ifcvf_start_hw(struct ifcvf_hw *hw);
+void ifcvf_stop_hw(struct ifcvf_hw *hw);
+void ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 log_size);
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size);
+void ifcvf_disable_logging(struct ifcvf_hw *hw);
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup);
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw);
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid);
+
+#endif /* _IFCVF_H_ */
#4 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

Quoted 9 lines · Jason Wang
Quoted 2 lines · Zhu Lingshan

This commit introduced ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations.

It's better to describe the difference between ifc vf and virtio in the commit log or is there a open doc for this?

Hi Jason,

Sure, I will split these code into small patches with detailed commit logs in v1 patchset.

Quoted 132 lines
Quoted 132 lines

---
  drivers/vhost/ifcvf/ifcvf_base.c | 390
+++++++++++++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++++++

  2 files changed, 527 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h

drivers/vhost/ifcvf/ifcvf_base.c new file +8 -0
new file mode 100644
index 000000000000..b85e14c9bdcf
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include "ifcvf_base.h"
+
+static void *get_cap_addr(struct ifcvf_hw *hw, struct virtio_pci_cap 

*cap)

drivers/vhost/ifcvf/ifcvf_base.c new file +22 -0
+{
+    u8 bar = cap->bar;
+    u32 length = cap->length;
+    u32 offset = cap->offset;
+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf);
+
+    if (bar >= IFCVF_PCI_MAX_RESOURCE) {
+        IFC_ERR(ifcvf->dev,
+            "Invalid bar number %u to get capabilities.\n", bar);
+        return NULL;
+    }
+
+    if (offset + length < offset) {
+        IFC_ERR(ifcvf->dev, "offset(%u) + length(%u) overflows\n",
+            offset, length);
+        return NULL;
+    }
+
+    if (offset + length > hw->mem_resource[cap->bar].len) {
+        IFC_ERR(ifcvf->dev,
+            "offset(%u) + len(%u) overflows bar%u to get 

capabilities.\n",
+            offset, length, bar);
+        return NULL;
+    }

drivers/vhost/ifcvf/ifcvf_base.c new file +81 -0
+
+    return hw->mem_resource[bar].addr + offset;
+}
+
+int ifcvf_read_config_range(struct pci_dev *dev,
+            uint32_t *val, int size, int where)
+{
+    int i;
+
+    for (i = 0; i < size; i += 4) {
+        if (pci_read_config_dword(dev, where + i, val + i / 4) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev)
+{
+    int ret;
+    u8 pos;
+    struct virtio_pci_cap cap;
+    u32 i;
+    u16 notify_off;
+
+    ret = pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &pos);
+
+    if (ret < 0) {
+        IFC_ERR(&dev->dev, "failed to read PCI capability list.\n");
+        return -EIO;
+    }
+
+    while (pos) {
+        ret = ifcvf_read_config_range(dev, (u32 *)&cap,
+                sizeof(cap), pos);
+
+        if (ret < 0) {
+            IFC_ERR(&dev->dev, "failed to get PCI capability at %x",
+                    pos);
+            break;
+        }
+
+        if (cap.cap_vndr != PCI_CAP_ID_VNDR)
+            goto next;
+
+        IFC_INFO(&dev->dev, "read PCI config:\n"
+                    "config type: %u.\n"
+                    "PCI bar: %u.\n"
+                    "PCI bar offset: %u.\n"
+                    "PCI config len: %u.\n",
+                    cap.cfg_type, cap.bar,
+                    cap.offset, cap.length);
+
+        switch (cap.cfg_type) {
+        case VIRTIO_PCI_CAP_COMMON_CFG:
+            hw->common_cfg = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->common_cfg = %p.\n",
+                    hw->common_cfg);
+            break;
+        case VIRTIO_PCI_CAP_NOTIFY_CFG:
+            pci_read_config_dword(dev, pos + sizeof(cap),
+                &hw->notify_off_multiplier);
+            hw->notify_bar = cap.bar;
+            hw->notify_base = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->notify_base = %p.\n",
+                    hw->notify_base);
+            break;
+        case VIRTIO_PCI_CAP_ISR_CFG:
+            hw->isr = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->isr = %p.\n", hw->isr);
+            break;
+        case VIRTIO_PCI_CAP_DEVICE_CFG:
+            hw->dev_cfg = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->dev_cfg = %p.\n", hw->dev_cfg);
+            break;
+        }
+next:
+        pos = cap.cap_next;
+    }
+
+    if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+        hw->isr == NULL || hw->dev_cfg == NULL) {
Quoted 8 lines
Quoted 5 lines

+        IFC_ERR(&dev->dev, "Incomplete PCI capabilities.\n"); +        return -1;
+    }
+
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {

Any reason for using hard coded queue pairs limit other than the max_queue_pairs in the net config?

Hi Jason, Thanks for your kindly comments. For now the driver don't support MQ, we intend to provide a minimal feature sets in this version 1 driver.

Quoted 36 lines
Quoted 36 lines

+        iowrite16(i, &hw->common_cfg->queue_select);
+        notify_off = ioread16(&hw->common_cfg->queue_notify_off); +        hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + +                notify_off * hw->notify_off_multiplier); +    }

drivers/vhost/ifcvf/ifcvf_base.c new file +29 -0
+
+    hw->lm_cfg = hw->mem_resource[4].addr;
+
+    IFC_INFO(&dev->dev, "PCI capability mapping:\n"
+                "common cfg: %p\n"
+                "notify base: %p\n"
+                "isr cfg: %p\n"
+                "device cfg: %p\n"
+                "multiplier: %u\n",
+                hw->common_cfg,
+                hw->notify_base,
+                hw->isr,
+                hw->dev_cfg,
+                hw->notify_off_multiplier);
+
+    return 0;
+}
+
+static u8 ifcvf_get_status(struct ifcvf_hw *hw)
+{
+    return ioread8(&hw->common_cfg->device_status);
+}
+
+static void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
+{
+    iowrite8(status, &hw->common_cfg->device_status);
+}
+
+static void ifcvf_reset(struct ifcvf_hw *hw)
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 158

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +5 -0
+{
+    ifcvf_set_status(hw, 0);
+
+    /* flush status write */
+    ifcvf_get_status(hw);

Why this flush is needed?

accoring to PCIE requirements, this get_status() after a set_status() is used to block the call chain, make sure the hardware has finished the write operation.

It is a bad comment anyway, I will remove it.

Quoted 47 lines
Quoted 47 lines

+    hw->generation++;

drivers/vhost/ifcvf/ifcvf_base.c new file +44 -0
+}
+
+static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status)
+{
+    if (status != 0)
+        status |= ifcvf_get_status(hw);
+
+    ifcvf_set_status(hw, status);
+    ifcvf_get_status(hw);
+}
+
+u64 ifcvf_get_features(struct ifcvf_hw *hw)
+{
+    u32 features_lo, features_hi;
+    struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+    iowrite32(0, &cfg->device_feature_select);
+    features_lo = ioread32(&cfg->device_feature);
+
+    iowrite32(1, &cfg->device_feature_select);
+    features_hi = ioread32(&cfg->device_feature);
+
+    return ((u64)features_hi << 32) | features_lo;
+}
+static int ifcvf_with_feature(struct ifcvf_hw *hw, u64 bit)
+{
+    return (hw->req_features & (1ULL << bit)) != 0;
+}
+
+static void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+               void *dst, int length)
+{
+    int i;
+    u8 *p;
+    u8 old_gen, new_gen;
+
+    do {
+        old_gen = ioread8(&hw->common_cfg->config_generation);
+
+        p = dst;
+        for (i = 0; i < length; i++)
+            *p++ = ioread8((u8 *)hw->dev_cfg + offset + i);
+
+        new_gen = ioread8(&hw->common_cfg->config_generation);
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 208

Quoted 5 lines

+    } while (old_gen != new_gen);

drivers/vhost/ifcvf/ifcvf_base.c new file +4 -0
+}
+
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup)
+{

Why not just return bollean?

sure, can do.

Quoted 32 lines
Quoted 32 lines

+    u16 status;
+    u64 host_features;

drivers/vhost/ifcvf/ifcvf_base.c new file +28 -0
+
+    host_features = ifcvf_get_features(hw);
+    if (ifcvf_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+        ifcvf_read_dev_config(hw,
+                offsetof(struct ifcvf_net_config, status),
+                &status, sizeof(status));
+        if ((status & VIRTIO_NET_S_LINK_UP) == 0)
+            (*is_linkup) = 1;
+        else
+            (*is_linkup) = 0;
+    } else
+        (*is_linkup) = 0;
+}
+
+static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+{
+    struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+    iowrite32(0, &cfg->guest_feature_select);
+    iowrite32(features & ((1ULL << 32) - 1), &cfg->guest_feature);
+
+    iowrite32(1, &cfg->guest_feature_select);
+    iowrite32(features >> 32, &cfg->guest_feature);
+}
+
+static int ifcvf_config_features(struct ifcvf_hw *hw)
+{
+    u64 host_features;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf); +
+    host_features = ifcvf_get_features(hw); +    hw->req_features &= host_features;

Is this a must, can't device deal with this?

I will usehw->req_features directly, thanks for point it out.

Quoted 120 lines
Quoted 120 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +103 -0
+
+    ifcvf_set_features(hw, hw->req_features);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK);
+
+    if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+        IFC_ERR(ifcvf->dev, "Failed to set FEATURES_OK status\n");
+        return -EIO;
+    }
+
+    return 0;
+}
+
+static void io_write64_twopart(u64 val, u32 *lo, u32 *hi)
+{
+    iowrite32(val & ((1ULL << 32) - 1), lo);
+    iowrite32(val >> 32, hi);
+}
+
+static int ifcvf_hw_enable(struct ifcvf_hw *hw)
+{
+    struct virtio_pci_common_cfg *cfg;
+    u8 *lm_cfg;
+    u32 i;
+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf);
+
+    cfg = hw->common_cfg;
+    lm_cfg = hw->lm_cfg;
+
+    iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
+    if (ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
+        IFC_ERR(ifcvf->dev, "No msix vector for device config.\n");
+        return -1;
+    }
+
+    for (i = 0; i < hw->nr_vring; i++) {
+        iowrite16(i, &cfg->queue_select);
+        io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+                &cfg->queue_desc_hi);
+        io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+                &cfg->queue_avail_hi);
+        io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+                &cfg->queue_used_hi);
+        iowrite16(hw->vring[i].size, &cfg->queue_size);
+
+        *(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+                (i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4) =
+            (u32)hw->vring[i].last_avail_idx |
+            ((u32)hw->vring[i].last_used_idx << 16);
+
+        iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
+        if (ioread16(&cfg->queue_msix_vector) ==
+                VIRTIO_MSI_NO_VECTOR) {
+            IFC_ERR(ifcvf->dev,
+                "No msix vector for queue %u.\n", i);
+            return -1;
+        }
+
+        iowrite16(1, &cfg->queue_enable);
+    }
+
+    return 0;
+}
+
+static void ifcvf_hw_disable(struct ifcvf_hw *hw)
+{
+    u32 i;
+    struct virtio_pci_common_cfg *cfg;
+
+    cfg = hw->common_cfg;
+
+    iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
+    for (i = 0; i < hw->nr_vring; i++) {
+        iowrite16(i, &cfg->queue_select);
+        iowrite16(0, &cfg->queue_enable);
+        iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
+    }
+}
+
+int ifcvf_start_hw(struct ifcvf_hw *hw)
+{
+    ifcvf_reset(hw);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER);
+
+    if (ifcvf_config_features(hw) < 0)
+        return -1;
+
+    if (ifcvf_hw_enable(hw) < 0)
+        return -1;
+
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK);
+
+    return 0;
+}
+
+void ifcvf_stop_hw(struct ifcvf_hw *hw)
+{
+    ifcvf_hw_disable(hw);
+    ifcvf_reset(hw);
+}
+
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 

log_size)

drivers/vhost/ifcvf/ifcvf_base.c new file +14 -0
+{
+    u8 *lm_cfg;
+
+    lm_cfg = hw->lm_cfg;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+        log_base & IFCVF_32_BIT_MASK;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =
+        (log_base >> 32) & IFCVF_32_BIT_MASK;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+        (log_base + log_size) & IFCVF_32_BIT_MASK;
+
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 365

Quoted 5 lines

+    *(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+        ((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;

drivers/vhost/ifcvf/ifcvf_base.c new file +3 -0
+
+    *(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}

Is the device using iova or gpa for the logging?

gpa, I will remove all LM related functions since we plan to support LM in next version driver.

Quoted 127 lines
Quoted 127 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+
+void ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+    u8 *lm_cfg;
+
+    lm_cfg = hw->lm_cfg;
+    *(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
+{
+
+    iowrite16(qid, hw->notify_addr[qid]);
+}
+
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw)
+{
+    return hw->notify_bar;
+}
+
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid)
+{
+    return (u8 *)hw->notify_addr[qid] -
+        (u8 *)hw->mem_resource[hw->notify_bar].addr;
+}
drivers/vhost/ifcvf/ifcvf_base.h new file +94 -0
new file mode 100644
index 000000000000..1ab1a1c40f24
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#ifndef _IFCVF_H_
+#define _IFCVF_H_
+
+#include <linux/virtio_mdev.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_pci.h>
+
+#define IFCVF_VENDOR_ID         0x1AF4
+#define IFCVF_DEVICE_ID         0x1041
+#define IFCVF_SUBSYS_VENDOR_ID  0x8086
+#define IFCVF_SUBSYS_DEVICE_ID  0x001A
+
+/*
+ * Some ifcvf feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. ifcvf_ring), the
+ * rest are per-device feature bits.
+ */
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+        ((1ULL << VIRTIO_NET_F_MAC)            | \
+         (1ULL << VIRTIO_F_ANY_LAYOUT)            | \
+         (1ULL << VIRTIO_F_VERSION_1) | \
+         (1ULL << VHOST_F_LOG_ALL)            | \
+         (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)        | \
+         (1ULL << VIRTIO_NET_F_CTRL_VQ)            | \
+         (1ULL << VIRTIO_NET_F_STATUS)            | \
+         (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */
+
+#define IFCVF_MAX_QUEUE_PAIRS        1
+#define IFCVF_MAX_QUEUES        2
+
+#define IFCVF_QUEUE_ALIGNMENT        PAGE_SIZE
+
+#define IFCVF_MSI_CONFIG_OFF    0
+#define IFCVF_MSI_QUEUE_OFF    1
+#define IFCVF_PCI_MAX_RESOURCE    6
+
+/* 46 bit CPU physical address, avoid overlap */
+#define LM_IOVA 0x400000000000
+
+#define IFCVF_LM_CFG_SIZE        0x40
+#define IFCVF_LM_RING_STATE_OFFSET    0x20
+
+#define IFCVF_LM_LOGGING_CTRL        0x0
+
+#define IFCVF_LM_BASE_ADDR_LOW        0x10
+#define IFCVF_LM_BASE_ADDR_HIGH        0x14
+#define IFCVF_LM_END_ADDR_LOW        0x18
+#define IFCVF_LM_END_ADDR_HIGH        0x1c
+
+#define IFCVF_LM_DISABLE        0x0
+#define IFCVF_LM_ENABLE_VF        0x1
+#define IFCVF_LM_ENABLE_PF        0x3
+
+#define IFCVF_32_BIT_MASK        0xffffffff
+
+#define IFC_ERR(dev, fmt, ...)    dev_err(dev, fmt, ##__VA_ARGS__)
+#define IFC_INFO(dev, fmt, ...)    dev_info(dev, fmt, ##__VA_ARGS__)
+
+struct ifcvf_net_config {
+    u8    mac[6];
+    u16   status;
+    u16   max_virtqueue_pairs;
+} __packed;
+
+struct ifcvf_pci_mem_resource {
+    u64      phys_addr; /**< Physical address, 0 if not resource. */
+    u64      len;       /**< Length of the resource. */
+    u8       *addr;     /**< Virtual address, NULL when not mapped. */
+};
+
+struct vring_info {
+    u64 desc;
+    u64 avail;
+    u64 used;
+    u16 size;
+    u16 last_avail_idx;
+    u16 last_used_idx;
+    bool ready;
+    char msix_name[256];
+    struct virtio_mdev_callback cb;
+};
+
+struct ifcvf_hw {
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 99

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+    u8    *isr;
+    u8    notify_bar;
+    u8    *lm_cfg;
+    u8    status;
+    u8    nr_vring;

Is the the number of queue currently used?

Do you mean nr_vring? Yes it is used in hardware enable / disable functions.

Quoted 13 lines
Quoted 13 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +9 -0
+    u16    *notify_base;
+    u16    *notify_addr[IFCVF_MAX_QUEUE_PAIRS * 2];
+    u32    generation;
+    u32    notify_off_multiplier;
+    u64    req_features;
+    struct    virtio_pci_common_cfg *common_cfg;
+    struct    ifcvf_net_config *dev_cfg;
+    struct    vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
+    struct    ifcvf_pci_mem_resource 

mem_resource[IFCVF_PCI_MAX_RESOURCE];
+};

Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 114

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+
+#define IFC_PRIVATE_TO_VF(adapter) \
+    (&((struct ifcvf_adapter *)adapter)->vf)
+
+#define IFCVF_MAX_INTR (IFCVF_MAX_QUEUE_PAIRS * 2 + 1)

The extra one means the config interrupt?

Yes.

Quoted 9 lines

drivers/vhost/ifcvf/ifcvf_base.h line 118

Quoted 6 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+
+struct ifcvf_adapter {
+    struct    device *dev;
+    struct    mutex mdev_lock;

Not used in the patch, move to next one?

Sure, these not used ones will be moved to small patches where they are used in v1 patchset.

Quoted 35 lines
Quoted 3 lines

+    int    mdev_count;

Not used.

Quoted 1 line

+    struct    list_head dma_maps;

This is not used.

Thanks

Quoted 19 lines

+    int    vectors;
+    struct    ifcvf_hw vf;

drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+};
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
+u64 ifcvf_get_features(struct ifcvf_hw *hw);
+int ifcvf_start_hw(struct ifcvf_hw *hw);
+void ifcvf_stop_hw(struct ifcvf_hw *hw);
+void ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 

log_size);
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size);

drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+void ifcvf_disable_logging(struct ifcvf_hw *hw);
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup);
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw);
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid);
+
+#endif /* _IFCVF_H_ */
#5 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

Quoted 154 lines · Lingshan
Quoted 9 lines · Jason Wang
Quoted 2 lines · Zhu Lingshan

This commit introduced ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations.

It's better to describe the difference between ifc vf and virtio in the commit log or is there a open doc for this?

Hi Jason,

Sure, I will split these code into small patches with detailed commit logs in v1 patchset.

Quoted 132 lines
Quoted 132 lines

---
  drivers/vhost/ifcvf/ifcvf_base.c | 390
+++++++++++++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++++++

  2 files changed, 527 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h

drivers/vhost/ifcvf/ifcvf_base.c new file +8 -0
new file mode 100644
index 000000000000..b85e14c9bdcf
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include "ifcvf_base.h"
+
+static void *get_cap_addr(struct ifcvf_hw *hw, struct 

virtio_pci_cap *cap)

drivers/vhost/ifcvf/ifcvf_base.c new file +22 -0
+{
+    u8 bar = cap->bar;
+    u32 length = cap->length;
+    u32 offset = cap->offset;
+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf);
+
+    if (bar >= IFCVF_PCI_MAX_RESOURCE) {
+        IFC_ERR(ifcvf->dev,
+            "Invalid bar number %u to get capabilities.\n", bar);
+        return NULL;
+    }
+
+    if (offset + length < offset) {
+        IFC_ERR(ifcvf->dev, "offset(%u) + length(%u) overflows\n",
+            offset, length);
+        return NULL;
+    }
+
+    if (offset + length > hw->mem_resource[cap->bar].len) {
+        IFC_ERR(ifcvf->dev,
+            "offset(%u) + len(%u) overflows bar%u to get 

capabilities.\n",
+            offset, length, bar);
+        return NULL;
+    }

drivers/vhost/ifcvf/ifcvf_base.c new file +81 -0
+
+    return hw->mem_resource[bar].addr + offset;
+}
+
+int ifcvf_read_config_range(struct pci_dev *dev,
+            uint32_t *val, int size, int where)
+{
+    int i;
+
+    for (i = 0; i < size; i += 4) {
+        if (pci_read_config_dword(dev, where + i, val + i / 4) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev)
+{
+    int ret;
+    u8 pos;
+    struct virtio_pci_cap cap;
+    u32 i;
+    u16 notify_off;
+
+    ret = pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &pos);
+
+    if (ret < 0) {
+        IFC_ERR(&dev->dev, "failed to read PCI capability list.\n");
+        return -EIO;
+    }
+
+    while (pos) {
+        ret = ifcvf_read_config_range(dev, (u32 *)&cap,
+                sizeof(cap), pos);
+
+        if (ret < 0) {
+            IFC_ERR(&dev->dev, "failed to get PCI capability at %x",
+                    pos);
+            break;
+        }
+
+        if (cap.cap_vndr != PCI_CAP_ID_VNDR)
+            goto next;
+
+        IFC_INFO(&dev->dev, "read PCI config:\n"
+                    "config type: %u.\n"
+                    "PCI bar: %u.\n"
+                    "PCI bar offset: %u.\n"
+                    "PCI config len: %u.\n",
+                    cap.cfg_type, cap.bar,
+                    cap.offset, cap.length);
+
+        switch (cap.cfg_type) {
+        case VIRTIO_PCI_CAP_COMMON_CFG:
+            hw->common_cfg = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->common_cfg = %p.\n",
+                    hw->common_cfg);
+            break;
+        case VIRTIO_PCI_CAP_NOTIFY_CFG:
+            pci_read_config_dword(dev, pos + sizeof(cap),
+                &hw->notify_off_multiplier);
+            hw->notify_bar = cap.bar;
+            hw->notify_base = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->notify_base = %p.\n",
+                    hw->notify_base);
+            break;
+        case VIRTIO_PCI_CAP_ISR_CFG:
+            hw->isr = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->isr = %p.\n", hw->isr);
+            break;
+        case VIRTIO_PCI_CAP_DEVICE_CFG:
+            hw->dev_cfg = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->dev_cfg = %p.\n", hw->dev_cfg);
+            break;
+        }
+next:
+        pos = cap.cap_next;
+    }
+
+    if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+        hw->isr == NULL || hw->dev_cfg == NULL) {
Quoted 8 lines
Quoted 5 lines

+        IFC_ERR(&dev->dev, "Incomplete PCI capabilities.\n"); +        return -1;
+    }
+
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {

Any reason for using hard coded queue pairs limit other than the max_queue_pairs in the net config?

Quoted 1 line

Hi Jason, Thanks for your kindly comments. For now the driver don't support MQ, we intend to provide a minimal feature sets in this version 1 driver.

Ok, it's better to add comment above IFCVF_MAX_QUEUE_PAIRS.

Quoted 45 lines
Quoted 36 lines
Quoted 36 lines

+        iowrite16(i, &hw->common_cfg->queue_select);
+        notify_off = ioread16(&hw->common_cfg->queue_notify_off); +        hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + +                notify_off * hw->notify_off_multiplier); +    }

drivers/vhost/ifcvf/ifcvf_base.c new file +29 -0
+
+    hw->lm_cfg = hw->mem_resource[4].addr;
+
+    IFC_INFO(&dev->dev, "PCI capability mapping:\n"
+                "common cfg: %p\n"
+                "notify base: %p\n"
+                "isr cfg: %p\n"
+                "device cfg: %p\n"
+                "multiplier: %u\n",
+                hw->common_cfg,
+                hw->notify_base,
+                hw->isr,
+                hw->dev_cfg,
+                hw->notify_off_multiplier);
+
+    return 0;
+}
+
+static u8 ifcvf_get_status(struct ifcvf_hw *hw)
+{
+    return ioread8(&hw->common_cfg->device_status);
+}
+
+static void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
+{
+    iowrite8(status, &hw->common_cfg->device_status);
+}
+
+static void ifcvf_reset(struct ifcvf_hw *hw)
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 158

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +5 -0
+{
+    ifcvf_set_status(hw, 0);
+
+    /* flush status write */
+    ifcvf_get_status(hw);

Why this flush is needed?

Quoted 3 lines

accoring to PCIE requirements, this get_status() after a set_status() is used to block the call chain, make sure the hardware has finished the write operation.

It is a bad comment anyway, I will remove it.

Interesting, does this mean if we need also fix the vp_set_status for kernel virtio_pci driver?

Quoted 226 lines
Quoted 48 lines
Quoted 48 lines

+    hw->generation++;

drivers/vhost/ifcvf/ifcvf_base.c new file +44 -0
+}
+
+static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status)
+{
+    if (status != 0)
+        status |= ifcvf_get_status(hw);
+
+    ifcvf_set_status(hw, status);
+    ifcvf_get_status(hw);
+}
+
+u64 ifcvf_get_features(struct ifcvf_hw *hw)
+{
+    u32 features_lo, features_hi;
+    struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+    iowrite32(0, &cfg->device_feature_select);
+    features_lo = ioread32(&cfg->device_feature);
+
+    iowrite32(1, &cfg->device_feature_select);
+    features_hi = ioread32(&cfg->device_feature);
+
+    return ((u64)features_hi << 32) | features_lo;
+}
+static int ifcvf_with_feature(struct ifcvf_hw *hw, u64 bit)
+{
+    return (hw->req_features & (1ULL << bit)) != 0;
+}
+
+static void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+               void *dst, int length)
+{
+    int i;
+    u8 *p;
+    u8 old_gen, new_gen;
+
+    do {
+        old_gen = ioread8(&hw->common_cfg->config_generation);
+
+        p = dst;
+        for (i = 0; i < length; i++)
+            *p++ = ioread8((u8 *)hw->dev_cfg + offset + i);
+
+        new_gen = ioread8(&hw->common_cfg->config_generation);
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 208

Quoted 5 lines

+    } while (old_gen != new_gen);

drivers/vhost/ifcvf/ifcvf_base.c new file +4 -0
+}
+
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup)
+{

Why not just return bollean?

sure, can do.

Quoted 32 lines
Quoted 32 lines

+    u16 status;
+    u64 host_features;

drivers/vhost/ifcvf/ifcvf_base.c new file +28 -0
+
+    host_features = ifcvf_get_features(hw);
+    if (ifcvf_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+        ifcvf_read_dev_config(hw,
+                offsetof(struct ifcvf_net_config, status),
+                &status, sizeof(status));
+        if ((status & VIRTIO_NET_S_LINK_UP) == 0)
+            (*is_linkup) = 1;
+        else
+            (*is_linkup) = 0;
+    } else
+        (*is_linkup) = 0;
+}
+
+static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+{
+    struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+    iowrite32(0, &cfg->guest_feature_select);
+    iowrite32(features & ((1ULL << 32) - 1), &cfg->guest_feature);
+
+    iowrite32(1, &cfg->guest_feature_select);
+    iowrite32(features >> 32, &cfg->guest_feature);
+}
+
+static int ifcvf_config_features(struct ifcvf_hw *hw)
+{
+    u64 host_features;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf); +
+    host_features = ifcvf_get_features(hw); +    hw->req_features &= host_features;

Is this a must, can't device deal with this?

I will usehw->req_features directly, thanks for point it out.

Quoted 120 lines
Quoted 120 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +103 -0
+
+    ifcvf_set_features(hw, hw->req_features);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK);
+
+    if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+        IFC_ERR(ifcvf->dev, "Failed to set FEATURES_OK status\n");
+        return -EIO;
+    }
+
+    return 0;
+}
+
+static void io_write64_twopart(u64 val, u32 *lo, u32 *hi)
+{
+    iowrite32(val & ((1ULL << 32) - 1), lo);
+    iowrite32(val >> 32, hi);
+}
+
+static int ifcvf_hw_enable(struct ifcvf_hw *hw)
+{
+    struct virtio_pci_common_cfg *cfg;
+    u8 *lm_cfg;
+    u32 i;
+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf);
+
+    cfg = hw->common_cfg;
+    lm_cfg = hw->lm_cfg;
+
+    iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
+    if (ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
+        IFC_ERR(ifcvf->dev, "No msix vector for device config.\n");
+        return -1;
+    }
+
+    for (i = 0; i < hw->nr_vring; i++) {
+        iowrite16(i, &cfg->queue_select);
+        io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+                &cfg->queue_desc_hi);
+        io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+                &cfg->queue_avail_hi);
+        io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+                &cfg->queue_used_hi);
+        iowrite16(hw->vring[i].size, &cfg->queue_size);
+
+        *(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+                (i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4) =
+            (u32)hw->vring[i].last_avail_idx |
+            ((u32)hw->vring[i].last_used_idx << 16);
+
+        iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
+        if (ioread16(&cfg->queue_msix_vector) ==
+                VIRTIO_MSI_NO_VECTOR) {
+            IFC_ERR(ifcvf->dev,
+                "No msix vector for queue %u.\n", i);
+            return -1;
+        }
+
+        iowrite16(1, &cfg->queue_enable);
+    }
+
+    return 0;
+}
+
+static void ifcvf_hw_disable(struct ifcvf_hw *hw)
+{
+    u32 i;
+    struct virtio_pci_common_cfg *cfg;
+
+    cfg = hw->common_cfg;
+
+    iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
+    for (i = 0; i < hw->nr_vring; i++) {
+        iowrite16(i, &cfg->queue_select);
+        iowrite16(0, &cfg->queue_enable);
+        iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
+    }
+}
+
+int ifcvf_start_hw(struct ifcvf_hw *hw)
+{
+    ifcvf_reset(hw);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER);
+
+    if (ifcvf_config_features(hw) < 0)
+        return -1;
+
+    if (ifcvf_hw_enable(hw) < 0)
+        return -1;
+
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK);
+
+    return 0;
+}
+
+void ifcvf_stop_hw(struct ifcvf_hw *hw)
+{
+    ifcvf_hw_disable(hw);
+    ifcvf_reset(hw);
+}
+
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 

log_size)

drivers/vhost/ifcvf/ifcvf_base.c new file +14 -0
+{
+    u8 *lm_cfg;
+
+    lm_cfg = hw->lm_cfg;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+        log_base & IFCVF_32_BIT_MASK;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =
+        (log_base >> 32) & IFCVF_32_BIT_MASK;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+        (log_base + log_size) & IFCVF_32_BIT_MASK;
+
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 365

Quoted 5 lines

+    *(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+        ((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;

drivers/vhost/ifcvf/ifcvf_base.c new file +3 -0
+
+    *(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}

Is the device using iova or gpa for the logging?

Quoted 1 line

gpa, I will remove all LM related functions since we plan to support LM in next version driver.

Ok, that's why vIOMMU is not fully supported in the case. So we need

1) Filter out _F_IOMMU_PLATFORM for vhost-mdev

2) Can keep it for virtio-mdev

But one more question is: how device know which kinds of address it is used? Or I guess the device doesn't know the only problem is IOVA->GPA conversion in the case of vIOMMU. If this is true, maybe we can introduce API to sync dirty pages and do the conversion there instead of using share memory as the log as what current vhost did.

Quoted 157 lines
Quoted 127 lines
Quoted 127 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+
+void ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+    u8 *lm_cfg;
+
+    lm_cfg = hw->lm_cfg;
+    *(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
+{
+
+    iowrite16(qid, hw->notify_addr[qid]);
+}
+
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw)
+{
+    return hw->notify_bar;
+}
+
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid)
+{
+    return (u8 *)hw->notify_addr[qid] -
+        (u8 *)hw->mem_resource[hw->notify_bar].addr;
+}
drivers/vhost/ifcvf/ifcvf_base.h new file +94 -0
new file mode 100644
index 000000000000..1ab1a1c40f24
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#ifndef _IFCVF_H_
+#define _IFCVF_H_
+
+#include <linux/virtio_mdev.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_pci.h>
+
+#define IFCVF_VENDOR_ID         0x1AF4
+#define IFCVF_DEVICE_ID         0x1041
+#define IFCVF_SUBSYS_VENDOR_ID  0x8086
+#define IFCVF_SUBSYS_DEVICE_ID  0x001A
+
+/*
+ * Some ifcvf feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. ifcvf_ring), the
+ * rest are per-device feature bits.
+ */
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+        ((1ULL << VIRTIO_NET_F_MAC)            | \
+         (1ULL << VIRTIO_F_ANY_LAYOUT)            | \
+         (1ULL << VIRTIO_F_VERSION_1) | \
+         (1ULL << VHOST_F_LOG_ALL)            | \
+         (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)        | \
+         (1ULL << VIRTIO_NET_F_CTRL_VQ)            | \
+         (1ULL << VIRTIO_NET_F_STATUS)            | \
+         (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */
+
+#define IFCVF_MAX_QUEUE_PAIRS        1
+#define IFCVF_MAX_QUEUES        2
+
+#define IFCVF_QUEUE_ALIGNMENT        PAGE_SIZE
+
+#define IFCVF_MSI_CONFIG_OFF    0
+#define IFCVF_MSI_QUEUE_OFF    1
+#define IFCVF_PCI_MAX_RESOURCE    6
+
+/* 46 bit CPU physical address, avoid overlap */
+#define LM_IOVA 0x400000000000
+
+#define IFCVF_LM_CFG_SIZE        0x40
+#define IFCVF_LM_RING_STATE_OFFSET    0x20
+
+#define IFCVF_LM_LOGGING_CTRL        0x0
+
+#define IFCVF_LM_BASE_ADDR_LOW        0x10
+#define IFCVF_LM_BASE_ADDR_HIGH        0x14
+#define IFCVF_LM_END_ADDR_LOW        0x18
+#define IFCVF_LM_END_ADDR_HIGH        0x1c
+
+#define IFCVF_LM_DISABLE        0x0
+#define IFCVF_LM_ENABLE_VF        0x1
+#define IFCVF_LM_ENABLE_PF        0x3
+
+#define IFCVF_32_BIT_MASK        0xffffffff
+
+#define IFC_ERR(dev, fmt, ...)    dev_err(dev, fmt, ##__VA_ARGS__)
+#define IFC_INFO(dev, fmt, ...)    dev_info(dev, fmt, ##__VA_ARGS__)
+
+struct ifcvf_net_config {
+    u8    mac[6];
+    u16   status;
+    u16   max_virtqueue_pairs;
+} __packed;
+
+struct ifcvf_pci_mem_resource {
+    u64      phys_addr; /**< Physical address, 0 if not resource. */
+    u64      len;       /**< Length of the resource. */
+    u8       *addr;     /**< Virtual address, NULL when not mapped. */
+};
+
+struct vring_info {
+    u64 desc;
+    u64 avail;
+    u64 used;
+    u16 size;
+    u16 last_avail_idx;
+    u16 last_used_idx;
+    bool ready;
+    char msix_name[256];
+    struct virtio_mdev_callback cb;
+};
+
+struct ifcvf_hw {
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 99

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+    u8    *isr;
+    u8    notify_bar;
+    u8    *lm_cfg;
+    u8    status;
+    u8    nr_vring;

Is the the number of queue currently used?

Do you mean nr_vring? Yes it is used in hardware enable / disable functions.

Quoted 13 lines
Quoted 13 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +9 -0
+    u16    *notify_base;
+    u16    *notify_addr[IFCVF_MAX_QUEUE_PAIRS * 2];
+    u32    generation;
+    u32    notify_off_multiplier;
+    u64    req_features;
+    struct    virtio_pci_common_cfg *common_cfg;
+    struct    ifcvf_net_config *dev_cfg;
+    struct    vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
+    struct    ifcvf_pci_mem_resource 

mem_resource[IFCVF_PCI_MAX_RESOURCE];
+};

Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 114

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+
+#define IFC_PRIVATE_TO_VF(adapter) \
+    (&((struct ifcvf_adapter *)adapter)->vf)
+
+#define IFCVF_MAX_INTR (IFCVF_MAX_QUEUE_PAIRS * 2 + 1)

The extra one means the config interrupt?

Quoted 1 line

Yes.

Ok, when we support control vq, it should be changed to 2*N + 2.

Thanks

Quoted 37 lines
Quoted 9 lines

drivers/vhost/ifcvf/ifcvf_base.h line 118

Quoted 6 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+
+struct ifcvf_adapter {
+    struct    device *dev;
+    struct    mutex mdev_lock;

Not used in the patch, move to next one?

Sure, these not used ones will be moved to small patches where they are used in v1 patchset.

Quoted 27 lines
Quoted 3 lines

+    int    mdev_count;

Not used.

Quoted 1 line

+    struct    list_head dma_maps;

This is not used.

Thanks

Quoted 11 lines

+    int    vectors;
+    struct    ifcvf_hw vf;

drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+};
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
+u64 ifcvf_get_features(struct ifcvf_hw *hw);
+int ifcvf_start_hw(struct ifcvf_hw *hw);
+void ifcvf_stop_hw(struct ifcvf_hw *hw);
+void ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 

log_size);
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64

Quoted 8 lines
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 137

Quoted 8 lines

log_size);

drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+void ifcvf_disable_logging(struct ifcvf_hw *hw);
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup);
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw);
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid);
+
+#endif /* _IFCVF_H_ */

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#6 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

In reply to Zhu, Lingshan
Quoted 154 lines · Lingshan
Quoted 9 lines · Jason Wang
Quoted 2 lines · Zhu Lingshan

This commit introduced ifcvf_base layer, which handles IFC VF NIC hardware operations and configurations.

It's better to describe the difference between ifc vf and virtio in the commit log or is there a open doc for this?

Hi Jason,

Sure, I will split these code into small patches with detailed commit logs in v1 patchset.

Quoted 132 lines
Quoted 132 lines

---
  drivers/vhost/ifcvf/ifcvf_base.c | 390
+++++++++++++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++++++

  2 files changed, 527 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h

drivers/vhost/ifcvf/ifcvf_base.c new file +8 -0
new file mode 100644
index 000000000000..b85e14c9bdcf
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include "ifcvf_base.h"
+
+static void *get_cap_addr(struct ifcvf_hw *hw, struct 

virtio_pci_cap *cap)

drivers/vhost/ifcvf/ifcvf_base.c new file +22 -0
+{
+    u8 bar = cap->bar;
+    u32 length = cap->length;
+    u32 offset = cap->offset;
+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf);
+
+    if (bar >= IFCVF_PCI_MAX_RESOURCE) {
+        IFC_ERR(ifcvf->dev,
+            "Invalid bar number %u to get capabilities.\n", bar);
+        return NULL;
+    }
+
+    if (offset + length < offset) {
+        IFC_ERR(ifcvf->dev, "offset(%u) + length(%u) overflows\n",
+            offset, length);
+        return NULL;
+    }
+
+    if (offset + length > hw->mem_resource[cap->bar].len) {
+        IFC_ERR(ifcvf->dev,
+            "offset(%u) + len(%u) overflows bar%u to get 

capabilities.\n",
+            offset, length, bar);
+        return NULL;
+    }

drivers/vhost/ifcvf/ifcvf_base.c new file +81 -0
+
+    return hw->mem_resource[bar].addr + offset;
+}
+
+int ifcvf_read_config_range(struct pci_dev *dev,
+            uint32_t *val, int size, int where)
+{
+    int i;
+
+    for (i = 0; i < size; i += 4) {
+        if (pci_read_config_dword(dev, where + i, val + i / 4) < 0)
+            return -1;
+    }
+    return 0;
+}
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev)
+{
+    int ret;
+    u8 pos;
+    struct virtio_pci_cap cap;
+    u32 i;
+    u16 notify_off;
+
+    ret = pci_read_config_byte(dev, PCI_CAPABILITY_LIST, &pos);
+
+    if (ret < 0) {
+        IFC_ERR(&dev->dev, "failed to read PCI capability list.\n");
+        return -EIO;
+    }
+
+    while (pos) {
+        ret = ifcvf_read_config_range(dev, (u32 *)&cap,
+                sizeof(cap), pos);
+
+        if (ret < 0) {
+            IFC_ERR(&dev->dev, "failed to get PCI capability at %x",
+                    pos);
+            break;
+        }
+
+        if (cap.cap_vndr != PCI_CAP_ID_VNDR)
+            goto next;
+
+        IFC_INFO(&dev->dev, "read PCI config:\n"
+                    "config type: %u.\n"
+                    "PCI bar: %u.\n"
+                    "PCI bar offset: %u.\n"
+                    "PCI config len: %u.\n",
+                    cap.cfg_type, cap.bar,
+                    cap.offset, cap.length);
+
+        switch (cap.cfg_type) {
+        case VIRTIO_PCI_CAP_COMMON_CFG:
+            hw->common_cfg = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->common_cfg = %p.\n",
+                    hw->common_cfg);
+            break;
+        case VIRTIO_PCI_CAP_NOTIFY_CFG:
+            pci_read_config_dword(dev, pos + sizeof(cap),
+                &hw->notify_off_multiplier);
+            hw->notify_bar = cap.bar;
+            hw->notify_base = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->notify_base = %p.\n",
+                    hw->notify_base);
+            break;
+        case VIRTIO_PCI_CAP_ISR_CFG:
+            hw->isr = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->isr = %p.\n", hw->isr);
+            break;
+        case VIRTIO_PCI_CAP_DEVICE_CFG:
+            hw->dev_cfg = get_cap_addr(hw, &cap);
+            IFC_INFO(&dev->dev, "hw->dev_cfg = %p.\n", hw->dev_cfg);
+            break;
+        }
+next:
+        pos = cap.cap_next;
+    }
+
+    if (hw->common_cfg == NULL || hw->notify_base == NULL ||
+        hw->isr == NULL || hw->dev_cfg == NULL) {
Quoted 8 lines
Quoted 5 lines

+        IFC_ERR(&dev->dev, "Incomplete PCI capabilities.\n"); +        return -1;
+    }
+
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {

Any reason for using hard coded queue pairs limit other than the max_queue_pairs in the net config?

Quoted 1 line

Hi Jason, Thanks for your kindly comments. For now the driver don't support MQ, we intend to provide a minimal feature sets in this version 1 driver.

Ok, it's better to add comment above IFCVF_MAX_QUEUE_PAIRS.

Quoted 45 lines
Quoted 36 lines
Quoted 36 lines

+        iowrite16(i, &hw->common_cfg->queue_select);
+        notify_off = ioread16(&hw->common_cfg->queue_notify_off); +        hw->notify_addr[i] = (void *)((u8 *)hw->notify_base + +                notify_off * hw->notify_off_multiplier); +    }

drivers/vhost/ifcvf/ifcvf_base.c new file +29 -0
+
+    hw->lm_cfg = hw->mem_resource[4].addr;
+
+    IFC_INFO(&dev->dev, "PCI capability mapping:\n"
+                "common cfg: %p\n"
+                "notify base: %p\n"
+                "isr cfg: %p\n"
+                "device cfg: %p\n"
+                "multiplier: %u\n",
+                hw->common_cfg,
+                hw->notify_base,
+                hw->isr,
+                hw->dev_cfg,
+                hw->notify_off_multiplier);
+
+    return 0;
+}
+
+static u8 ifcvf_get_status(struct ifcvf_hw *hw)
+{
+    return ioread8(&hw->common_cfg->device_status);
+}
+
+static void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
+{
+    iowrite8(status, &hw->common_cfg->device_status);
+}
+
+static void ifcvf_reset(struct ifcvf_hw *hw)
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 158

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +5 -0
+{
+    ifcvf_set_status(hw, 0);
+
+    /* flush status write */
+    ifcvf_get_status(hw);

Why this flush is needed?

Quoted 3 lines

accoring to PCIE requirements, this get_status() after a set_status() is used to block the call chain, make sure the hardware has finished the write operation.

It is a bad comment anyway, I will remove it.

Interesting, does this mean if we need also fix the vp_set_status for kernel virtio_pci driver?

Quoted 226 lines
Quoted 48 lines
Quoted 48 lines

+    hw->generation++;

drivers/vhost/ifcvf/ifcvf_base.c new file +44 -0
+}
+
+static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status)
+{
+    if (status != 0)
+        status |= ifcvf_get_status(hw);
+
+    ifcvf_set_status(hw, status);
+    ifcvf_get_status(hw);
+}
+
+u64 ifcvf_get_features(struct ifcvf_hw *hw)
+{
+    u32 features_lo, features_hi;
+    struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+    iowrite32(0, &cfg->device_feature_select);
+    features_lo = ioread32(&cfg->device_feature);
+
+    iowrite32(1, &cfg->device_feature_select);
+    features_hi = ioread32(&cfg->device_feature);
+
+    return ((u64)features_hi << 32) | features_lo;
+}
+static int ifcvf_with_feature(struct ifcvf_hw *hw, u64 bit)
+{
+    return (hw->req_features & (1ULL << bit)) != 0;
+}
+
+static void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset,
+               void *dst, int length)
+{
+    int i;
+    u8 *p;
+    u8 old_gen, new_gen;
+
+    do {
+        old_gen = ioread8(&hw->common_cfg->config_generation);
+
+        p = dst;
+        for (i = 0; i < length; i++)
+            *p++ = ioread8((u8 *)hw->dev_cfg + offset + i);
+
+        new_gen = ioread8(&hw->common_cfg->config_generation);
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 208

Quoted 5 lines

+    } while (old_gen != new_gen);

drivers/vhost/ifcvf/ifcvf_base.c new file +4 -0
+}
+
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup)
+{

Why not just return bollean?

sure, can do.

Quoted 32 lines
Quoted 32 lines

+    u16 status;
+    u64 host_features;

drivers/vhost/ifcvf/ifcvf_base.c new file +28 -0
+
+    host_features = ifcvf_get_features(hw);
+    if (ifcvf_with_feature(hw, VIRTIO_NET_F_STATUS)) {
+        ifcvf_read_dev_config(hw,
+                offsetof(struct ifcvf_net_config, status),
+                &status, sizeof(status));
+        if ((status & VIRTIO_NET_S_LINK_UP) == 0)
+            (*is_linkup) = 1;
+        else
+            (*is_linkup) = 0;
+    } else
+        (*is_linkup) = 0;
+}
+
+static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
+{
+    struct virtio_pci_common_cfg *cfg = hw->common_cfg;
+
+    iowrite32(0, &cfg->guest_feature_select);
+    iowrite32(features & ((1ULL << 32) - 1), &cfg->guest_feature);
+
+    iowrite32(1, &cfg->guest_feature_select);
+    iowrite32(features >> 32, &cfg->guest_feature);
+}
+
+static int ifcvf_config_features(struct ifcvf_hw *hw)
+{
+    u64 host_features;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf); +
+    host_features = ifcvf_get_features(hw); +    hw->req_features &= host_features;

Is this a must, can't device deal with this?

I will usehw->req_features directly, thanks for point it out.

Quoted 120 lines
Quoted 120 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +103 -0
+
+    ifcvf_set_features(hw, hw->req_features);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK);
+
+    if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) {
+        IFC_ERR(ifcvf->dev, "Failed to set FEATURES_OK status\n");
+        return -EIO;
+    }
+
+    return 0;
+}
+
+static void io_write64_twopart(u64 val, u32 *lo, u32 *hi)
+{
+    iowrite32(val & ((1ULL << 32) - 1), lo);
+    iowrite32(val >> 32, hi);
+}
+
+static int ifcvf_hw_enable(struct ifcvf_hw *hw)
+{
+    struct virtio_pci_common_cfg *cfg;
+    u8 *lm_cfg;
+    u32 i;
+    struct ifcvf_adapter *ifcvf =
+        container_of(hw, struct ifcvf_adapter, vf);
+
+    cfg = hw->common_cfg;
+    lm_cfg = hw->lm_cfg;
+
+    iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
+    if (ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
+        IFC_ERR(ifcvf->dev, "No msix vector for device config.\n");
+        return -1;
+    }
+
+    for (i = 0; i < hw->nr_vring; i++) {
+        iowrite16(i, &cfg->queue_select);
+        io_write64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
+                &cfg->queue_desc_hi);
+        io_write64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
+                &cfg->queue_avail_hi);
+        io_write64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
+                &cfg->queue_used_hi);
+        iowrite16(hw->vring[i].size, &cfg->queue_size);
+
+        *(u32 *)(lm_cfg + IFCVF_LM_RING_STATE_OFFSET +
+                (i / 2) * IFCVF_LM_CFG_SIZE + (i % 2) * 4) =
+            (u32)hw->vring[i].last_avail_idx |
+            ((u32)hw->vring[i].last_used_idx << 16);
+
+        iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
+        if (ioread16(&cfg->queue_msix_vector) ==
+                VIRTIO_MSI_NO_VECTOR) {
+            IFC_ERR(ifcvf->dev,
+                "No msix vector for queue %u.\n", i);
+            return -1;
+        }
+
+        iowrite16(1, &cfg->queue_enable);
+    }
+
+    return 0;
+}
+
+static void ifcvf_hw_disable(struct ifcvf_hw *hw)
+{
+    u32 i;
+    struct virtio_pci_common_cfg *cfg;
+
+    cfg = hw->common_cfg;
+
+    iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
+    for (i = 0; i < hw->nr_vring; i++) {
+        iowrite16(i, &cfg->queue_select);
+        iowrite16(0, &cfg->queue_enable);
+        iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
+    }
+}
+
+int ifcvf_start_hw(struct ifcvf_hw *hw)
+{
+    ifcvf_reset(hw);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE);
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER);
+
+    if (ifcvf_config_features(hw) < 0)
+        return -1;
+
+    if (ifcvf_hw_enable(hw) < 0)
+        return -1;
+
+    ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK);
+
+    return 0;
+}
+
+void ifcvf_stop_hw(struct ifcvf_hw *hw)
+{
+    ifcvf_hw_disable(hw);
+    ifcvf_reset(hw);
+}
+
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 

log_size)

drivers/vhost/ifcvf/ifcvf_base.c new file +14 -0
+{
+    u8 *lm_cfg;
+
+    lm_cfg = hw->lm_cfg;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_LOW) =
+        log_base & IFCVF_32_BIT_MASK;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_BASE_ADDR_HIGH) =
+        (log_base >> 32) & IFCVF_32_BIT_MASK;
+
+    *(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_LOW) =
+        (log_base + log_size) & IFCVF_32_BIT_MASK;
+
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.c line 365

Quoted 5 lines

+    *(u32 *)(lm_cfg + IFCVF_LM_END_ADDR_HIGH) =
+        ((log_base + log_size) >> 32) & IFCVF_32_BIT_MASK;

drivers/vhost/ifcvf/ifcvf_base.c new file +3 -0
+
+    *(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_ENABLE_VF;
+}

Is the device using iova or gpa for the logging?

Quoted 1 line

gpa, I will remove all LM related functions since we plan to support LM in next version driver.

Ok, that's why vIOMMU is not fully supported in the case. So we need

1) Filter out _F_IOMMU_PLATFORM for vhost-mdev

2) Can keep it for virtio-mdev

But one more question is: how device know which kinds of address it is used? Or I guess the device doesn't know the only problem is IOVA->GPA conversion in the case of vIOMMU. If this is true, maybe we can introduce API to sync dirty pages and do the conversion there instead of using share memory as the log as what current vhost did.

Quoted 157 lines
Quoted 127 lines
Quoted 127 lines
drivers/vhost/ifcvf/ifcvf_base.c new file +25 -0
+
+void ifcvf_disable_logging(struct ifcvf_hw *hw)
+{
+    u8 *lm_cfg;
+
+    lm_cfg = hw->lm_cfg;
+    *(u32 *)(lm_cfg + IFCVF_LM_LOGGING_CTRL) = IFCVF_LM_DISABLE;
+}
+
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
+{
+
+    iowrite16(qid, hw->notify_addr[qid]);
+}
+
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw)
+{
+    return hw->notify_bar;
+}
+
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid)
+{
+    return (u8 *)hw->notify_addr[qid] -
+        (u8 *)hw->mem_resource[hw->notify_bar].addr;
+}
drivers/vhost/ifcvf/ifcvf_base.h new file +94 -0
new file mode 100644
index 000000000000..1ab1a1c40f24
@@ -0,0 +1,137 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#ifndef _IFCVF_H_
+#define _IFCVF_H_
+
+#include <linux/virtio_mdev.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <uapi/linux/virtio_net.h>
+#include <uapi/linux/virtio_config.h>
+#include <uapi/linux/virtio_pci.h>
+
+#define IFCVF_VENDOR_ID         0x1AF4
+#define IFCVF_DEVICE_ID         0x1041
+#define IFCVF_SUBSYS_VENDOR_ID  0x8086
+#define IFCVF_SUBSYS_DEVICE_ID  0x001A
+
+/*
+ * Some ifcvf feature bits (currently bits 28 through 31) are
+ * reserved for the transport being used (eg. ifcvf_ring), the
+ * rest are per-device feature bits.
+ */
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+        ((1ULL << VIRTIO_NET_F_MAC)            | \
+         (1ULL << VIRTIO_F_ANY_LAYOUT)            | \
+         (1ULL << VIRTIO_F_VERSION_1) | \
+         (1ULL << VHOST_F_LOG_ALL)            | \
+         (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)        | \
+         (1ULL << VIRTIO_NET_F_CTRL_VQ)            | \
+         (1ULL << VIRTIO_NET_F_STATUS)            | \
+         (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */
+
+#define IFCVF_MAX_QUEUE_PAIRS        1
+#define IFCVF_MAX_QUEUES        2
+
+#define IFCVF_QUEUE_ALIGNMENT        PAGE_SIZE
+
+#define IFCVF_MSI_CONFIG_OFF    0
+#define IFCVF_MSI_QUEUE_OFF    1
+#define IFCVF_PCI_MAX_RESOURCE    6
+
+/* 46 bit CPU physical address, avoid overlap */
+#define LM_IOVA 0x400000000000
+
+#define IFCVF_LM_CFG_SIZE        0x40
+#define IFCVF_LM_RING_STATE_OFFSET    0x20
+
+#define IFCVF_LM_LOGGING_CTRL        0x0
+
+#define IFCVF_LM_BASE_ADDR_LOW        0x10
+#define IFCVF_LM_BASE_ADDR_HIGH        0x14
+#define IFCVF_LM_END_ADDR_LOW        0x18
+#define IFCVF_LM_END_ADDR_HIGH        0x1c
+
+#define IFCVF_LM_DISABLE        0x0
+#define IFCVF_LM_ENABLE_VF        0x1
+#define IFCVF_LM_ENABLE_PF        0x3
+
+#define IFCVF_32_BIT_MASK        0xffffffff
+
+#define IFC_ERR(dev, fmt, ...)    dev_err(dev, fmt, ##__VA_ARGS__)
+#define IFC_INFO(dev, fmt, ...)    dev_info(dev, fmt, ##__VA_ARGS__)
+
+struct ifcvf_net_config {
+    u8    mac[6];
+    u16   status;
+    u16   max_virtqueue_pairs;
+} __packed;
+
+struct ifcvf_pci_mem_resource {
+    u64      phys_addr; /**< Physical address, 0 if not resource. */
+    u64      len;       /**< Length of the resource. */
+    u8       *addr;     /**< Virtual address, NULL when not mapped. */
+};
+
+struct vring_info {
+    u64 desc;
+    u64 avail;
+    u64 used;
+    u16 size;
+    u16 last_avail_idx;
+    u16 last_used_idx;
+    bool ready;
+    char msix_name[256];
+    struct virtio_mdev_callback cb;
+};
+
+struct ifcvf_hw {
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 99

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+    u8    *isr;
+    u8    notify_bar;
+    u8    *lm_cfg;
+    u8    status;
+    u8    nr_vring;

Is the the number of queue currently used?

Do you mean nr_vring? Yes it is used in hardware enable / disable functions.

Quoted 13 lines
Quoted 13 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +9 -0
+    u16    *notify_base;
+    u16    *notify_addr[IFCVF_MAX_QUEUE_PAIRS * 2];
+    u32    generation;
+    u32    notify_off_multiplier;
+    u64    req_features;
+    struct    virtio_pci_common_cfg *common_cfg;
+    struct    ifcvf_net_config *dev_cfg;
+    struct    vring_info vring[IFCVF_MAX_QUEUE_PAIRS * 2];
+    struct    ifcvf_pci_mem_resource 

mem_resource[IFCVF_PCI_MAX_RESOURCE];
+};

Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_base.h line 114

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +5 -0
+
+#define IFC_PRIVATE_TO_VF(adapter) \
+    (&((struct ifcvf_adapter *)adapter)->vf)
+
+#define IFCVF_MAX_INTR (IFCVF_MAX_QUEUE_PAIRS * 2 + 1)

The extra one means the config interrupt?

Quoted 1 line

Yes.

Ok, when we support control vq, it should be changed to 2*N + 2.

Thanks

Quoted 45 lines
Quoted 9 lines

drivers/vhost/ifcvf/ifcvf_base.h line 118

Quoted 6 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+
+struct ifcvf_adapter {
+    struct    device *dev;
+    struct    mutex mdev_lock;

Not used in the patch, move to next one?

Sure, these not used ones will be moved to small patches where they are used in v1 patchset.

Quoted 35 lines
Quoted 3 lines

+    int    mdev_count;

Not used.

Quoted 1 line

+    struct    list_head dma_maps;

This is not used.

Thanks

Quoted 19 lines

+    int    vectors;
+    struct    ifcvf_hw vf;

drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+};
+
+int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev);
+u64 ifcvf_get_features(struct ifcvf_hw *hw);
+int ifcvf_start_hw(struct ifcvf_hw *hw);
+void ifcvf_stop_hw(struct ifcvf_hw *hw);
+void ifcvf_enable_logging(struct ifcvf_hw *hw, u64 log_base, u64 

log_size);
+void ifcvf_enable_logging_vf(struct ifcvf_hw *hw, u64 log_base, u64 log_size);

drivers/vhost/ifcvf/ifcvf_base.h new file +7 -0
+void ifcvf_disable_logging(struct ifcvf_hw *hw);
+void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid);
+void ifcvf_get_linkstatus(struct ifcvf_hw *hw, u8 *is_linkup);
+u8 ifcvf_get_notify_region(struct ifcvf_hw *hw);
+u64 ifcvf_get_queue_notify_off(struct ifcvf_hw *hw, int qid);
+
+#endif /* _IFCVF_H_ */
#7 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

In reply to Zhu Lingshan

drivers/vhost/ifcvf/ifcvf_base.h line 33

Quoted 9 lines · Zhu Lingshan

+ */

drivers/vhost/ifcvf/ifcvf_base.h new file +8 -0
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+		((1ULL << VIRTIO_NET_F_MAC)			| \
+		 (1ULL << VIRTIO_F_ANY_LAYOUT)			| \
+		 (1ULL << VIRTIO_F_VERSION_1)			| \
+		 (1ULL << VHOST_F_LOG_ALL)			| \

Let's avoid using VHOST_F_LOG_ALL, using the get_mdev_features() instead.

drivers/vhost/ifcvf/ifcvf_base.h line 37

Quoted 4 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+		 (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)		| \
+		 (1ULL << VIRTIO_NET_F_CTRL_VQ)			| \
+		 (1ULL << VIRTIO_NET_F_STATUS)			| \
+		 (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */

Why not having VIRTIO_F_IOMMU_PLATFORM and VIRTIO_F_ORDER_PLATFORM?

Thanks

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#8 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

In reply to Zhu Lingshan

drivers/vhost/ifcvf/ifcvf_base.h line 33

Quoted 9 lines · Zhu Lingshan

+ */

drivers/vhost/ifcvf/ifcvf_base.h new file +8 -0
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+		((1ULL << VIRTIO_NET_F_MAC)			| \
+		 (1ULL << VIRTIO_F_ANY_LAYOUT)			| \
+		 (1ULL << VIRTIO_F_VERSION_1)			| \
+		 (1ULL << VHOST_F_LOG_ALL)			| \

Let's avoid using VHOST_F_LOG_ALL, using the get_mdev_features() instead.

drivers/vhost/ifcvf/ifcvf_base.h line 37

Quoted 4 lines
drivers/vhost/ifcvf/ifcvf_base.h new file +4 -0
+		 (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)		| \
+		 (1ULL << VIRTIO_NET_F_CTRL_VQ)			| \
+		 (1ULL << VIRTIO_NET_F_STATUS)			| \
+		 (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */

Why not having VIRTIO_F_IOMMU_PLATFORM and VIRTIO_F_ORDER_PLATFORM?

Thanks

#9 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

Quoted 14 lines · Jason Wang

drivers/vhost/ifcvf/ifcvf_base.h line 33

Quoted 9 lines · Zhu Lingshan

+ */

drivers/vhost/ifcvf/ifcvf_base.h new file +8 -0
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+        ((1ULL << VIRTIO_NET_F_MAC)            | \
+         (1ULL << VIRTIO_F_ANY_LAYOUT)            | \
+         (1ULL << VIRTIO_F_VERSION_1) | \
+         (1ULL << VHOST_F_LOG_ALL)            | \

Let's avoid using VHOST_F_LOG_ALL, using the get_mdev_features() instead.

Thanks, I will remove VHOST_F_LOG_ALL

Quoted 9 lines
Quoted 6 lines

+         (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)        | \
+         (1ULL << VIRTIO_NET_F_CTRL_VQ)            | \
+         (1ULL << VIRTIO_NET_F_STATUS)            | \
+         (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */

Why not having VIRTIO_F_IOMMU_PLATFORM and VIRTIO_F_ORDER_PLATFORM?

I will add VIRTIO_F_ORDER_PLATFORM, for VIRTIO_F_IOMMU_PLATFORM, if we add this bit, QEMU may enable viommu, can cause troubles in LM (through we don't support LM in this version driver)

Thanks,
BR
Zhu Lingshan

Quoted 3 lines

Thanks

#10 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

Quoted 25 lines · Lingshan
Quoted 14 lines · Jason Wang

drivers/vhost/ifcvf/ifcvf_base.h line 33

Quoted 9 lines · Zhu Lingshan

+ */

drivers/vhost/ifcvf/ifcvf_base.h new file +8 -0
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+        ((1ULL << VIRTIO_NET_F_MAC)            | \
+         (1ULL << VIRTIO_F_ANY_LAYOUT)            | \
+         (1ULL << VIRTIO_F_VERSION_1) | \
+         (1ULL << VHOST_F_LOG_ALL)            | \

Let's avoid using VHOST_F_LOG_ALL, using the get_mdev_features() instead.

Thanks, I will remove VHOST_F_LOG_ALL

Quoted 8 lines
Quoted 6 lines

+         (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)        | \
+         (1ULL << VIRTIO_NET_F_CTRL_VQ)            | \
+         (1ULL << VIRTIO_NET_F_STATUS)            | \
+         (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */

Quoted 3 lines
Quoted 1 line

Why not having VIRTIO_F_IOMMU_PLATFORM and VIRTIO_F_ORDER_PLATFORM?

I will add VIRTIO_F_ORDER_PLATFORM, for VIRTIO_F_IOMMU_PLATFORM, if we add this bit, QEMU may enable viommu, can cause troubles in LM

Qemu has mature support of vIOMMU support for VFIO device, it can shadow IO page tables and setup them through DMA ioctl of vfio containers. Any issue you saw here?

Btw, to test them quickly, you can implement set_config/get_config and test them through virtio-mdev/kernel drivers as well.

Thanks

Quoted 8 lines

(through we don't support LM in this version driver)

Thanks,
BR
Zhu Lingshan

Quoted 3 lines

Thanks

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#11 lore raw

Re: [RFC 1/2] vhost: IFC VF hardware operation layer

In reply to Zhu, Lingshan
Quoted 25 lines · Lingshan
Quoted 14 lines · Jason Wang

drivers/vhost/ifcvf/ifcvf_base.h line 33

Quoted 9 lines · Zhu Lingshan

+ */

drivers/vhost/ifcvf/ifcvf_base.h new file +8 -0
+#define IFCVF_TRANSPORT_F_START 28
+#define IFCVF_TRANSPORT_F_END   34
+
+#define IFC_SUPPORTED_FEATURES \
+        ((1ULL << VIRTIO_NET_F_MAC)            | \
+         (1ULL << VIRTIO_F_ANY_LAYOUT)            | \
+         (1ULL << VIRTIO_F_VERSION_1) | \
+         (1ULL << VHOST_F_LOG_ALL)            | \

Let's avoid using VHOST_F_LOG_ALL, using the get_mdev_features() instead.

Thanks, I will remove VHOST_F_LOG_ALL

Quoted 8 lines
Quoted 6 lines

+         (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE)        | \
+         (1ULL << VIRTIO_NET_F_CTRL_VQ)            | \
+         (1ULL << VIRTIO_NET_F_STATUS)            | \
+         (1ULL << VIRTIO_NET_F_MRG_RXBUF)) /* not fully supported */

Quoted 3 lines
Quoted 1 line

Why not having VIRTIO_F_IOMMU_PLATFORM and VIRTIO_F_ORDER_PLATFORM?

I will add VIRTIO_F_ORDER_PLATFORM, for VIRTIO_F_IOMMU_PLATFORM, if we add this bit, QEMU may enable viommu, can cause troubles in LM

Qemu has mature support of vIOMMU support for VFIO device, it can shadow IO page tables and setup them through DMA ioctl of vfio containers. Any issue you saw here?

Btw, to test them quickly, you can implement set_config/get_config and test them through virtio-mdev/kernel drivers as well.

Thanks

Quoted 8 lines

(through we don't support LM in this version driver)

Thanks,
BR
Zhu Lingshan

Quoted 3 lines

Thanks

#12 lore raw

RFC 2/2 vhost: IFC VF vdpa layer

In reply to Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
 drivers/vhost/ifcvf/ifcvf_main.c | 541 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 541 insertions(+)
 create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c
drivers/vhost/ifcvf/ifcvf_main.c new file +541 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING	"0.1"
+#define DRIVER_AUTHOR	"Intel Corporation"
+#define IFCVF_DRIVER_NAME	"ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+	struct vring_info *vring = arg;
+
+	if (vring->cb.callback)
+		return vring->cb.callback(vring->cb.private);
+
+	return IRQ_HANDLED;
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+	return IFC_SUPPORTED_FEATURES;
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 features)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->req_features = features;
+
+	return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->vring[qid].last_avail_idx;
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 qid, u64 num)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].last_used_idx = num;
+	vf->vring[qid].last_avail_idx = num;
+
+	return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 idx,
+				     u64 desc_area, u64 driver_area,
+				     u64 device_area)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[idx].desc = desc_area;
+	vf->vring[idx].avail = driver_area;
+	vf->vring[idx].used = device_area;
+
+	return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 qid, u32 num)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+				u16 qid, bool ready)
+{
+
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+				 struct virtio_mdev_callback *cb)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+	return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+	return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+	return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+	return IFCVF_QUEUE_ALIGNMENT;
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+	int i, ret;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+
+	for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {
+		if (!vf->vring[i].ready)
+			break;
+
+		if (!vf->vring[i].size)
+			break;
+
+		if (!vf->vring[i].desc || !vf->vring[i].avail ||
+			!vf->vring[i].used)
+			break;
+	}
+	vf->nr_vring = i;
+
+	ret = ifcvf_start_hw(vf);
+	return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+	int i;
+
+	for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+		vf->vring[i].cb.callback = NULL;
+
+	ifcvf_stop_hw(vf);
+
+	return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+	int i;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vf->vring[i].last_used_idx = 0;
+		vf->vring[i].last_avail_idx = 0;
+		vf->vring[i].desc = 0;
+		vf->vring[i].avail = 0;
+		vf->vring[i].used = 0;
+		vf->vring[i].ready = 0;
+		vf->vring->cb.callback = NULL;
+		vf->vring->cb.private = NULL;
+	}
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->status = status;
+
+	if (status == 0) {
+		ifcvf_stop_datapath(adapter);
+		ifcvf_reset_vring(adapter);
+		return;
+	}
+
+	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+		ifcvf_start_datapath(adapter);
+		return;
+	}
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+	return IFCVF_MAX_QUEUES;
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+	.get_features  = ifcvf_mdev_get_features,
+	.set_features  = ifcvf_mdev_set_features,
+	.get_status    = ifcvf_mdev_get_status,
+	.set_status    = ifcvf_mdev_set_status,
+	.get_queue_max = ifcvf_mdev_get_queue_max,
+	.get_vq_state   = ifcvf_mdev_get_vq_state,
+	.set_vq_state   = ifcvf_mdev_set_vq_state,
+	.set_vq_cb      = ifcvf_mdev_set_vq_cb,
+	.set_vq_ready   = ifcvf_mdev_set_vq_ready,
+	.get_vq_ready	= ifcvf_mdev_get_vq_ready,
+	.set_vq_num     = ifcvf_mdev_set_vq_num,
+	.set_vq_address = ifcvf_mdev_set_vq_address,
+	.kick_vq        = ifcvf_mdev_kick_vq,
+	.get_generation	= ifcvf_mdev_get_generation,
+	.get_version	= ifcvf_mdev_get_version,
+	.get_device_id	= ifcvf_mdev_get_device_id,
+	.get_vendor_id	= ifcvf_mdev_get_vendor_id,
+	.get_vq_align	= ifcvf_mdev_get_vq_align,
+};
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+	int vector, i, ret, irq;
+	struct pci_dev *pdev = to_pci_dev(adapter->dev);
+	struct ifcvf_hw *vf = &adapter->vf;
+
+	ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+			IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+	if (ret < 0) {
+		IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+		return ret;
+	}
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		ret = request_irq(irq, ifcvf_intr_handler, 0,
+				pci_name(pdev), &vf->vring[i]);
+		if (ret) {
+			IFC_ERR(adapter->dev,
+				"Failed to request irq for vq %d.\n", i);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+	int i, vector, irq;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+	struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		free_irq(irq, &vf->vring[i]);
+	}
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+	const char *name = "vhost accelerator (virtio ring compatible)";
+
+	return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+			       char *buf)
+{
+	return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+					struct device *dev, char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+	return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+			struct device *dev, char *buf)
+{
+	return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+
+static struct attribute *mdev_types_attrs[] = {
+	&mdev_type_attr_name.attr,
+	&mdev_type_attr_device_api.attr,
+	&mdev_type_attr_available_instances.attr,
+	&mdev_type_attr_type.attr,
+	NULL,
+};
+
+static struct attribute_group mdev_type_group = {
+	.name  = "vdpa_virtio",
+	.attrs = mdev_types_attrs,
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+	&mdev_type_group,
+	NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+	NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
+{
+	struct device *dev = mdev_parent_dev(mdev);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	int ret = 0;
+
+	mutex_lock(&adapter->mdev_lock);
+
+	if (adapter->mdev_count < 1) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	mdev_set_class_id(mdev, MDEV_ID_VHOST);
+	mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+	mdev_set_drvdata(mdev, adapter);
+	mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+	INIT_LIST_HEAD(&adapter->dma_maps);
+	adapter->mdev_count--;
+
+out:
+	mutex_unlock(&adapter->mdev_lock);
+	return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+	struct device *dev = mdev_parent_dev(mdev);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+	mutex_lock(&adapter->mdev_lock);
+	adapter->mdev_count++;
+	mutex_unlock(&adapter->mdev_lock);
+
+	return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+	.owner			= THIS_MODULE,
+	.supported_type_groups	= mdev_type_groups,
+	.mdev_attr_groups	= mdev_dev_groups,
+	.create			= ifcvf_mdev_create,
+	.remove			= ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct device *dev = &pdev->dev;
+	struct ifcvf_adapter *adapter;
+	struct ifcvf_hw *vf;
+	int ret, i;
+
+	adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+	if (adapter == NULL) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	mutex_init(&adapter->mdev_lock);
+	adapter->mdev_count = 1;
+	adapter->dev = dev;
+
+	pci_set_drvdata(pdev, adapter);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to enable device.\n");
+		goto free_adapter;
+	}
+
+	ret = pci_request_regions(pdev, IFCVF_DRIVER_NAME);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to request MMIO region.\n");
+		goto disable_device;
+	}
+
+	pci_set_master(pdev);
+
+	ret = ifcvf_init_msix(adapter);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to initialize MSIX.\n");
+		goto free_msix;
+	}
+
+	vf = &adapter->vf;
+	for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+		vf->mem_resource[i].phys_addr = pci_resource_start(pdev, i);
+		vf->mem_resource[i].len = pci_resource_len(pdev, i);
+		if (!vf->mem_resource[i].len) {
+			vf->mem_resource[i].addr = NULL;
+			continue;
+		}
+
+		vf->mem_resource[i].addr = pci_iomap_range(pdev, i, 0,
+				vf->mem_resource[i].len);
+		if (!vf->mem_resource[i].addr) {
+			IFC_ERR(adapter->dev, "Failed to map IO resource %d\n",
+				i);
+			return -1;
+		}
+	}
+
+	if (ifcvf_init_hw(vf, pdev) < 0)
+		return -1;
+
+	ret = mdev_register_device(dev, &ifcvf_mdev_fops);
+	if (ret) {
+		IFC_ERR(adapter->dev,  "Failed to register mdev device\n");
+		goto destroy_adapter;
+	}
+
+	return 0;
+
+destroy_adapter:
+	ifcvf_destroy_adapter(adapter);
+free_msix:
+	pci_free_irq_vectors(pdev);
+	pci_release_regions(pdev);
+disable_device:
+	pci_disable_device(pdev);
+free_adapter:
+	kfree(adapter);
+fail:
+	return ret;
+}
+
+static void ifcvf_remove(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	struct ifcvf_hw *vf;
+	int i;
+
+	mdev_unregister_device(dev);
+
+	vf = &adapter->vf;
+	for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+		if (vf->mem_resource[i].addr) {
+			pci_iounmap(pdev, vf->mem_resource[i].addr);
+			vf->mem_resource[i].addr = NULL;
+		}
+	}
+
+	ifcvf_destroy_adapter(adapter);
+	pci_free_irq_vectors(pdev);
+
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+
+	kfree(adapter);
+}
+
+static struct pci_device_id ifcvf_pci_ids[] = {
+	{ PCI_DEVICE_SUB(IFCVF_VENDOR_ID,
+			IFCVF_DEVICE_ID,
+			IFCVF_SUBSYS_VENDOR_ID,
+			IFCVF_SUBSYS_DEVICE_ID) },
+	{ 0 },
+};
+MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
+
+static struct pci_driver ifcvf_driver = {
+	.name     = IFCVF_DRIVER_NAME,
+	.id_table = ifcvf_pci_ids,
+	.probe    = ifcvf_probe,
+	.remove   = ifcvf_remove,
+};
+
+static int __init ifcvf_init_module(void)
+{
+	int ret;
+
+	ret = pci_register_driver(&ifcvf_driver);
+	return ret;
+}
+
+static void __exit ifcvf_exit_module(void)
+{
+	pci_unregister_driver(&ifcvf_driver);
+}
+
+module_init(ifcvf_init_module);
+module_exit(ifcvf_exit_module);
+
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(VERSION_STRING);
+MODULE_AUTHOR(DRIVER_AUTHOR);
Signature · 1 line
2.16.4
#13 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 36 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541 +++++++++++++++++++++++++++++++++++++++
  1 file changed, 541 insertions(+)

create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING	"0.1"
+#define DRIVER_AUTHOR	"Intel Corporation"
+#define IFCVF_DRIVER_NAME	"ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+	struct vring_info *vring = arg;

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+	if (vring->cb.callback)
+		return vring->cb.callback(vring->cb.private);
+
+	return IRQ_HANDLED;
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+	return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 8 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 features)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->req_features = features;

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+	return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 8 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 qid, u64 num)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

drivers/vhost/ifcvf/ifcvf_main.c line 57

Quoted 1 line
drivers/vhost/ifcvf/ifcvf_main.c new file +1 -0
+	vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 96 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +96 -0
+
+	return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 idx,
+				     u64 desc_area, u64 driver_area,
+				     u64 device_area)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[idx].desc = desc_area;
+	vf->vring[idx].avail = driver_area;
+	vf->vring[idx].used = device_area;
+
+	return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 qid, u32 num)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+				u16 qid, bool ready)
+{
+
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+				 struct virtio_mdev_callback *cb)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+	return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+	return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+	return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+	return IFCVF_QUEUE_ALIGNMENT;

drivers/vhost/ifcvf/ifcvf_main.c line 163

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+	int i, ret;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+
+	for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {
+		if (!vf->vring[i].ready)
+			break;

Looks like error should be returned here?

Quoted 11 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +11 -0
+
+		if (!vf->vring[i].size)
+			break;
+
+		if (!vf->vring[i].desc || !vf->vring[i].avail ||
+			!vf->vring[i].used)
+			break;
+	}
+	vf->nr_vring = i;
+
+	ret = ifcvf_start_hw(vf);

drivers/vhost/ifcvf/ifcvf_main.c line 184

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+	return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+	int i;
+
+	for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+		vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 35 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +35 -0
+
+	ifcvf_stop_hw(vf);
+
+	return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+	int i;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vf->vring[i].last_used_idx = 0;
+		vf->vring[i].last_avail_idx = 0;
+		vf->vring[i].desc = 0;
+		vf->vring[i].avail = 0;
+		vf->vring[i].used = 0;
+		vf->vring[i].ready = 0;
+		vf->vring->cb.callback = NULL;
+		vf->vring->cb.private = NULL;
+	}
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->status = status;
+
+	if (status == 0) {
+		ifcvf_stop_datapath(adapter);
+		ifcvf_reset_vring(adapter);
+		return;
+	}

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+		ifcvf_start_datapath(adapter);
+		return;
+	}
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+	return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

Quoted 12 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+	.get_features  = ifcvf_mdev_get_features,
+	.set_features  = ifcvf_mdev_set_features,
+	.get_status    = ifcvf_mdev_get_status,
+	.set_status    = ifcvf_mdev_set_status,
+	.get_queue_max = ifcvf_mdev_get_queue_max,
+	.get_vq_state   = ifcvf_mdev_get_vq_state,
+	.set_vq_state   = ifcvf_mdev_set_vq_state,
+	.set_vq_cb      = ifcvf_mdev_set_vq_cb,
+	.set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+	.get_vq_ready	= ifcvf_mdev_get_vq_ready,
+	.set_vq_num     = ifcvf_mdev_set_vq_num,
+	.set_vq_address = ifcvf_mdev_set_vq_address,
+	.kick_vq        = ifcvf_mdev_kick_vq,
+	.get_generation	= ifcvf_mdev_get_generation,
+	.get_version	= ifcvf_mdev_get_version,
+	.get_device_id	= ifcvf_mdev_get_device_id,
+	.get_vendor_id	= ifcvf_mdev_get_vendor_id,
+	.get_vq_align	= ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

Quoted 15 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +15 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+	int vector, i, ret, irq;
+	struct pci_dev *pdev = to_pci_dev(adapter->dev);
+	struct ifcvf_hw *vf = &adapter->vf;
+
+	ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+			IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+	if (ret < 0) {
+		IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+		return ret;
+	}
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {

drivers/vhost/ifcvf/ifcvf_main.c line 276

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		ret = request_irq(irq, ifcvf_intr_handler, 0,
+				pci_name(pdev), &vf->vring[i]);
+		if (ret) {
+			IFC_ERR(adapter->dev,
+				"Failed to request irq for vq %d.\n", i);
+			return ret;
+		}
+	}

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 52 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +52 -0
+
+	return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+	int i, vector, irq;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+	struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		free_irq(irq, &vf->vring[i]);
+	}
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+	const char *name = "vhost accelerator (virtio ring compatible)";
+
+	return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+			       char *buf)
+{
+	return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+					struct device *dev, char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+	return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+			struct device *dev, char *buf)
+{
+	return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+static struct attribute *mdev_types_attrs[] = {
+	&mdev_type_attr_name.attr,
+	&mdev_type_attr_device_api.attr,
+	&mdev_type_attr_available_instances.attr,
+	&mdev_type_attr_type.attr,
+	NULL,
+};
+
+static struct attribute_group mdev_type_group = {
+	.name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

Quoted 66 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +66 -0
+	.attrs = mdev_types_attrs,
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+	&mdev_type_group,
+	NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+	NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
+{
+	struct device *dev = mdev_parent_dev(mdev);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	int ret = 0;
+
+	mutex_lock(&adapter->mdev_lock);
+
+	if (adapter->mdev_count < 1) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	mdev_set_class_id(mdev, MDEV_ID_VHOST);
+	mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+	mdev_set_drvdata(mdev, adapter);
+	mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+	INIT_LIST_HEAD(&adapter->dma_maps);
+	adapter->mdev_count--;
+
+out:
+	mutex_unlock(&adapter->mdev_lock);
+	return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+	struct device *dev = mdev_parent_dev(mdev);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+	mutex_lock(&adapter->mdev_lock);
+	adapter->mdev_count++;
+	mutex_unlock(&adapter->mdev_lock);
+
+	return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+	.owner			= THIS_MODULE,
+	.supported_type_groups	= mdev_type_groups,
+	.mdev_attr_groups	= mdev_dev_groups,
+	.create			= ifcvf_mdev_create,
+	.remove			= ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct device *dev = &pdev->dev;
+	struct ifcvf_adapter *adapter;
+	struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+	int ret, i;
+
+	adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+	if (adapter == NULL) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	mutex_init(&adapter->mdev_lock);
+	adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Thanks

Quoted 117 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +117 -0
+	adapter->dev = dev;
+
+	pci_set_drvdata(pdev, adapter);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to enable device.\n");
+		goto free_adapter;
+	}
+
+	ret = pci_request_regions(pdev, IFCVF_DRIVER_NAME);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to request MMIO region.\n");
+		goto disable_device;
+	}
+
+	pci_set_master(pdev);
+
+	ret = ifcvf_init_msix(adapter);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to initialize MSIX.\n");
+		goto free_msix;
+	}
+
+	vf = &adapter->vf;
+	for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+		vf->mem_resource[i].phys_addr = pci_resource_start(pdev, i);
+		vf->mem_resource[i].len = pci_resource_len(pdev, i);
+		if (!vf->mem_resource[i].len) {
+			vf->mem_resource[i].addr = NULL;
+			continue;
+		}
+
+		vf->mem_resource[i].addr = pci_iomap_range(pdev, i, 0,
+				vf->mem_resource[i].len);
+		if (!vf->mem_resource[i].addr) {
+			IFC_ERR(adapter->dev, "Failed to map IO resource %d\n",
+				i);
+			return -1;
+		}
+	}
+
+	if (ifcvf_init_hw(vf, pdev) < 0)
+		return -1;
+
+	ret = mdev_register_device(dev, &ifcvf_mdev_fops);
+	if (ret) {
+		IFC_ERR(adapter->dev,  "Failed to register mdev device\n");
+		goto destroy_adapter;
+	}
+
+	return 0;
+
+destroy_adapter:
+	ifcvf_destroy_adapter(adapter);
+free_msix:
+	pci_free_irq_vectors(pdev);
+	pci_release_regions(pdev);
+disable_device:
+	pci_disable_device(pdev);
+free_adapter:
+	kfree(adapter);
+fail:
+	return ret;
+}
+
+static void ifcvf_remove(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	struct ifcvf_hw *vf;
+	int i;
+
+	mdev_unregister_device(dev);
+
+	vf = &adapter->vf;
+	for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+		if (vf->mem_resource[i].addr) {
+			pci_iounmap(pdev, vf->mem_resource[i].addr);
+			vf->mem_resource[i].addr = NULL;
+		}
+	}
+
+	ifcvf_destroy_adapter(adapter);
+	pci_free_irq_vectors(pdev);
+
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+
+	kfree(adapter);
+}
+
+static struct pci_device_id ifcvf_pci_ids[] = {
+	{ PCI_DEVICE_SUB(IFCVF_VENDOR_ID,
+			IFCVF_DEVICE_ID,
+			IFCVF_SUBSYS_VENDOR_ID,
+			IFCVF_SUBSYS_DEVICE_ID) },
+	{ 0 },
+};
+MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
+
+static struct pci_driver ifcvf_driver = {
+	.name     = IFCVF_DRIVER_NAME,
+	.id_table = ifcvf_pci_ids,
+	.probe    = ifcvf_probe,
+	.remove   = ifcvf_remove,
+};
+
+static int __init ifcvf_init_module(void)
+{
+	int ret;
+
+	ret = pci_register_driver(&ifcvf_driver);
+	return ret;
+}
+
+static void __exit ifcvf_exit_module(void)

drivers/vhost/ifcvf/ifcvf_main.c line 541

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+	pci_unregister_driver(&ifcvf_driver);
+}
+
+module_init(ifcvf_init_module);
+module_exit(ifcvf_exit_module);
+
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(VERSION_STRING);
+MODULE_AUTHOR(DRIVER_AUTHOR);

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#14 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

In reply to Zhu Lingshan
Quoted 36 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541 +++++++++++++++++++++++++++++++++++++++
  1 file changed, 541 insertions(+)

create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING	"0.1"
+#define DRIVER_AUTHOR	"Intel Corporation"
+#define IFCVF_DRIVER_NAME	"ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+	struct vring_info *vring = arg;

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+	if (vring->cb.callback)
+		return vring->cb.callback(vring->cb.private);
+
+	return IRQ_HANDLED;
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+	return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 8 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 features)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->req_features = features;

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+	return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 8 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 qid, u64 num)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

drivers/vhost/ifcvf/ifcvf_main.c line 57

Quoted 1 line
drivers/vhost/ifcvf/ifcvf_main.c new file +1 -0
+	vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 96 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +96 -0
+
+	return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 idx,
+				     u64 desc_area, u64 driver_area,
+				     u64 device_area)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[idx].desc = desc_area;
+	vf->vring[idx].avail = driver_area;
+	vf->vring[idx].used = device_area;
+
+	return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 qid, u32 num)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+				u16 qid, bool ready)
+{
+
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+				 struct virtio_mdev_callback *cb)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+	return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+	return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+	return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+	return IFCVF_QUEUE_ALIGNMENT;

drivers/vhost/ifcvf/ifcvf_main.c line 163

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+	int i, ret;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+
+	for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {
+		if (!vf->vring[i].ready)
+			break;

Looks like error should be returned here?

Quoted 11 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +11 -0
+
+		if (!vf->vring[i].size)
+			break;
+
+		if (!vf->vring[i].desc || !vf->vring[i].avail ||
+			!vf->vring[i].used)
+			break;
+	}
+	vf->nr_vring = i;
+
+	ret = ifcvf_start_hw(vf);

drivers/vhost/ifcvf/ifcvf_main.c line 184

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+	return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+	int i;
+
+	for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+		vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 35 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +35 -0
+
+	ifcvf_stop_hw(vf);
+
+	return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+	int i;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vf->vring[i].last_used_idx = 0;
+		vf->vring[i].last_avail_idx = 0;
+		vf->vring[i].desc = 0;
+		vf->vring[i].avail = 0;
+		vf->vring[i].used = 0;
+		vf->vring[i].ready = 0;
+		vf->vring->cb.callback = NULL;
+		vf->vring->cb.private = NULL;
+	}
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+	struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+	vf->status = status;
+
+	if (status == 0) {
+		ifcvf_stop_datapath(adapter);
+		ifcvf_reset_vring(adapter);
+		return;
+	}

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+	if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+		ifcvf_start_datapath(adapter);
+		return;
+	}
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+	return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

Quoted 12 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+	.get_features  = ifcvf_mdev_get_features,
+	.set_features  = ifcvf_mdev_set_features,
+	.get_status    = ifcvf_mdev_get_status,
+	.set_status    = ifcvf_mdev_set_status,
+	.get_queue_max = ifcvf_mdev_get_queue_max,
+	.get_vq_state   = ifcvf_mdev_get_vq_state,
+	.set_vq_state   = ifcvf_mdev_set_vq_state,
+	.set_vq_cb      = ifcvf_mdev_set_vq_cb,
+	.set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+	.get_vq_ready	= ifcvf_mdev_get_vq_ready,
+	.set_vq_num     = ifcvf_mdev_set_vq_num,
+	.set_vq_address = ifcvf_mdev_set_vq_address,
+	.kick_vq        = ifcvf_mdev_kick_vq,
+	.get_generation	= ifcvf_mdev_get_generation,
+	.get_version	= ifcvf_mdev_get_version,
+	.get_device_id	= ifcvf_mdev_get_device_id,
+	.get_vendor_id	= ifcvf_mdev_get_vendor_id,
+	.get_vq_align	= ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

Quoted 15 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +15 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+	int vector, i, ret, irq;
+	struct pci_dev *pdev = to_pci_dev(adapter->dev);
+	struct ifcvf_hw *vf = &adapter->vf;
+
+	ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+			IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+	if (ret < 0) {
+		IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+		return ret;
+	}
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {

drivers/vhost/ifcvf/ifcvf_main.c line 276

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		ret = request_irq(irq, ifcvf_intr_handler, 0,
+				pci_name(pdev), &vf->vring[i]);
+		if (ret) {
+			IFC_ERR(adapter->dev,
+				"Failed to request irq for vq %d.\n", i);
+			return ret;
+		}
+	}

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 52 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +52 -0
+
+	return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+	int i, vector, irq;
+	struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+	struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+	for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+		vector = i + IFCVF_MSI_QUEUE_OFF;
+		irq = pci_irq_vector(pdev, vector);
+		free_irq(irq, &vf->vring[i]);
+	}
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, char *buf)
+{
+	const char *name = "vhost accelerator (virtio ring compatible)";
+
+	return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+			       char *buf)
+{
+	return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+					struct device *dev, char *buf)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+	return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+			struct device *dev, char *buf)
+{
+	return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+static struct attribute *mdev_types_attrs[] = {
+	&mdev_type_attr_name.attr,
+	&mdev_type_attr_device_api.attr,
+	&mdev_type_attr_available_instances.attr,
+	&mdev_type_attr_type.attr,
+	NULL,
+};
+
+static struct attribute_group mdev_type_group = {
+	.name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

Quoted 66 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +66 -0
+	.attrs = mdev_types_attrs,
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+	&mdev_type_group,
+	NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+	NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct mdev_device *mdev)
+{
+	struct device *dev = mdev_parent_dev(mdev);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	int ret = 0;
+
+	mutex_lock(&adapter->mdev_lock);
+
+	if (adapter->mdev_count < 1) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	mdev_set_class_id(mdev, MDEV_ID_VHOST);
+	mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+	mdev_set_drvdata(mdev, adapter);
+	mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+	INIT_LIST_HEAD(&adapter->dma_maps);
+	adapter->mdev_count--;
+
+out:
+	mutex_unlock(&adapter->mdev_lock);
+	return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+	struct device *dev = mdev_parent_dev(mdev);
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+	mutex_lock(&adapter->mdev_lock);
+	adapter->mdev_count++;
+	mutex_unlock(&adapter->mdev_lock);
+
+	return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+	.owner			= THIS_MODULE,
+	.supported_type_groups	= mdev_type_groups,
+	.mdev_attr_groups	= mdev_dev_groups,
+	.create			= ifcvf_mdev_create,
+	.remove			= ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct device *dev = &pdev->dev;
+	struct ifcvf_adapter *adapter;
+	struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+	int ret, i;
+
+	adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+	if (adapter == NULL) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	mutex_init(&adapter->mdev_lock);
+	adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Thanks

Quoted 127 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +127 -0
+	adapter->dev = dev;
+
+	pci_set_drvdata(pdev, adapter);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to enable device.\n");
+		goto free_adapter;
+	}
+
+	ret = pci_request_regions(pdev, IFCVF_DRIVER_NAME);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to request MMIO region.\n");
+		goto disable_device;
+	}
+
+	pci_set_master(pdev);
+
+	ret = ifcvf_init_msix(adapter);
+	if (ret) {
+		IFC_ERR(adapter->dev, "Failed to initialize MSIX.\n");
+		goto free_msix;
+	}
+
+	vf = &adapter->vf;
+	for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+		vf->mem_resource[i].phys_addr = pci_resource_start(pdev, i);
+		vf->mem_resource[i].len = pci_resource_len(pdev, i);
+		if (!vf->mem_resource[i].len) {
+			vf->mem_resource[i].addr = NULL;
+			continue;
+		}
+
+		vf->mem_resource[i].addr = pci_iomap_range(pdev, i, 0,
+				vf->mem_resource[i].len);
+		if (!vf->mem_resource[i].addr) {
+			IFC_ERR(adapter->dev, "Failed to map IO resource %d\n",
+				i);
+			return -1;
+		}
+	}
+
+	if (ifcvf_init_hw(vf, pdev) < 0)
+		return -1;
+
+	ret = mdev_register_device(dev, &ifcvf_mdev_fops);
+	if (ret) {
+		IFC_ERR(adapter->dev,  "Failed to register mdev device\n");
+		goto destroy_adapter;
+	}
+
+	return 0;
+
+destroy_adapter:
+	ifcvf_destroy_adapter(adapter);
+free_msix:
+	pci_free_irq_vectors(pdev);
+	pci_release_regions(pdev);
+disable_device:
+	pci_disable_device(pdev);
+free_adapter:
+	kfree(adapter);
+fail:
+	return ret;
+}
+
+static void ifcvf_remove(struct pci_dev *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+	struct ifcvf_hw *vf;
+	int i;
+
+	mdev_unregister_device(dev);
+
+	vf = &adapter->vf;
+	for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+		if (vf->mem_resource[i].addr) {
+			pci_iounmap(pdev, vf->mem_resource[i].addr);
+			vf->mem_resource[i].addr = NULL;
+		}
+	}
+
+	ifcvf_destroy_adapter(adapter);
+	pci_free_irq_vectors(pdev);
+
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+
+	kfree(adapter);
+}
+
+static struct pci_device_id ifcvf_pci_ids[] = {
+	{ PCI_DEVICE_SUB(IFCVF_VENDOR_ID,
+			IFCVF_DEVICE_ID,
+			IFCVF_SUBSYS_VENDOR_ID,
+			IFCVF_SUBSYS_DEVICE_ID) },
+	{ 0 },
+};
+MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
+
+static struct pci_driver ifcvf_driver = {
+	.name     = IFCVF_DRIVER_NAME,
+	.id_table = ifcvf_pci_ids,
+	.probe    = ifcvf_probe,
+	.remove   = ifcvf_remove,
+};
+
+static int __init ifcvf_init_module(void)
+{
+	int ret;
+
+	ret = pci_register_driver(&ifcvf_driver);
+	return ret;
+}
+
+static void __exit ifcvf_exit_module(void)
+{
+	pci_unregister_driver(&ifcvf_driver);
+}
+
+module_init(ifcvf_init_module);
+module_exit(ifcvf_exit_module);
+
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(VERSION_STRING);
+MODULE_AUTHOR(DRIVER_AUTHOR);
#15 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Hello Jason,

Thanks for your comments, I am on a conference travel, I will reply next Monday.

Thanks,
BR
Zhu Lingshan

Quoted 626 lines · Jason Wang
Quoted 37 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541
+++++++++++++++++++++++++++++++++++++++

  1 file changed, 541 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING    "0.1"
+#define DRIVER_AUTHOR    "Intel Corporation"
+#define IFCVF_DRIVER_NAME    "ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+    struct vring_info *vring = arg;

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+    if (vring->cb.callback)
+        return vring->cb.callback(vring->cb.private);
+
+    return IRQ_HANDLED;
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+    return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 9 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 9 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 

qid, u64 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

Quoted 1 line

+    vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 97 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +19 -0
+
+    return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 idx,
+                     u64 desc_area, u64 driver_area,
+                     u64 device_area)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].desc = desc_area;
+    vf->vring[idx].avail = driver_area;
+    vf->vring[idx].used = device_area;
+
+    return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 qid, 

u32 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +77 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+                u16 qid, bool ready)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+                 struct virtio_mdev_callback *cb)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+    return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+    return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+    return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+    return IFCVF_QUEUE_ALIGNMENT;

drivers/vhost/ifcvf/ifcvf_main.c line 163

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+    int i, ret;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) {
+        if (!vf->vring[i].ready)
+            break;

Looks like error should be returned here?

Quoted 11 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +11 -0
+
+        if (!vf->vring[i].size)
+            break;
+
+        if (!vf->vring[i].desc || !vf->vring[i].avail ||
+            !vf->vring[i].used)
+            break;
+    }
+    vf->nr_vring = i;
+
+    ret = ifcvf_start_hw(vf);

drivers/vhost/ifcvf/ifcvf_main.c line 184

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+    int i;
+
+    for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+        vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 35 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +35 -0
+
+    ifcvf_stop_hw(vf);
+
+    return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+    int i;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vf->vring[i].last_used_idx = 0;
+        vf->vring[i].last_avail_idx = 0;
+        vf->vring[i].desc = 0;
+        vf->vring[i].avail = 0;
+        vf->vring[i].used = 0;
+        vf->vring[i].ready = 0;
+        vf->vring->cb.callback = NULL;
+        vf->vring->cb.private = NULL;
+    }
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->status = status;
+
+    if (status == 0) {
+        ifcvf_stop_datapath(adapter);
+        ifcvf_reset_vring(adapter);
+        return;
+    }

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+    if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+        ifcvf_start_datapath(adapter);
+        return;
+    }
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+    return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

Quoted 12 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+    .get_features  = ifcvf_mdev_get_features,
+    .set_features  = ifcvf_mdev_set_features,
+    .get_status    = ifcvf_mdev_get_status,
+    .set_status    = ifcvf_mdev_set_status,
+    .get_queue_max = ifcvf_mdev_get_queue_max,
+    .get_vq_state   = ifcvf_mdev_get_vq_state,
+    .set_vq_state   = ifcvf_mdev_set_vq_state,
+    .set_vq_cb      = ifcvf_mdev_set_vq_cb,
+    .set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    .get_vq_ready    = ifcvf_mdev_get_vq_ready,
+    .set_vq_num     = ifcvf_mdev_set_vq_num,
+    .set_vq_address = ifcvf_mdev_set_vq_address,
+    .kick_vq        = ifcvf_mdev_kick_vq,
+    .get_generation    = ifcvf_mdev_get_generation,
+    .get_version    = ifcvf_mdev_get_version,
+    .get_device_id    = ifcvf_mdev_get_device_id,
+    .get_vendor_id    = ifcvf_mdev_get_vendor_id,
+    .get_vq_align    = ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

Quoted 15 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +15 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+    int vector, i, ret, irq;
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+    struct ifcvf_hw *vf = &adapter->vf;
+
+    ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+            IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+    if (ret < 0) {
+        IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+        return ret;
+    }
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {

drivers/vhost/ifcvf/ifcvf_main.c line 276

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        ret = request_irq(irq, ifcvf_intr_handler, 0,
+                pci_name(pdev), &vf->vring[i]);
+        if (ret) {
+            IFC_ERR(adapter->dev,
+                "Failed to request irq for vq %d.\n", i);
+            return ret;
+        }
+    }

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 54 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+
+    return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+    int i, vector, irq;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        free_irq(irq, &vf->vring[i]);
+    }
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, 

char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+{
+    const char *name = "vhost accelerator (virtio ring compatible)";
+
+    return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device 

*dev,
+                   char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +25 -0
+{
+    return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                    struct device *dev, char *buf)
+{
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+            struct device *dev, char *buf)
+{
+    return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+static struct attribute *mdev_types_attrs[] = {
+    &mdev_type_attr_name.attr,
+    &mdev_type_attr_device_api.attr,
+    &mdev_type_attr_available_instances.attr,
+    &mdev_type_attr_type.attr,
+    NULL,
+};
+
+static struct attribute_group mdev_type_group = {
+    .name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

Quoted 68 lines

+    .attrs = mdev_types_attrs,

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+    &mdev_type_group,
+    NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+    NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct 

mdev_device *mdev)

drivers/vhost/ifcvf/ifcvf_main.c new file +49 -0
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    int ret = 0;
+
+    mutex_lock(&adapter->mdev_lock);
+
+    if (adapter->mdev_count < 1) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    mdev_set_class_id(mdev, MDEV_ID_VHOST);
+    mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+    mdev_set_drvdata(mdev, adapter);
+    mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+    INIT_LIST_HEAD(&adapter->dma_maps);
+    adapter->mdev_count--;
+
+out:
+    mutex_unlock(&adapter->mdev_lock);
+    return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    mutex_lock(&adapter->mdev_lock);
+    adapter->mdev_count++;
+    mutex_unlock(&adapter->mdev_lock);
+
+    return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+    .owner            = THIS_MODULE,
+    .supported_type_groups    = mdev_type_groups,
+    .mdev_attr_groups    = mdev_dev_groups,
+    .create            = ifcvf_mdev_create,
+    .remove            = ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct 

pci_device_id *id)

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter;
+    struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    int ret, i;
+
+    adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+    if (adapter == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    mutex_init(&adapter->mdev_lock);
+    adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Thanks

Quoted 127 lines

+    adapter->dev = dev;

drivers/vhost/ifcvf/ifcvf_main.c new file +126 -0
+
+    pci_set_drvdata(pdev, adapter);
+
+    ret = pci_enable_device(pdev);
+    if (ret) {
+        IFC_ERR(adapter->dev, "Failed to enable device.\n");
+        goto free_adapter;
+    }
+
+    ret = pci_request_regions(pdev, IFCVF_DRIVER_NAME);
+    if (ret) {
+        IFC_ERR(adapter->dev, "Failed to request MMIO region.\n");
+        goto disable_device;
+    }
+
+    pci_set_master(pdev);
+
+    ret = ifcvf_init_msix(adapter);
+    if (ret) {
+        IFC_ERR(adapter->dev, "Failed to initialize MSIX.\n");
+        goto free_msix;
+    }
+
+    vf = &adapter->vf;
+    for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+        vf->mem_resource[i].phys_addr = pci_resource_start(pdev, i);
+        vf->mem_resource[i].len = pci_resource_len(pdev, i);
+        if (!vf->mem_resource[i].len) {
+            vf->mem_resource[i].addr = NULL;
+            continue;
+        }
+
+        vf->mem_resource[i].addr = pci_iomap_range(pdev, i, 0,
+                vf->mem_resource[i].len);
+        if (!vf->mem_resource[i].addr) {
+            IFC_ERR(adapter->dev, "Failed to map IO resource %d\n",
+                i);
+            return -1;
+        }
+    }
+
+    if (ifcvf_init_hw(vf, pdev) < 0)
+        return -1;
+
+    ret = mdev_register_device(dev, &ifcvf_mdev_fops);
+    if (ret) {
+        IFC_ERR(adapter->dev,  "Failed to register mdev device\n");
+        goto destroy_adapter;
+    }
+
+    return 0;
+
+destroy_adapter:
+    ifcvf_destroy_adapter(adapter);
+free_msix:
+    pci_free_irq_vectors(pdev);
+    pci_release_regions(pdev);
+disable_device:
+    pci_disable_device(pdev);
+free_adapter:
+    kfree(adapter);
+fail:
+    return ret;
+}
+
+static void ifcvf_remove(struct pci_dev *pdev)
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    struct ifcvf_hw *vf;
+    int i;
+
+    mdev_unregister_device(dev);
+
+    vf = &adapter->vf;
+    for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+        if (vf->mem_resource[i].addr) {
+            pci_iounmap(pdev, vf->mem_resource[i].addr);
+            vf->mem_resource[i].addr = NULL;
+        }
+    }
+
+    ifcvf_destroy_adapter(adapter);
+    pci_free_irq_vectors(pdev);
+
+    pci_release_regions(pdev);
+    pci_disable_device(pdev);
+
+    kfree(adapter);
+}
+
+static struct pci_device_id ifcvf_pci_ids[] = {
+    { PCI_DEVICE_SUB(IFCVF_VENDOR_ID,
+            IFCVF_DEVICE_ID,
+            IFCVF_SUBSYS_VENDOR_ID,
+            IFCVF_SUBSYS_DEVICE_ID) },
+    { 0 },
+};
+MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
+
+static struct pci_driver ifcvf_driver = {
+    .name     = IFCVF_DRIVER_NAME,
+    .id_table = ifcvf_pci_ids,
+    .probe    = ifcvf_probe,
+    .remove   = ifcvf_remove,
+};
+
+static int __init ifcvf_init_module(void)
+{
+    int ret;
+
+    ret = pci_register_driver(&ifcvf_driver);
+    return ret;
+}
+
+static void __exit ifcvf_exit_module(void)
+{
+    pci_unregister_driver(&ifcvf_driver);
+}
+
+module_init(ifcvf_init_module);
+module_exit(ifcvf_exit_module);
+
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(VERSION_STRING);
+MODULE_AUTHOR(DRIVER_AUTHOR);
#16 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

In reply to Jason Wang
Quoted 44 lines · Jason Wang
Quoted 42 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541
+++++++++++++++++++++++++++++++++++++++

  1 file changed, 541 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +25 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING    "0.1"
+#define DRIVER_AUTHOR    "Intel Corporation"
+#define IFCVF_DRIVER_NAME    "ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+    struct vring_info *vring = arg;
+
+    if (vring->cb.callback)
+        return vring->cb.callback(vring->cb.private);
+
+    return IRQ_HANDLED;
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+    return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Hi Jason,

Thanks for your comments, for now driver just support these features.

Quoted 16 lines
Quoted 16 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

for now, it's working, we intend to support LM in next version drivers.

Quoted 14 lines

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 11 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 

qid, u64 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

I will remove it.

Quoted 110 lines
Quoted 3 lines

+    vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 102 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +19 -0
+
+    return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 idx,
+                     u64 desc_area, u64 driver_area,
+                     u64 device_area)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].desc = desc_area;
+    vf->vring[idx].avail = driver_area;
+    vf->vring[idx].used = device_area;
+
+    return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 qid, 

u32 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +82 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+                u16 qid, bool ready)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+                 struct virtio_mdev_callback *cb)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+    return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+    return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+    return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+    return IFCVF_QUEUE_ALIGNMENT;
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+    int i, ret;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) { +        if (!vf->vring[i].ready)
+            break;

Looks like error should be returned here?

agreed!

Quoted 18 lines
Quoted 18 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +16 -0
+
+        if (!vf->vring[i].size)
+            break;
+
+        if (!vf->vring[i].desc || !vf->vring[i].avail ||
+            !vf->vring[i].used)
+            break;
+    }
+    vf->nr_vring = i;
+
+    ret = ifcvf_start_hw(vf);
+    return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +    int i;
+
+    for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+        vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

I think even we set callback = NULL, the code is still there, on-going routines would not be effected.

Quoted 42 lines
Quoted 42 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +40 -0
+
+    ifcvf_stop_hw(vf);
+
+    return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+    int i;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vf->vring[i].last_used_idx = 0;
+        vf->vring[i].last_avail_idx = 0;
+        vf->vring[i].desc = 0;
+        vf->vring[i].avail = 0;
+        vf->vring[i].used = 0;
+        vf->vring[i].ready = 0;
+        vf->vring->cb.callback = NULL;
+        vf->vring->cb.private = NULL;
+    }
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->status = status;
+
+    if (status == 0) {
+        ifcvf_stop_datapath(adapter);
+        ifcvf_reset_vring(adapter);
+        return;
+    }
+
+    if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+        ifcvf_start_datapath(adapter);
+        return;
+    }
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+    return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

will change that.

Quoted 28 lines
Quoted 14 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+    .get_features  = ifcvf_mdev_get_features,
+    .set_features  = ifcvf_mdev_set_features,
+    .get_status    = ifcvf_mdev_get_status,
+    .set_status    = ifcvf_mdev_set_status,
+    .get_queue_max = ifcvf_mdev_get_queue_max,
+    .get_vq_state   = ifcvf_mdev_get_vq_state,
+    .set_vq_state   = ifcvf_mdev_set_vq_state,
+    .set_vq_cb      = ifcvf_mdev_set_vq_cb,
+    .set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    .get_vq_ready    = ifcvf_mdev_get_vq_ready,
+    .set_vq_num     = ifcvf_mdev_set_vq_num,
+    .set_vq_address = ifcvf_mdev_set_vq_address,
+    .kick_vq        = ifcvf_mdev_kick_vq,
+    .get_generation    = ifcvf_mdev_get_generation,
+    .get_version    = ifcvf_mdev_get_version,
+    .get_device_id    = ifcvf_mdev_get_device_id,
+    .get_vendor_id    = ifcvf_mdev_get_vendor_id,
+    .get_vq_align    = ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

Quoted 1 line

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

IMHO, currently the driver can work without set/get_config, vhost_mdev doesn't call them for now.

Quoted 22 lines
Quoted 22 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+    int vector, i, ret, irq;
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+    struct ifcvf_hw *vf = &adapter->vf;
+
+    ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+            IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+    if (ret < 0) {
+        IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+        return ret;
+    }
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        ret = request_irq(irq, ifcvf_intr_handler, 0,
+                pci_name(pdev), &vf->vring[i]);
+        if (ret) {
Quoted 8 lines
Quoted 5 lines

+            IFC_ERR(adapter->dev,
+                "Failed to request irq for vq %d.\n", i); +            return ret;
+        }
+    }

Do we need to provide fallback when we can't do per vq MSIX?

I think it would be very rarely that can not get enough vectors.

Quoted 61 lines
Quoted 61 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+
+    return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+    int i, vector, irq;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        free_irq(irq, &vf->vring[i]);
+    }
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, 

char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+{
+    const char *name = "vhost accelerator (virtio ring compatible)";
+
+    return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device 

*dev,
+                   char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +30 -0
+{
+    return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                    struct device *dev, char *buf)
+{
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+            struct device *dev, char *buf)
+{
+    return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+
+static struct attribute *mdev_types_attrs[] = {
+    &mdev_type_attr_name.attr,
+    &mdev_type_attr_device_api.attr,
+    &mdev_type_attr_available_instances.attr,
+    &mdev_type_attr_type.attr,
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 5 lines

+    NULL,

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+};
+
+static struct attribute_group mdev_type_group = {
+    .name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

agreed!

Quoted 84 lines
Quoted 70 lines

+    .attrs = mdev_types_attrs,

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+    &mdev_type_group,
+    NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+    NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct 

mdev_device *mdev)

drivers/vhost/ifcvf/ifcvf_main.c new file +49 -0
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    int ret = 0;
+
+    mutex_lock(&adapter->mdev_lock);
+
+    if (adapter->mdev_count < 1) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    mdev_set_class_id(mdev, MDEV_ID_VHOST);
+    mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+    mdev_set_drvdata(mdev, adapter);
+    mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+    INIT_LIST_HEAD(&adapter->dma_maps);
+    adapter->mdev_count--;
+
+out:
+    mutex_unlock(&adapter->mdev_lock);
+    return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    mutex_lock(&adapter->mdev_lock);
+    adapter->mdev_count++;
+    mutex_unlock(&adapter->mdev_lock);
+
+    return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+    .owner            = THIS_MODULE,
+    .supported_type_groups    = mdev_type_groups,
+    .mdev_attr_groups    = mdev_dev_groups,
+    .create            = ifcvf_mdev_create,
+    .remove            = ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct 

pci_device_id *id)

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter;
+    struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    int ret, i;
+
+    adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+    if (adapter == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    mutex_init(&adapter->mdev_lock);
+    adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Quoted 1 line

Thanks

the driver scope for now doesn't support that, we can add these feature in next releases.

Quoted 129 lines
Quoted 129 lines

+    adapter->dev = dev;

drivers/vhost/ifcvf/ifcvf_main.c new file +126 -0
+
+    pci_set_drvdata(pdev, adapter);
+
+    ret = pci_enable_device(pdev);
+    if (ret) {
+        IFC_ERR(adapter->dev, "Failed to enable device.\n");
+        goto free_adapter;
+    }
+
+    ret = pci_request_regions(pdev, IFCVF_DRIVER_NAME);
+    if (ret) {
+        IFC_ERR(adapter->dev, "Failed to request MMIO region.\n");
+        goto disable_device;
+    }
+
+    pci_set_master(pdev);
+
+    ret = ifcvf_init_msix(adapter);
+    if (ret) {
+        IFC_ERR(adapter->dev, "Failed to initialize MSIX.\n");
+        goto free_msix;
+    }
+
+    vf = &adapter->vf;
+    for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+        vf->mem_resource[i].phys_addr = pci_resource_start(pdev, i);
+        vf->mem_resource[i].len = pci_resource_len(pdev, i);
+        if (!vf->mem_resource[i].len) {
+            vf->mem_resource[i].addr = NULL;
+            continue;
+        }
+
+        vf->mem_resource[i].addr = pci_iomap_range(pdev, i, 0,
+                vf->mem_resource[i].len);
+        if (!vf->mem_resource[i].addr) {
+            IFC_ERR(adapter->dev, "Failed to map IO resource %d\n",
+                i);
+            return -1;
+        }
+    }
+
+    if (ifcvf_init_hw(vf, pdev) < 0)
+        return -1;
+
+    ret = mdev_register_device(dev, &ifcvf_mdev_fops);
+    if (ret) {
+        IFC_ERR(adapter->dev,  "Failed to register mdev device\n");
+        goto destroy_adapter;
+    }
+
+    return 0;
+
+destroy_adapter:
+    ifcvf_destroy_adapter(adapter);
+free_msix:
+    pci_free_irq_vectors(pdev);
+    pci_release_regions(pdev);
+disable_device:
+    pci_disable_device(pdev);
+free_adapter:
+    kfree(adapter);
+fail:
+    return ret;
+}
+
+static void ifcvf_remove(struct pci_dev *pdev)
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    struct ifcvf_hw *vf;
+    int i;
+
+    mdev_unregister_device(dev);
+
+    vf = &adapter->vf;
+    for (i = 0; i < IFCVF_PCI_MAX_RESOURCE; i++) {
+        if (vf->mem_resource[i].addr) {
+            pci_iounmap(pdev, vf->mem_resource[i].addr);
+            vf->mem_resource[i].addr = NULL;
+        }
+    }
+
+    ifcvf_destroy_adapter(adapter);
+    pci_free_irq_vectors(pdev);
+
+    pci_release_regions(pdev);
+    pci_disable_device(pdev);
+
+    kfree(adapter);
+}
+
+static struct pci_device_id ifcvf_pci_ids[] = {
+    { PCI_DEVICE_SUB(IFCVF_VENDOR_ID,
+            IFCVF_DEVICE_ID,
+            IFCVF_SUBSYS_VENDOR_ID,
+            IFCVF_SUBSYS_DEVICE_ID) },
+    { 0 },
+};
+MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);
+
+static struct pci_driver ifcvf_driver = {
+    .name     = IFCVF_DRIVER_NAME,
+    .id_table = ifcvf_pci_ids,
+    .probe    = ifcvf_probe,
+    .remove   = ifcvf_remove,
+};
+
+static int __init ifcvf_init_module(void)
+{
+    int ret;
+
+    ret = pci_register_driver(&ifcvf_driver);
+    return ret;
+}
+
+static void __exit ifcvf_exit_module(void)
+{
+    pci_unregister_driver(&ifcvf_driver);
+}
+
+module_init(ifcvf_init_module);
+module_exit(ifcvf_exit_module);
+
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(VERSION_STRING);
+MODULE_AUTHOR(DRIVER_AUTHOR);
#17 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 55 lines · Lingshan
Quoted 44 lines · Jason Wang
Quoted 42 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541
+++++++++++++++++++++++++++++++++++++++

  1 file changed, 541 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +25 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING    "0.1"
+#define DRIVER_AUTHOR    "Intel Corporation"
+#define IFCVF_DRIVER_NAME    "ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+    struct vring_info *vring = arg;
+
+    if (vring->cb.callback)
+        return vring->cb.callback(vring->cb.private);
+
+    return IRQ_HANDLED;
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+    return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 3 lines

Hi Jason,

Thanks for your comments, for now driver just support these features.

Ok, it should work but less flexible, we can change it in the future.

Quoted 25 lines
Quoted 17 lines
Quoted 17 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

Quoted 161 lines
Quoted 14 lines

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 11 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 

qid, u64 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

I will remove it.

Quoted 111 lines
Quoted 3 lines

+    vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 103 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+
+    return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 

idx,
+                     u64 desc_area, u64 driver_area, +                     u64 device_area)

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].desc = desc_area;
+    vf->vring[idx].avail = driver_area;
+    vf->vring[idx].used = device_area;
+
+    return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 

qid, u32 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +82 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+                u16 qid, bool ready)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+                 struct virtio_mdev_callback *cb)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+    return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+    return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+    return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+    return IFCVF_QUEUE_ALIGNMENT;
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+    int i, ret;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) { +        if (!vf->vring[i].ready)
+            break;

Looks like error should be returned here?

agreed!

Quoted 18 lines
Quoted 18 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +16 -0
+
+        if (!vf->vring[i].size)
+            break;
+
+        if (!vf->vring[i].desc || !vf->vring[i].avail ||
+            !vf->vring[i].used)
+            break;
+    }
+    vf->nr_vring = i;
+
+    ret = ifcvf_start_hw(vf);
+    return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +    int i;
+
+    for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+        vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 1 line

I think even we set callback = NULL, the code is still there, on-going routines would not be effected.

Ok I think you mean when ifcvf_stop_hw() return, hardware will not respond to e.g kick and other events etc.

Quoted 79 lines
Quoted 42 lines
Quoted 42 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +40 -0
+
+    ifcvf_stop_hw(vf);
+
+    return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+    int i;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vf->vring[i].last_used_idx = 0;
+        vf->vring[i].last_avail_idx = 0;
+        vf->vring[i].desc = 0;
+        vf->vring[i].avail = 0;
+        vf->vring[i].used = 0;
+        vf->vring[i].ready = 0;
+        vf->vring->cb.callback = NULL;
+        vf->vring->cb.private = NULL;
+    }
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->status = status;
+
+    if (status == 0) {
+        ifcvf_stop_datapath(adapter);
+        ifcvf_reset_vring(adapter);
+        return;
+    }
+
+    if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+        ifcvf_start_datapath(adapter);
+        return;
+    }
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+    return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

will change that.

Quoted 28 lines
Quoted 14 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+    .get_features  = ifcvf_mdev_get_features,
+    .set_features  = ifcvf_mdev_set_features,
+    .get_status    = ifcvf_mdev_get_status,
+    .set_status    = ifcvf_mdev_set_status,
+    .get_queue_max = ifcvf_mdev_get_queue_max,
+    .get_vq_state   = ifcvf_mdev_get_vq_state,
+    .set_vq_state   = ifcvf_mdev_set_vq_state,
+    .set_vq_cb      = ifcvf_mdev_set_vq_cb,
+    .set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    .get_vq_ready    = ifcvf_mdev_get_vq_ready,
+    .set_vq_num     = ifcvf_mdev_set_vq_num,
+    .set_vq_address = ifcvf_mdev_set_vq_address,
+    .kick_vq        = ifcvf_mdev_kick_vq,
+    .get_generation    = ifcvf_mdev_get_generation,
+    .get_version    = ifcvf_mdev_get_version,
+    .get_device_id    = ifcvf_mdev_get_device_id,
+    .get_vendor_id    = ifcvf_mdev_get_vendor_id,
+    .get_vq_align    = ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

Quoted 2 lines
Quoted 1 line

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

IMHO, currently the driver can work without set/get_config, vhost_mdev doesn't call them for now.

Yes, but it was required by virtio_mdev for host driver to work, and it looks to me it's not hard to add them. If possible please add them and "virtio" type then we can use the ops for both the case of VM and containers.

Quoted 30 lines
Quoted 22 lines
Quoted 22 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+    int vector, i, ret, irq;
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+    struct ifcvf_hw *vf = &adapter->vf;
+
+    ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+            IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+    if (ret < 0) {
+        IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+        return ret;
+    }
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        ret = request_irq(irq, ifcvf_intr_handler, 0,
+                pci_name(pdev), &vf->vring[i]);
+        if (ret) {
Quoted 8 lines
Quoted 5 lines

+            IFC_ERR(adapter->dev,
+                "Failed to request irq for vq %d.\n", i); +            return ret;
+        }
+    }

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 1 line

I think it would be very rarely that can not get enough vectors.

Right.

Quoted 154 lines
Quoted 61 lines
Quoted 61 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+
+    return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+    int i, vector, irq;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        free_irq(irq, &vf->vring[i]);
+    }
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, 

char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+{
+    const char *name = "vhost accelerator (virtio ring compatible)";
+
+    return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device 

*dev,
+                   char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +30 -0
+{
+    return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                    struct device *dev, char *buf)
+{
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+            struct device *dev, char *buf)
+{
+    return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+
+static struct attribute *mdev_types_attrs[] = {
+    &mdev_type_attr_name.attr,
+    &mdev_type_attr_device_api.attr,
+    &mdev_type_attr_available_instances.attr,
+    &mdev_type_attr_type.attr,
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 5 lines

+    NULL,

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+};
+
+static struct attribute_group mdev_type_group = {
+    .name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

agreed!

Quoted 84 lines
Quoted 70 lines

+    .attrs = mdev_types_attrs,

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+    &mdev_type_group,
+    NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+    NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct 

mdev_device *mdev)

drivers/vhost/ifcvf/ifcvf_main.c new file +49 -0
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    int ret = 0;
+
+    mutex_lock(&adapter->mdev_lock);
+
+    if (adapter->mdev_count < 1) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    mdev_set_class_id(mdev, MDEV_ID_VHOST);
+    mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+    mdev_set_drvdata(mdev, adapter);
+    mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+    INIT_LIST_HEAD(&adapter->dma_maps);
+    adapter->mdev_count--;
+
+out:
+    mutex_unlock(&adapter->mdev_lock);
+    return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    mutex_lock(&adapter->mdev_lock);
+    adapter->mdev_count++;
+    mutex_unlock(&adapter->mdev_lock);
+
+    return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+    .owner            = THIS_MODULE,
+    .supported_type_groups    = mdev_type_groups,
+    .mdev_attr_groups    = mdev_dev_groups,
+    .create            = ifcvf_mdev_create,
+    .remove            = ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct 

pci_device_id *id)

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter;
+    struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    int ret, i;
+
+    adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+    if (adapter == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    mutex_init(&adapter->mdev_lock);
+    adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Quoted 2 lines
Quoted 1 line

Thanks

the driver scope for now doesn't support that, we can add these feature in next releases.

Not a must for this series, but to have a better interaction with management like libvirt, it's better.

Btw, do you have the plan to post PF drivers?

Thanks

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#18 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

In reply to Zhu, Lingshan
Quoted 55 lines · Lingshan
Quoted 44 lines · Jason Wang
Quoted 42 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541
+++++++++++++++++++++++++++++++++++++++

  1 file changed, 541 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +25 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING    "0.1"
+#define DRIVER_AUTHOR    "Intel Corporation"
+#define IFCVF_DRIVER_NAME    "ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+    struct vring_info *vring = arg;
+
+    if (vring->cb.callback)
+        return vring->cb.callback(vring->cb.private);
+
+    return IRQ_HANDLED;
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+    return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 3 lines

Hi Jason,

Thanks for your comments, for now driver just support these features.

Ok, it should work but less flexible, we can change it in the future.

Quoted 25 lines
Quoted 17 lines
Quoted 17 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

Quoted 161 lines
Quoted 14 lines

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 11 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 

qid, u64 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

I will remove it.

Quoted 111 lines
Quoted 3 lines

+    vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 103 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+
+    return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 

idx,
+                     u64 desc_area, u64 driver_area, +                     u64 device_area)

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].desc = desc_area;
+    vf->vring[idx].avail = driver_area;
+    vf->vring[idx].used = device_area;
+
+    return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 

qid, u32 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +82 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+                u16 qid, bool ready)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 qid)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+                 struct virtio_mdev_callback *cb)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+    return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+    return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+    return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+    return IFCVF_QUEUE_ALIGNMENT;
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+    int i, ret;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) { +        if (!vf->vring[i].ready)
+            break;

Looks like error should be returned here?

agreed!

Quoted 18 lines
Quoted 18 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +16 -0
+
+        if (!vf->vring[i].size)
+            break;
+
+        if (!vf->vring[i].desc || !vf->vring[i].avail ||
+            !vf->vring[i].used)
+            break;
+    }
+    vf->nr_vring = i;
+
+    ret = ifcvf_start_hw(vf);
+    return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +    int i;
+
+    for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+        vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 1 line

I think even we set callback = NULL, the code is still there, on-going routines would not be effected.

Ok I think you mean when ifcvf_stop_hw() return, hardware will not respond to e.g kick and other events etc.

Quoted 79 lines
Quoted 42 lines
Quoted 42 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +40 -0
+
+    ifcvf_stop_hw(vf);
+
+    return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+    int i;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vf->vring[i].last_used_idx = 0;
+        vf->vring[i].last_avail_idx = 0;
+        vf->vring[i].desc = 0;
+        vf->vring[i].avail = 0;
+        vf->vring[i].used = 0;
+        vf->vring[i].ready = 0;
+        vf->vring->cb.callback = NULL;
+        vf->vring->cb.private = NULL;
+    }
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 status)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->status = status;
+
+    if (status == 0) {
+        ifcvf_stop_datapath(adapter);
+        ifcvf_reset_vring(adapter);
+        return;
+    }
+
+    if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+        ifcvf_start_datapath(adapter);
+        return;
+    }
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+    return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

will change that.

Quoted 28 lines
Quoted 14 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+    .get_features  = ifcvf_mdev_get_features,
+    .set_features  = ifcvf_mdev_set_features,
+    .get_status    = ifcvf_mdev_get_status,
+    .set_status    = ifcvf_mdev_set_status,
+    .get_queue_max = ifcvf_mdev_get_queue_max,
+    .get_vq_state   = ifcvf_mdev_get_vq_state,
+    .set_vq_state   = ifcvf_mdev_set_vq_state,
+    .set_vq_cb      = ifcvf_mdev_set_vq_cb,
+    .set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    .get_vq_ready    = ifcvf_mdev_get_vq_ready,
+    .set_vq_num     = ifcvf_mdev_set_vq_num,
+    .set_vq_address = ifcvf_mdev_set_vq_address,
+    .kick_vq        = ifcvf_mdev_kick_vq,
+    .get_generation    = ifcvf_mdev_get_generation,
+    .get_version    = ifcvf_mdev_get_version,
+    .get_device_id    = ifcvf_mdev_get_device_id,
+    .get_vendor_id    = ifcvf_mdev_get_vendor_id,
+    .get_vq_align    = ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

Quoted 2 lines
Quoted 1 line

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

IMHO, currently the driver can work without set/get_config, vhost_mdev doesn't call them for now.

Yes, but it was required by virtio_mdev for host driver to work, and it looks to me it's not hard to add them. If possible please add them and "virtio" type then we can use the ops for both the case of VM and containers.

Quoted 30 lines
Quoted 22 lines
Quoted 22 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+    int vector, i, ret, irq;
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+    struct ifcvf_hw *vf = &adapter->vf;
+
+    ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+            IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+    if (ret < 0) {
+        IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+        return ret;
+    }
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        ret = request_irq(irq, ifcvf_intr_handler, 0,
+                pci_name(pdev), &vf->vring[i]);
+        if (ret) {
Quoted 8 lines
Quoted 5 lines

+            IFC_ERR(adapter->dev,
+                "Failed to request irq for vq %d.\n", i); +            return ret;
+        }
+    }

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 1 line

I think it would be very rarely that can not get enough vectors.

Right.

Quoted 154 lines
Quoted 61 lines
Quoted 61 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+
+    return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+    int i, vector, irq;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        free_irq(irq, &vf->vring[i]);
+    }
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, 

char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+{
+    const char *name = "vhost accelerator (virtio ring compatible)";
+
+    return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device 

*dev,
+                   char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +30 -0
+{
+    return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                    struct device *dev, char *buf)
+{
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+            struct device *dev, char *buf)
+{
+    return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+
+static struct attribute *mdev_types_attrs[] = {
+    &mdev_type_attr_name.attr,
+    &mdev_type_attr_device_api.attr,
+    &mdev_type_attr_available_instances.attr,
+    &mdev_type_attr_type.attr,
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 5 lines

+    NULL,

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+};
+
+static struct attribute_group mdev_type_group = {
+    .name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

agreed!

Quoted 84 lines
Quoted 70 lines

+    .attrs = mdev_types_attrs,

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+    &mdev_type_group,
+    NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+    NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct 

mdev_device *mdev)

drivers/vhost/ifcvf/ifcvf_main.c new file +49 -0
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    int ret = 0;
+
+    mutex_lock(&adapter->mdev_lock);
+
+    if (adapter->mdev_count < 1) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    mdev_set_class_id(mdev, MDEV_ID_VHOST);
+    mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+    mdev_set_drvdata(mdev, adapter);
+    mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+    INIT_LIST_HEAD(&adapter->dma_maps);
+    adapter->mdev_count--;
+
+out:
+    mutex_unlock(&adapter->mdev_lock);
+    return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    mutex_lock(&adapter->mdev_lock);
+    adapter->mdev_count++;
+    mutex_unlock(&adapter->mdev_lock);
+
+    return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+    .owner            = THIS_MODULE,
+    .supported_type_groups    = mdev_type_groups,
+    .mdev_attr_groups    = mdev_dev_groups,
+    .create            = ifcvf_mdev_create,
+    .remove            = ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct 

pci_device_id *id)

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter;
+    struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    int ret, i;
+
+    adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+    if (adapter == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    mutex_init(&adapter->mdev_lock);
+    adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Quoted 2 lines
Quoted 1 line

Thanks

the driver scope for now doesn't support that, we can add these feature in next releases.

Not a must for this series, but to have a better interaction with management like libvirt, it's better.

Btw, do you have the plan to post PF drivers?

Thanks

#19 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 57 lines · Jason Wang
Quoted 55 lines · Lingshan
Quoted 52 lines · Jason Wang
Quoted 37 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541
+++++++++++++++++++++++++++++++++++++++

  1 file changed, 541 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING    "0.1"
+#define DRIVER_AUTHOR    "Intel Corporation"
+#define IFCVF_DRIVER_NAME    "ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+    struct vring_info *vring = arg;

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+    if (vring->cb.callback)
+        return vring->cb.callback(vring->cb.private);
+
+    return IRQ_HANDLED;
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+    return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 6 lines
Quoted 3 lines

Hi Jason,

Thanks for your comments, for now driver just support these features.

Ok, it should work but less flexible, we can change it in the future.

sure!

Quoted 27 lines
Quoted 27 lines
Quoted 27 lines
Quoted 14 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 qid)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 6 lines
Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Quoted 164 lines
Quoted 164 lines
Quoted 4 lines
Quoted 12 lines

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 9 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 

qid, u64 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

I will remove it.

Quoted 112 lines
Quoted 3 lines

+ vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 104 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+
+    return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, u16 

idx,
+                     u64 desc_area, u64 driver_area, +                     u64 device_area)

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].desc = desc_area;
+    vf->vring[idx].avail = driver_area;
+    vf->vring[idx].used = device_area;
+
+    return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 

qid, u32 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+                u16 qid, bool ready)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 

qid)

drivers/vhost/ifcvf/ifcvf_main.c new file +64 -0
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+                 struct virtio_mdev_callback *cb)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+    return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+    return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+    return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+    return IFCVF_QUEUE_ALIGNMENT;
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+    int i, ret;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) { +        if (!vf->vring[i].ready)
+            break;

Looks like error should be returned here?

agreed!

Quoted 26 lines
Quoted 13 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +11 -0
+
+        if (!vf->vring[i].size)
+            break;
+
+        if (!vf->vring[i].desc || !vf->vring[i].avail ||
+            !vf->vring[i].used)
+            break;
+    }
+    vf->nr_vring = i;
+
+    ret = ifcvf_start_hw(vf);

drivers/vhost/ifcvf/ifcvf_main.c line 184

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+    int i;
+
+    for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+        vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 4 lines
Quoted 1 line

I think even we set callback = NULL, the code is still there, on-going routines would not be effected.

Ok I think you mean when ifcvf_stop_hw() return, hardware will not respond to e.g kick and other events etc.

Yes, ifcvf_stop_hw() would disable the hw.

Quoted 82 lines
Quoted 82 lines
Quoted 45 lines
Quoted 45 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +24 -0
+
+    ifcvf_stop_hw(vf);
+
+    return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+    int i;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vf->vring[i].last_used_idx = 0;
+        vf->vring[i].last_avail_idx = 0;
+        vf->vring[i].desc = 0;
+        vf->vring[i].avail = 0;
+        vf->vring[i].used = 0;
+        vf->vring[i].ready = 0;
+        vf->vring->cb.callback = NULL;
+        vf->vring->cb.private = NULL;
+    }
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 

status)

drivers/vhost/ifcvf/ifcvf_main.c new file +16 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->status = status;
+
+    if (status == 0) {
+        ifcvf_stop_datapath(adapter);
+        ifcvf_reset_vring(adapter);
+        return;
+    }
+
+    if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+        ifcvf_start_datapath(adapter);
+        return;
+    }
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+    return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

will change that.

Quoted 28 lines
Quoted 14 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+    .get_features  = ifcvf_mdev_get_features,
+    .set_features  = ifcvf_mdev_set_features,
+    .get_status    = ifcvf_mdev_get_status,
+    .set_status    = ifcvf_mdev_set_status,
+    .get_queue_max = ifcvf_mdev_get_queue_max,
+    .get_vq_state   = ifcvf_mdev_get_vq_state,
+    .set_vq_state   = ifcvf_mdev_set_vq_state,
+    .set_vq_cb      = ifcvf_mdev_set_vq_cb,
+    .set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    .get_vq_ready    = ifcvf_mdev_get_vq_ready,
+    .set_vq_num     = ifcvf_mdev_set_vq_num,
+    .set_vq_address = ifcvf_mdev_set_vq_address,
+    .kick_vq        = ifcvf_mdev_kick_vq,
+    .get_generation    = ifcvf_mdev_get_generation,
+    .get_version    = ifcvf_mdev_get_version,
+    .get_device_id    = ifcvf_mdev_get_device_id,
+    .get_vendor_id    = ifcvf_mdev_get_vendor_id,
+    .get_vq_align    = ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

Quoted 5 lines
Quoted 2 lines
Quoted 1 line

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

IMHO, currently the driver can work without set/get_config, vhost_mdev doesn't call them for now.

Yes, but it was required by virtio_mdev for host driver to work, and it looks to me it's not hard to add them. If possible please add them and "virtio" type then we can use the ops for both the case of VM and containers.

sure

Quoted 193 lines
Quoted 32 lines
Quoted 24 lines
Quoted 24 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+    int vector, i, ret, irq;
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+    struct ifcvf_hw *vf = &adapter->vf;
+
+    ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+            IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+    if (ret < 0) {
+        IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+        return ret;
+    }
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        ret = request_irq(irq, ifcvf_intr_handler, 0,
+                pci_name(pdev), &vf->vring[i]);
+        if (ret) {
Quoted 8 lines
Quoted 5 lines

+            IFC_ERR(adapter->dev,
+                "Failed to request irq for vq %d.\n", i); +            return ret;
+        }
+    }

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 1 line

I think it would be very rarely that can not get enough vectors.

Right.

Quoted 155 lines
Quoted 61 lines
Quoted 61 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+
+    return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+    int i, vector, irq;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        free_irq(irq, &vf->vring[i]);
+    }
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device *dev, 

char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+{
+    const char *name = "vhost accelerator (virtio ring compatible)";
+
+    return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct device 

*dev,
+                   char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +30 -0
+{
+    return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                    struct device *dev, char *buf)
+{
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+            struct device *dev, char *buf)
+{
+    return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+
+static struct attribute *mdev_types_attrs[] = {
+    &mdev_type_attr_name.attr,
+    &mdev_type_attr_device_api.attr,
+    &mdev_type_attr_available_instances.attr,
+    &mdev_type_attr_type.attr,
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 5 lines

+    NULL,

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+};
+
+static struct attribute_group mdev_type_group = {
+    .name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

agreed!

Quoted 85 lines
Quoted 70 lines

+    .attrs = mdev_types_attrs,

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+    &mdev_type_group,
+    NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+    NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct 

mdev_device *mdev)

drivers/vhost/ifcvf/ifcvf_main.c new file +49 -0
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    int ret = 0;
+
+    mutex_lock(&adapter->mdev_lock);
+
+    if (adapter->mdev_count < 1) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    mdev_set_class_id(mdev, MDEV_ID_VHOST);
+    mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+    mdev_set_drvdata(mdev, adapter);
+    mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+    INIT_LIST_HEAD(&adapter->dma_maps);
+    adapter->mdev_count--;
+
+out:
+    mutex_unlock(&adapter->mdev_lock);
+    return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    mutex_lock(&adapter->mdev_lock);
+    adapter->mdev_count++;
+    mutex_unlock(&adapter->mdev_lock);
+
+    return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+    .owner            = THIS_MODULE,
+    .supported_type_groups    = mdev_type_groups,
+    .mdev_attr_groups    = mdev_dev_groups,
+    .create            = ifcvf_mdev_create,
+    .remove            = ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct 

pci_device_id *id)

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter;
+    struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    int ret, i;
+
+    adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+    if (adapter == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    mutex_init(&adapter->mdev_lock);
+    adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Thanks

Quoted 8 lines
Quoted 1 line

the driver scope for now doesn't support that, we can add these feature in next releases.

Not a must for this series, but to have a better interaction with management like libvirt, it's better.

Btw, do you have the plan to post PF drivers?

Thanks

Maybe we can workout PF driver in next release :)
THanks,
BR
Zhu Lingshan

Quoted 2 lines
#20 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 14 lines · Zhu Lingshan
Quoted 12 lines · Jason Wang
Quoted 10 lines · Lingshan
Quoted 8 lines · Jason Wang
Quoted 6 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---

[...]

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 20 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 

qid)

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Just make sure I understand here, I believe you mean reading last_avail_idx through IO bar here?

Thanks

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#21 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

In reply to Zhu Lingshan
Quoted 14 lines · Zhu Lingshan
Quoted 12 lines · Jason Wang
Quoted 10 lines · Lingshan
Quoted 8 lines · Jason Wang
Quoted 6 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---

[...]

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 20 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 

qid)

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Just make sure I understand here, I believe you mean reading last_avail_idx through IO bar here?

Thanks

#22 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 54 lines · Jason Wang
Quoted 14 lines · Zhu Lingshan
Quoted 12 lines · Jason Wang
Quoted 10 lines · Lingshan
Quoted 8 lines · Jason Wang
Quoted 6 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---

[...]

Quoted 33 lines
Quoted 33 lines
Quoted 28 lines
Quoted 20 lines
Quoted 20 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 

qid)

Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Quoted 8 lines
Quoted 3 lines
Quoted 1 line

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Just make sure I understand here, I believe you mean reading last_avail_idx through IO bar here?

Thanks

Hi Jason,

Yes, I mean last_avail_idx. is that correct?

THanks

Quoted 2 lines
#23 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 63 lines · Lingshan
Quoted 61 lines · Jason Wang
Quoted 14 lines · Zhu Lingshan
Quoted 12 lines · Jason Wang
Quoted 10 lines · Lingshan
Quoted 8 lines · Jason Wang
Quoted 6 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---

[...]

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 20 lines
(quoted diff) +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, 

u64 features)

(quoted diff) +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, 

u16 qid)

Review

Quoted 5 lines
(quoted diff) +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Just make sure I understand here, I believe you mean reading last_avail_idx through IO bar here?

Quoted 7 lines
Quoted 1 line

Thanks

Hi Jason,

Yes, I mean last_avail_idx. is that correct?

THanks

Yes.

Thanks

Quoted 3 lines
Quoted 3 lines

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#24 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

In reply to Zhu, Lingshan
Quoted 63 lines · Lingshan
Quoted 61 lines · Jason Wang
Quoted 14 lines · Zhu Lingshan
Quoted 12 lines · Jason Wang
Quoted 10 lines · Lingshan
Quoted 8 lines · Jason Wang
Quoted 6 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---

[...]

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 20 lines
(quoted diff) +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, 

u64 features)

(quoted diff) +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, 

u16 qid)

Review

Quoted 5 lines
(quoted diff) +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 8 lines
Quoted 6 lines
Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Just make sure I understand here, I believe you mean reading last_avail_idx through IO bar here?

Quoted 7 lines
Quoted 1 line

Thanks

Hi Jason,

Yes, I mean last_avail_idx. is that correct?

THanks

Yes.

Thanks

Quoted 3 lines
Quoted 3 lines
#25 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

In reply to Zhu Lingshan
Quoted 354 lines · Zhu Lingshan
Quoted 57 lines · Jason Wang
Quoted 55 lines · Lingshan
Quoted 52 lines · Jason Wang
Quoted 37 lines · Zhu Lingshan

This commit introduced IFC VF operations for vdpa, which complys to vhost_mdev interfaces, handles IFC VF initialization, configuration and removal.

---
  drivers/vhost/ifcvf/ifcvf_main.c | 541
+++++++++++++++++++++++++++++++++++++++

  1 file changed, 541 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
new file mode 100644
index 000000000000..c48a29969a85
@@ -0,0 +1,541 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2019 Intel Corporation.
+ */
+
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/mdev.h>
+#include <linux/pci.h>
+#include <linux/sysfs.h>
+
+#include "ifcvf_base.h"
+
+#define VERSION_STRING    "0.1"
+#define DRIVER_AUTHOR    "Intel Corporation"
+#define IFCVF_DRIVER_NAME    "ifcvf"
+
+static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
+{
+    struct vring_info *vring = arg;

drivers/vhost/ifcvf/ifcvf_main.c line 30

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+
+    if (vring->cb.callback)
+        return vring->cb.callback(vring->cb.private);
+
+    return IRQ_HANDLED;
+}
+
+static u64 ifcvf_mdev_get_features(struct mdev_device *mdev)
+{
+    return IFC_SUPPORTED_FEATURES;

I would expect this should be done by querying the hw. Or IFC VF can't get any update through its firmware?

Quoted 6 lines
Quoted 3 lines

Hi Jason,

Thanks for your comments, for now driver just support these features.

Ok, it should work but less flexible, we can change it in the future.

sure!

Quoted 28 lines
Quoted 28 lines
Quoted 28 lines
Quoted 20 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_features(struct mdev_device *mdev, u64 

features)

drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->req_features = features;
+
+    return 0;
+}
+
+static u64 ifcvf_mdev_get_vq_state(struct mdev_device *mdev, u16 

qid)

drivers/vhost/ifcvf/ifcvf_main.c line 48

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].last_avail_idx;

Does this really work? I'd expect it should be fetched from hw since it's an internal state.

Quoted 6 lines
Quoted 1 line

for now, it's working, we intend to support LM in next version drivers.

I'm not sure I understand here, I don't see any synchronization between the hardware and last_avail_idx, so last_avail_idx should not change.

Btw, what did "LM" mean :) ?

I can add bar IO operations here, LM = live migration, sorry for the abbreviation.

Quoted 164 lines
Quoted 164 lines
Quoted 4 lines
Quoted 12 lines

drivers/vhost/ifcvf/ifcvf_main.c line 56

Quoted 9 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +3 -0
+}
+
+static int ifcvf_mdev_set_vq_state(struct mdev_device *mdev, u16 

qid, u64 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].last_used_idx = num;

I fail to understand why last_used_idx is needed. It looks to me the used idx in the used ring is sufficient.

I will remove it.

Quoted 112 lines
Quoted 3 lines

+ vf->vring[qid].last_avail_idx = num;

Do we need a synchronization with hw immediately here?

Quoted 104 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+
+    return 0;
+}
+
+static int ifcvf_mdev_set_vq_address(struct mdev_device *mdev, 

u16 idx,
+                     u64 desc_area, u64 driver_area, +                     u64 device_area)

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].desc = desc_area;
+    vf->vring[idx].avail = driver_area;
+    vf->vring[idx].used = device_area;
+
+    return 0;
+}
+
+static void ifcvf_mdev_set_vq_num(struct mdev_device *mdev, u16 

qid, u32 num)

drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].size = num;
+}
+
+static void ifcvf_mdev_set_vq_ready(struct mdev_device *mdev,
+                u16 qid, bool ready)
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[qid].ready = ready;
+}
+
+static bool ifcvf_mdev_get_vq_ready(struct mdev_device *mdev, u16 

qid)

drivers/vhost/ifcvf/ifcvf_main.c new file +64 -0
+{
+
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->vring[qid].ready;
+}
+
+static void ifcvf_mdev_set_vq_cb(struct mdev_device *mdev, u16 idx,
+                 struct virtio_mdev_callback *cb)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->vring[idx].cb = *cb;
+}
+
+static void ifcvf_mdev_kick_vq(struct mdev_device *mdev, u16 idx)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    ifcvf_notify_queue(vf, idx);
+}
+
+static u8 ifcvf_mdev_get_status(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->status;
+}
+
+static u32 ifcvf_mdev_get_generation(struct mdev_device *mdev)
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    return vf->generation;
+}
+
+static int ifcvf_mdev_get_version(struct mdev_device *mdev)
+{
+    return VIRTIO_MDEV_VERSION;
+}
+
+static u32 ifcvf_mdev_get_device_id(struct mdev_device *mdev)
+{
+    return IFCVF_DEVICE_ID;
+}
+
+static u32 ifcvf_mdev_get_vendor_id(struct mdev_device *mdev)
+{
+    return IFCVF_VENDOR_ID;
+}
+
+static u16 ifcvf_mdev_get_vq_align(struct mdev_device *mdev)
+{
+    return IFCVF_QUEUE_ALIGNMENT;
+}
+
+static int ifcvf_start_datapath(void *private)
+{
+    int i, ret;
Quoted 8 lines
Quoted 5 lines

+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private); +
+    for (i = 0; i < (IFCVF_MAX_QUEUE_PAIRS * 2); i++) { +        if (!vf->vring[i].ready)
+            break;

Looks like error should be returned here?

agreed!

Quoted 26 lines
Quoted 13 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +11 -0
+
+        if (!vf->vring[i].size)
+            break;
+
+        if (!vf->vring[i].desc || !vf->vring[i].avail ||
+            !vf->vring[i].used)
+            break;
+    }
+    vf->nr_vring = i;
+
+    ret = ifcvf_start_hw(vf);

drivers/vhost/ifcvf/ifcvf_main.c line 184

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    return ret;
+}
+
+static int ifcvf_stop_datapath(void *private)
+{
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(private);
+    int i;
+
+    for (i = 0; i < IFCVF_MAX_QUEUES; i++)
+        vf->vring[i].cb.callback = NULL;

Any synchronization is needed for the vq irq handler?

Quoted 4 lines
Quoted 1 line

I think even we set callback = NULL, the code is still there, on-going routines would not be effected.

Ok I think you mean when ifcvf_stop_hw() return, hardware will not respond to e.g kick and other events etc.

Yes, ifcvf_stop_hw() would disable the hw.

Quoted 82 lines
Quoted 82 lines
Quoted 45 lines
Quoted 45 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +24 -0
+
+    ifcvf_stop_hw(vf);
+
+    return 0;
+}
+
+static void ifcvf_reset_vring(struct ifcvf_adapter *adapter)
+{
+    int i;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vf->vring[i].last_used_idx = 0;
+        vf->vring[i].last_avail_idx = 0;
+        vf->vring[i].desc = 0;
+        vf->vring[i].avail = 0;
+        vf->vring[i].used = 0;
+        vf->vring[i].ready = 0;
+        vf->vring->cb.callback = NULL;
+        vf->vring->cb.private = NULL;
+    }
+}
+
+static void ifcvf_mdev_set_status(struct mdev_device *mdev, u8 

status)

drivers/vhost/ifcvf/ifcvf_main.c new file +16 -0
+{
+    struct ifcvf_adapter *adapter = mdev_get_drvdata(mdev);
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+
+    vf->status = status;
+
+    if (status == 0) {
+        ifcvf_stop_datapath(adapter);
+        ifcvf_reset_vring(adapter);
+        return;
+    }
+
+    if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+        ifcvf_start_datapath(adapter);
+        return;
+    }
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 229

Quoted 5 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +5 -0
+}
+
+static u16 ifcvf_mdev_get_queue_max(struct mdev_device *mdev)
+{
+    return IFCVF_MAX_QUEUES;

The name is confusing, it was used to return the maximum queue size. In new version of virtio-mdev, the callback was renamed as get_vq_num_max().

will change that.

Quoted 28 lines
Quoted 14 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+}
+
+static struct virtio_mdev_device_ops ifc_mdev_ops = {
+    .get_features  = ifcvf_mdev_get_features,
+    .set_features  = ifcvf_mdev_set_features,
+    .get_status    = ifcvf_mdev_get_status,
+    .set_status    = ifcvf_mdev_set_status,
+    .get_queue_max = ifcvf_mdev_get_queue_max,
+    .get_vq_state   = ifcvf_mdev_get_vq_state,
+    .set_vq_state   = ifcvf_mdev_set_vq_state,
+    .set_vq_cb      = ifcvf_mdev_set_vq_cb,
+    .set_vq_ready   = ifcvf_mdev_set_vq_ready,

drivers/vhost/ifcvf/ifcvf_main.c line 251

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    .get_vq_ready    = ifcvf_mdev_get_vq_ready,
+    .set_vq_num     = ifcvf_mdev_set_vq_num,
+    .set_vq_address = ifcvf_mdev_set_vq_address,
+    .kick_vq        = ifcvf_mdev_kick_vq,
+    .get_generation    = ifcvf_mdev_get_generation,
+    .get_version    = ifcvf_mdev_get_version,
+    .get_device_id    = ifcvf_mdev_get_device_id,
+    .get_vendor_id    = ifcvf_mdev_get_vendor_id,
+    .get_vq_align    = ifcvf_mdev_get_vq_align,
+};

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

Quoted 6 lines
Quoted 5 lines
Quoted 2 lines
Quoted 1 line

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

IMHO, currently the driver can work without set/get_config, vhost_mdev doesn't call them for now.

Yes, but it was required by virtio_mdev for host driver to work, and it looks to me it's not hard to add them. If possible please add them and "virtio" type then we can use the ops for both the case of VM and containers.

sure

Hello Jason,

Just want to double confirm the implementation of set/get_config, for now, dev_cfg only contains mac[6], status and max_virtqueue_pairs, is that enough to support virtio_mdev?

THanks!

Quoted 207 lines
Quoted 193 lines
Quoted 32 lines
Quoted 24 lines
Quoted 24 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +20 -0
+
+static int ifcvf_init_msix(struct ifcvf_adapter *adapter)
+{
+    int vector, i, ret, irq;
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+    struct ifcvf_hw *vf = &adapter->vf;
+
+    ret = pci_alloc_irq_vectors(pdev, IFCVF_MAX_INTR,
+            IFCVF_MAX_INTR, PCI_IRQ_MSIX);
+    if (ret < 0) {
+        IFC_ERR(adapter->dev, "Failed to alloc irq vectors.\n");
+        return ret;
+    }
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        ret = request_irq(irq, ifcvf_intr_handler, 0,
+                pci_name(pdev), &vf->vring[i]);
+        if (ret) {
Quoted 8 lines
Quoted 5 lines

+            IFC_ERR(adapter->dev,
+                "Failed to request irq for vq %d.\n", i); +            return ret;
+        }
+    }

Do we need to provide fallback when we can't do per vq MSIX?

Quoted 1 line

I think it would be very rarely that can not get enough vectors.

Right.

Quoted 155 lines
Quoted 61 lines
Quoted 61 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +18 -0
+
+    return 0;
+}
+
+static void ifcvf_destroy_adapter(struct ifcvf_adapter *adapter)
+{
+    int i, vector, irq;
+    struct ifcvf_hw *vf = IFC_PRIVATE_TO_VF(adapter);
+    struct pci_dev *pdev = to_pci_dev(adapter->dev);
+
+    for (i = 0; i < IFCVF_MAX_QUEUE_PAIRS * 2; i++) {
+        vector = i + IFCVF_MSI_QUEUE_OFF;
+        irq = pci_irq_vector(pdev, vector);
+        free_irq(irq, &vf->vring[i]);
+    }
+}
+
+static ssize_t name_show(struct kobject *kobj, struct device 

*dev, char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +8 -0
+{
+    const char *name = "vhost accelerator (virtio ring compatible)";
+
+    return sprintf(buf, "%s\n", name);
+}
+MDEV_TYPE_ATTR_RO(name);
+
+static ssize_t device_api_show(struct kobject *kobj, struct 

device *dev,
+                   char *buf)

drivers/vhost/ifcvf/ifcvf_main.c new file +30 -0
+{
+    return sprintf(buf, "%s\n", VIRTIO_MDEV_DEVICE_API_STRING);
+}
+MDEV_TYPE_ATTR_RO(device_api);
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                    struct device *dev, char *buf)
+{
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    return sprintf(buf, "%d\n", adapter->mdev_count);
+}
+
+MDEV_TYPE_ATTR_RO(available_instances);
+
+static ssize_t type_show(struct kobject *kobj,
+            struct device *dev, char *buf)
+{
+    return sprintf(buf, "%s\n", "net");
+}
+
+MDEV_TYPE_ATTR_RO(type);
+
+
+static struct attribute *mdev_types_attrs[] = {
+    &mdev_type_attr_name.attr,
+    &mdev_type_attr_device_api.attr,
+ &mdev_type_attr_available_instances.attr,
+    &mdev_type_attr_type.attr,
Quoted 8 lines

drivers/vhost/ifcvf/ifcvf_main.c line 338

Quoted 5 lines

+    NULL,

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+};
+
+static struct attribute_group mdev_type_group = {
+    .name  = "vdpa_virtio",

To be consistent, it should be "vhost" or "virtio".

agreed!

Quoted 85 lines
Quoted 70 lines

+    .attrs = mdev_types_attrs,

drivers/vhost/ifcvf/ifcvf_main.c new file +12 -0
+};
+
+static struct attribute_group *mdev_type_groups[] = {
+    &mdev_type_group,
+    NULL,
+};
+
+const struct attribute_group *mdev_dev_groups[] = {
+    NULL,
+};
+
+static int ifcvf_mdev_create(struct kobject *kobj, struct 

mdev_device *mdev)

drivers/vhost/ifcvf/ifcvf_main.c new file +49 -0
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+    int ret = 0;
+
+    mutex_lock(&adapter->mdev_lock);
+
+    if (adapter->mdev_count < 1) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    mdev_set_class_id(mdev, MDEV_ID_VHOST);
+    mdev_set_dev_ops(mdev, &ifc_mdev_ops);
+
+    mdev_set_drvdata(mdev, adapter);
+    mdev_set_iommu_device(mdev_dev(mdev), dev);
+
+    INIT_LIST_HEAD(&adapter->dma_maps);
+    adapter->mdev_count--;
+
+out:
+    mutex_unlock(&adapter->mdev_lock);
+    return ret;
+}
+
+static int ifcvf_mdev_remove(struct mdev_device *mdev)
+{
+    struct device *dev = mdev_parent_dev(mdev);
+    struct pci_dev *pdev = to_pci_dev(dev);
+    struct ifcvf_adapter *adapter = pci_get_drvdata(pdev);
+
+    mutex_lock(&adapter->mdev_lock);
+    adapter->mdev_count++;
+    mutex_unlock(&adapter->mdev_lock);
+
+    return 0;
+}
+
+static struct mdev_parent_ops ifcvf_mdev_fops = {
+    .owner            = THIS_MODULE,
+    .supported_type_groups    = mdev_type_groups,
+    .mdev_attr_groups    = mdev_dev_groups,
+    .create            = ifcvf_mdev_create,
+    .remove            = ifcvf_mdev_remove,
+};
+
+static int ifcvf_probe(struct pci_dev *pdev, const struct 

pci_device_id *id)

drivers/vhost/ifcvf/ifcvf_main.c new file +4 -0
+{
+    struct device *dev = &pdev->dev;
+    struct ifcvf_adapter *adapter;
+    struct ifcvf_hw *vf;

drivers/vhost/ifcvf/ifcvf_main.c line 414

Quoted 10 lines
drivers/vhost/ifcvf/ifcvf_main.c new file +10 -0
+    int ret, i;
+
+    adapter = kzalloc(sizeof(struct ifcvf_adapter), GFP_KERNEL);
+    if (adapter == NULL) {
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    mutex_init(&adapter->mdev_lock);
+    adapter->mdev_count = 1;

So this is per VF based vDPA implementation, which seems not convenient for management.  Anyhow we can control the creation in PF?

Thanks

Quoted 8 lines
Quoted 1 line

the driver scope for now doesn't support that, we can add these feature in next releases.

Not a must for this series, but to have a better interaction with management like libvirt, it's better.

Btw, do you have the plan to post PF drivers?

Thanks

Maybe we can workout PF driver in next release :)
THanks,
BR
Zhu Lingshan

Quoted 2 lines
#26 lore raw

Re: [RFC 2/2] vhost: IFC VF vdpa layer

Quoted 8 lines · Lingshan
Quoted 8 lines
Quoted 8 lines
Quoted 6 lines
Quoted 5 lines

set_config/get_config is missing. It looks to me they are not hard, just implementing the access to dev_cfg. It's key to make kernel virtio driver to work.

And in the new version of virito-mdev, features like _F_LOG_ALL should be advertised through get_mdev_features.

IMHO, currently the driver can work without set/get_config, vhost_mdev doesn't call them for now.

Quoted 8 lines
Quoted 2 lines
Quoted 1 line

Yes, but it was required by virtio_mdev for host driver to work, and it looks to me it's not hard to add them. If possible please add them and "virtio" type then we can use the ops for both the case of VM and containers.

sure

Hello Jason,

Just want to double confirm the implementation of set/get_config, for now, dev_cfg only contains mac[6], status and max_virtqueue_pairs, is that enough to support virtio_mdev?

THanks!

Yes, and it depends on the features that you want to advertise. If you don't want to advertise MQ, there's no need to expose max_virtqueue_pairs.

Thanks

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#27 lore raw
In reply to Zhu Lingshan

failed to send to kvm list, resend, sorry for the inconvenience.

THanks,
BR
Zhu Lingshan

Quoted 29 lines · Zhu Lingshan

Hi all:

This series intends to introduce Intel IFC VF NIC driver for Vhost Data Plane Acceleration.

Here comes two main parts, one is ifcvf_base layer, which handles hardware operations. The other is ifcvf_main layer handles VF initialization, configuration and removal, which depends on and complys to vhost_mdev https://lkml.org/lkml/2019/9/26/15

This is a first RFC try, please help review.

Thanks!
BR
Zhu Lingshan

Zhu Lingshan (2):
vhost: IFC VF hardware operation layer
vhost: IFC VF vdpa layer

  drivers/vhost/ifcvf/ifcvf_base.c | 390 ++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++
  drivers/vhost/ifcvf/ifcvf_main.c | 541 +++++++++++++++++++++++++++++++++++++++
  3 files changed, 1068 insertions(+)

create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

#28 lore raw
Quoted 15 lines · Zhu Lingshan

failed to send to kvm list, resend, sorry for the inconvenience.

THanks,
BR
Zhu Lingshan

Quoted 8 lines · Zhu Lingshan

Hi all:
  This series intends to introduce Intel IFC VF NIC driver for Vhost Data Plane Acceleration.
  Here comes two main parts, one is ifcvf_base layer, which handles hardware operations. The other is ifcvf_main layer handles VF initialization, configuration and removal, which depends on and complys to vhost_mdev https://lkml.org/lkml/2019/9/26/15
  This is a first RFC try, please help review.

It would be helpful if yon can describe which kinds of test that has been done for this series.

Thanks

Quoted 9 lines
Quoted 9 lines

  Thanks!
BR
Zhu Lingshan

Zhu Lingshan (2):
   vhost: IFC VF hardware operation layer
   vhost: IFC VF vdpa layer

Quoted 9 lines
Quoted 9 lines
  drivers/vhost/ifcvf/ifcvf_base.c | 390 ++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++
  drivers/vhost/ifcvf/ifcvf_main.c | 541 
+++++++++++++++++++++++++++++++++++++++
  3 files changed, 1068 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

_______________________________________________
Virtualization mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

#29 lore raw
In reply to Zhu Lingshan
Quoted 15 lines · Zhu Lingshan

failed to send to kvm list, resend, sorry for the inconvenience.

THanks,
BR
Zhu Lingshan

Quoted 8 lines · Zhu Lingshan

Hi all:
  This series intends to introduce Intel IFC VF NIC driver for Vhost Data Plane Acceleration.
  Here comes two main parts, one is ifcvf_base layer, which handles hardware operations. The other is ifcvf_main layer handles VF initialization, configuration and removal, which depends on and complys to vhost_mdev https://lkml.org/lkml/2019/9/26/15
  This is a first RFC try, please help review.

It would be helpful if yon can describe which kinds of test that has been done for this series.

Thanks

Quoted 18 lines
Quoted 18 lines

  Thanks!
BR
Zhu Lingshan

Zhu Lingshan (2):
   vhost: IFC VF hardware operation layer
   vhost: IFC VF vdpa layer

  drivers/vhost/ifcvf/ifcvf_base.c | 390 ++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++
  drivers/vhost/ifcvf/ifcvf_main.c | 541 
+++++++++++++++++++++++++++++++++++++++
  3 files changed, 1068 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c

#30 lore raw
Quoted 19 lines · Jason Wang
Quoted 15 lines · Zhu Lingshan

failed to send to kvm list, resend, sorry for the inconvenience.

THanks,
BR
Zhu Lingshan

Quoted 8 lines · Zhu Lingshan

Hi all:
  This series intends to introduce Intel IFC VF NIC driver for Vhost Data Plane Acceleration.
  Here comes two main parts, one is ifcvf_base layer, which handles hardware operations. The other is ifcvf_main layer handles VF initialization, configuration and removal, which depends on and complys to vhost_mdev https://lkml.org/lkml/2019/9/26/15
  This is a first RFC try, please help review.

Quoted 4 lines

It would be helpful if yon can describe which kinds of test that has been done for this series.

Thanks

Hi Jason,

It passed ping and netperf tests(two VMs, each has one ifc vf backed mdev nic), I will mention this in next cover letter Thanks!

BR
Zhu Lingshan

Quoted 18 lines
Quoted 18 lines
Quoted 18 lines

  Thanks!
BR
Zhu Lingshan

Zhu Lingshan (2):
   vhost: IFC VF hardware operation layer
   vhost: IFC VF vdpa layer

  drivers/vhost/ifcvf/ifcvf_base.c | 390 ++++++++++++++++++++++++++++
  drivers/vhost/ifcvf/ifcvf_base.h | 137 ++++++++++
  drivers/vhost/ifcvf/ifcvf_main.c | 541 
+++++++++++++++++++++++++++++++++++++++
  3 files changed, 1068 insertions(+)

  create mode 100644 drivers/vhost/ifcvf/ifcvf_base.c create mode 100644 drivers/vhost/ifcvf/ifcvf_base.h create mode 100644 drivers/vhost/ifcvf/ifcvf_main.c