PCI bus support

Cc Thomas Huth <[email protected]>

Cc Andrew Jones <[email protected]>

Cc Peter Xu <[email protected]>

Reviewed-by Andrew Jones <[email protected]>

Signed-off-by Alexander Gordeev <[email protected]>

---
 lib/pci.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 lib/pci.h |  3 +++
 2 files changed, 81 insertions(+)

lib/pci.c +30 -0

index 42f47d93abfd..462cddc8a806 100644
@@ -126,3 +126,81 @@ bool pci_bar_is64(pcidevaddr_t dev, int bar_num)
 	return (bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK) ==
 		      PCI_BASE_ADDRESS_MEM_TYPE_64;
 }
+
+static void pci_dev_print(pcidevaddr_t dev)
+{
+	uint16_t vendor_id = pci_config_readw(dev, PCI_VENDOR_ID);
+	uint16_t device_id = pci_config_readw(dev, PCI_DEVICE_ID);
+	uint8_t header = pci_config_readb(dev, PCI_HEADER_TYPE);
+	uint8_t progif = pci_config_readb(dev, PCI_CLASS_PROG);
+	uint8_t subclass = pci_config_readb(dev, PCI_CLASS_DEVICE);
+	uint8_t class = pci_config_readb(dev, PCI_CLASS_DEVICE + 1);
+	int i;
+
+	printf("dev %2d fn %d vendor_id %04x device_id %04x type %02x "
+	       "progif %02x class %02x subclass %02x\n",
+	       dev / 8, dev % 8, vendor_id, device_id, header,
+	       progif, class, subclass);
+
+	if ((header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL)
+		return;
+
+	for (i = 0; i < 6; i++) {
+		phys_addr_t size, start, end;
+		uint32_t bar;
+
+		size = pci_bar_size(dev, i);
+		if (!size)
+			continue;
+
+		start = pci_bar_get_addr(dev, i);
+		end = start + size - 1;
+

lib/pci.c +38 -0

+
+		if (bar & PCI_BASE_ADDRESS_SPACE_IO) {
+			printf("PIO]\n");
+			continue;
+		}
+
+		printf("MEM");
+
+		switch (bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK) {
+		case PCI_BASE_ADDRESS_MEM_TYPE_32:
+			printf("32");
+			break;
+		case PCI_BASE_ADDRESS_MEM_TYPE_1M:
+			printf("1M");
+			break;
+		case PCI_BASE_ADDRESS_MEM_TYPE_64:
+			printf("64");
+			break;
+		default:
+			assert(0);
+		}
+
+		if (bar & PCI_BASE_ADDRESS_MEM_PREFETCH)
+			printf("/p");
+
+		printf("]\n");
+	}
+}
+
+void pci_print(void)
+{
+	pcidevaddr_t dev;
+
+	for (dev = 0; dev < 256; ++dev) {
+		if (pci_dev_exists(dev))
+			pci_dev_print(dev);
+	}
+}

lib/pci.h +3 -0

index 1462aa2f0e1a..fc0940adc299 100644
@@ -15,6 +15,7 @@ enum {
 	PCIDEVADDR_INVALID = 0xffff,
 };
 
+extern void pci_print(void);
 extern bool pci_dev_exists(pcidevaddr_t dev);
 extern pcidevaddr_t pci_find_dev(uint16_t vendor_id, uint16_t device_id);
 

@@ -57,4 +58,6 @@ struct pci_test_dev_hdr {
 	uint8_t  name[];
 };
 
+#define  PCI_HEADER_TYPE_MASK		0x7f
+
 #endif /* PCI_H */

Signature · 6 lines

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Unlike x86, other architectures using generic ECAM PCI host do not have a luxury of PCI bus initialized by a BIOS and ready to use at start. Thus, we need allocate and assign resources to all devices, much like an architecture's firmware would do.

There is no any sort of resource management for memory and io spaces, since only ones-per-BAR allocations are expected and no deallocations at all.

Cc Thomas Huth <[email protected]>

Cc Andrew Jones <[email protected]>

Cc Peter Xu <[email protected]>

Reviewed-by Andrew Jones <[email protected]>

Signed-off-by Alexander Gordeev <[email protected]>

---
 lib/asm-generic/pci-host-bridge.h |  26 ++++
 lib/pci-host-generic.c            | 295 ++++++++++++++++++++++++++++++++++++++
 lib/pci-host-generic.h            |  46 ++++++
 lib/pci.c                         |   4 +-
 lib/pci.h                         |   3 +
 5 files changed, 372 insertions(+), 2 deletions(-)
 create mode 100644 lib/asm-generic/pci-host-bridge.h
 create mode 100644 lib/pci-host-generic.c
 create mode 100644 lib/pci-host-generic.h

lib/asm-generic/pci-host-bridge.h new file +26 -0

new file mode 100644
index 000000000000..ac32b85198e9
@@ -0,0 +1,26 @@
+#ifndef _ASM_PCI_HOST_BRIDGE_H_
+#define _ASM_PCI_HOST_BRIDGE_H_
+/*
+ * Copyright (C) 2016, Red Hat Inc, Alexander Gordeev <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.
+ */
+#include "libcflat.h"
+
+phys_addr_t pci_host_bridge_get_paddr(uint64_t addr);
+
+static inline
+phys_addr_t pci_translate_addr(pcidevaddr_t dev __unused, uint64_t addr)
+{
+	/*
+	 * Assume we only have single PCI host bridge in a system.
+	 */
+	return pci_host_bridge_get_paddr(addr);
+}
+
+uint8_t pci_config_readb(pcidevaddr_t dev, uint8_t reg);
+uint16_t pci_config_readw(pcidevaddr_t dev, uint8_t reg);
+uint32_t pci_config_readl(pcidevaddr_t dev, uint8_t reg);
+void pci_config_writel(pcidevaddr_t dev, uint8_t reg, uint32_t val);
+
+#endif

lib/pci-host-generic.c new file +172 -0

new file mode 100644
index 000000000000..96f4b4b37d34
@@ -0,0 +1,295 @@
+/*
+ * Generic PCI host controller as described in PCI Bus Binding to Open Firmware
+ *
+ * Copyright (C) 2016, Red Hat Inc, Alexander Gordeev <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.
+ */
+#include "libcflat.h"
+#include "devicetree.h"
+#include "alloc.h"
+#include "pci.h"
+#include "asm/pci.h"
+#include "asm/io.h"
+#include "pci-host-generic.h"
+#include <linux/pci_regs.h>
+
+static struct pci_host_bridge *pci_host_bridge;
+
+static int of_flags_to_pci_type(u32 of_flags)
+{
+	static int type_map[] = {
+		[1] = PCI_BASE_ADDRESS_SPACE_IO,
+		[2] = PCI_BASE_ADDRESS_MEM_TYPE_32,
+		[3] = PCI_BASE_ADDRESS_MEM_TYPE_64
+	};
+	int idx = (of_flags >> 24) & 0x03;
+	int res;
+
+	assert(idx > 0);
+	res = type_map[idx];
+
+	if (of_flags & 0x40000000)
+		res |= PCI_BASE_ADDRESS_MEM_PREFETCH;
+
+	return res;
+}
+
+static int pci_bar_type(u32 bar)
+{
+	if (bar & PCI_BASE_ADDRESS_SPACE)
+		return PCI_BASE_ADDRESS_SPACE_IO;
+	else
+		return bar & (PCI_BASE_ADDRESS_MEM_TYPE_MASK |
+			      PCI_BASE_ADDRESS_MEM_PREFETCH);
+}
+
+/*
+ * Probe DT for a generic PCI host controller
+ * See kernel Documentation/devicetree/bindings/pci/host-generic-pci.txt
+ * and function gen_pci_probe() in drivers/pci/host/pci-host-generic.c
+ */
+static struct pci_host_bridge *pci_dt_probe(void)
+{
+	struct pci_host_bridge *host;
+	const void *fdt = dt_fdt();
+	const struct fdt_property *prop;
+	struct dt_pbus_reg base;
+	struct dt_device dt_dev;
+	struct dt_bus dt_bus;
+	struct pci_addr_space *as;
+	fdt32_t *data;
+	u32 bus, bus_max;
+	u32 nac, nsc, nac_root, nsc_root;
+	int nr_range_cells, nr_addr_spaces;
+	int ret, node, len, i;
+
+	if (!dt_available()) {
+		printf("No device tree found\n");
+		return NULL;
+	}
+
+	dt_bus_init_defaults(&dt_bus);
+	dt_device_init(&dt_dev, &dt_bus, NULL);
+
+	node = fdt_path_offset(fdt, "/");
+	assert(node >= 0);
+
+	ret = dt_get_nr_cells(node, &nac_root, &nsc_root);
+	assert(ret == 0);
+	assert(nac_root == 1 || nac_root == 2);
+
+	node = fdt_node_offset_by_compatible(fdt, node,
+					     "pci-host-ecam-generic");
+	if (node == -FDT_ERR_NOTFOUND) {
+		printf("No PCIe ECAM compatible controller found\n");
+		return NULL;
+	}
+	assert(node >= 0);
+
+	prop = fdt_get_property(fdt, node, "device_type", &len);
+	assert(prop && len == 4 && !strcmp((char *)prop->data, "pci"));
+
+	dt_device_bind_node(&dt_dev, node);
+	ret = dt_pbus_get_base(&dt_dev, &base);
+	assert(ret == 0);
+
+	prop = fdt_get_property(fdt, node, "bus-range", &len);
+	if (prop == NULL) {
+		assert(len == -FDT_ERR_NOTFOUND);
+		bus		= 0x00;
+		bus_max		= 0xff;
+	} else {
+		data		= (fdt32_t *)prop->data;
+		bus		= fdt32_to_cpu(data[0]);
+		bus_max		= fdt32_to_cpu(data[1]);
+		assert(bus <= bus_max);
+	}
+	assert(bus_max < base.size / (1 << PCI_ECAM_BUS_SHIFT));
+
+	ret = dt_get_nr_cells(node, &nac, &nsc);
+	assert(ret == 0);
+	assert(nac == 3 && nsc == 2);
+
+	prop = fdt_get_property(fdt, node, "ranges", &len);
+	assert(prop != NULL);
+
+	nr_range_cells = nac + nsc + nac_root;
+	nr_addr_spaces = (len / 4) / nr_range_cells;
+	assert(nr_addr_spaces);
+
+	host = malloc(sizeof(*host) +
+		      sizeof(host->addr_space[0]) * nr_addr_spaces);
+	assert(host != NULL);
+
+	host->start		= base.addr;
+	host->size		= base.size;
+	host->bus		= bus;
+	host->bus_max		= bus_max;
+	host->nr_addr_spaces	= nr_addr_spaces;
+
+	data = (fdt32_t *)prop->data;
+	as = &host->addr_space[0];
+
+	for (i = 0; i < nr_addr_spaces; i++) {
+		/*
+		 * The PCI binding encodes the PCI address with three
+		 * cells as follows:
+		 *
+		 * phys.hi  cell: npt000ss bbbbbbbb dddddfff rrrrrrrr
+		 * phys.mid cell: hhhhhhhh hhhhhhhh hhhhhhhh hhhhhhhh
+		 * phys.lo  cell: llllllll llllllll llllllll llllllll
+		 *
+		 * PCI device bus address and flags are encoded into phys.high
+		 * PCI 64 bit address is encoded into phys.mid and phys.low
+		 */
+		as->type = of_flags_to_pci_type(fdt32_to_cpu(data[0]));
+		as->pci_start = ((u64)fdt32_to_cpu(data[1]) << 32) |
+				fdt32_to_cpu(data[2]);
+
+		if (nr_range_cells == 6) {
+			as->start = fdt32_to_cpu(data[3]);
+			as->size  = ((u64)fdt32_to_cpu(data[4]) << 32) |
+				    fdt32_to_cpu(data[5]);
+		} else {
+			as->start = ((u64)fdt32_to_cpu(data[3]) << 32) |
+				    fdt32_to_cpu(data[4]);
+			as->size  = ((u64)fdt32_to_cpu(data[5]) << 32) |
+				    fdt32_to_cpu(data[6]);
+		}
+
+		data += nr_range_cells;
+		as++;
+	}
+
+	return host;
+}
+
+static bool pci_alloc_resource(u64 *addr, u32 bar, u64 size)
+{
+	struct pci_host_bridge *host = pci_host_bridge;
+	struct pci_addr_space *as = &host->addr_space[0];
+	u32 mask;

lib/pci-host-generic.c new file +48 -0

+	}
+
+	mask = pci_bar_mask(bar);
+	size = ALIGN(size, ~mask + 1);
+	assert(as->allocated + size <= as->size);
+
+	*addr = as->pci_start + as->allocated;
+	as->allocated += size;
+
+	return true;
+}
+
+bool pci_probe(void)
+{
+	pcidevaddr_t dev;
+	u8 header;
+	u32 cmd;
+	int i;
+
+	assert(!pci_host_bridge);
+	pci_host_bridge = pci_dt_probe();
+	if (!pci_host_bridge)
+		return false;
+
+	for (dev = 0; dev < 256; dev++) {
+		if (!pci_dev_exists(dev))
+			continue;
+
+		/* We are only interested in normal PCI devices */
+		header = pci_config_readb(dev, PCI_HEADER_TYPE);
+		if ((header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL)
+			continue;
+
+		cmd = PCI_COMMAND_SERR;
+
+		for (i = 0; i < 6; i++) {
+			u64 addr, size;
+			u32 bar;
+
+			size = pci_bar_size(dev, i);
+			if (!size)
+				continue;
+
+			bar = pci_bar_get(dev, i);
+			if (pci_alloc_resource(&addr, bar, size)) {
+				pci_bar_set_addr(dev, i, addr);
+
+				if (pci_bar_is_memory(dev, i))

lib/pci-host-generic.c new file +55 -0

+	}
+
+	return true;
+}
+
+/*
+ * This function is to be called from pci_translate_addr() to provide
+ * mapping between this host bridge's PCI busses address and CPU physical
+ * address.
+ */
+phys_addr_t pci_host_bridge_get_paddr(u64 pci_addr)
+{
+	struct pci_host_bridge *host = pci_host_bridge;
+	struct pci_addr_space *as = &host->addr_space[0];
+	int i;
+
+	for (i = 0; i < host->nr_addr_spaces; i++) {
+		if (pci_addr >= as->pci_start &&
+		    pci_addr < as->pci_start + as->size)
+			return as->start + (pci_addr - as->pci_start);
+		as++;
+	}
+
+	return 0;
+}
+
+static void __iomem *pci_get_dev_conf(struct pci_host_bridge *host, int devfn)
+{
+	return (void __iomem *)(unsigned long)
+		host->start + (devfn << PCI_ECAM_DEVFN_SHIFT);
+}
+
+u8 pci_config_readb(pcidevaddr_t dev, u8 off)
+{
+	void __iomem *conf = pci_get_dev_conf(pci_host_bridge, dev);
+	return readb(conf + off);
+}
+
+u16 pci_config_readw(pcidevaddr_t dev, u8 off)
+{
+	void __iomem *conf = pci_get_dev_conf(pci_host_bridge, dev);
+	return readw(conf + off);
+}
+
+u32 pci_config_readl(pcidevaddr_t dev, u8 off)
+{
+	void __iomem *conf = pci_get_dev_conf(pci_host_bridge, dev);
+	return readl(conf + off);
+}
+
+void pci_config_writel(pcidevaddr_t dev, u8 off, u32 val)
+{
+	void __iomem *conf = pci_get_dev_conf(pci_host_bridge, dev);
+	writel(val, conf + off);
+}

lib/pci-host-generic.h new file +46 -0

new file mode 100644
index 000000000000..fd30e7c74ed8
@@ -0,0 +1,46 @@
+#ifndef PCI_HOST_GENERIC_H
+#define PCI_HOST_GENERIC_H
+/*
+ * PCI host bridge supporting structures and constants
+ *
+ * Copyright (C) 2016, Red Hat Inc, Alexander Gordeev <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.
+ */
+#include "libcflat.h"
+
+struct pci_addr_space {
+	phys_addr_t		pci_start;
+	phys_addr_t		start;
+	phys_addr_t		size;
+	phys_addr_t		allocated;
+	int			type;
+};
+
+struct pci_host_bridge {
+	phys_addr_t		start;
+	phys_addr_t		size;
+	int			bus;
+	int			bus_max;
+	int			nr_addr_spaces;
+	struct pci_addr_space	addr_space[];
+};
+
+/*
+ * The following constants are derived from Linux, see this source:
+ *
+ *         drivers/pci/host/pci-host-generic.c
+ *                 struct gen_pci_cfg_bus_ops::bus_shift
+ *                 int gen_pci_parse_map_cfg_windows(struct gen_pci *pci)
+ *
+ * Documentation/devicetree/bindings/pci/host-generic-pci.txt describes
+ * ECAM Configuration Space is be memory-mapped by concatenating the various
+ * components to form an offset:
+ *
+ *	cfg_offset(bus, device, function, register) =
+ *		   bus << 20 | device << 15 | function << 12 | register
+ */
+#define PCI_ECAM_BUS_SHIFT	20
+#define PCI_ECAM_DEVFN_SHIFT	12
+
+#endif

lib/pci.c +2 -2

index 462cddc8a806..2dbbba428144 100644
@@ -27,13 +27,13 @@ pcidevaddr_t pci_find_dev(uint16_t vendor_id, uint16_t device_id)
 	return PCIDEVADDR_INVALID;
 }
 
-static uint32_t pci_bar_mask(uint32_t bar)
+uint32_t pci_bar_mask(uint32_t bar)
 {
 	return (bar & PCI_BASE_ADDRESS_SPACE_IO) ?
 		PCI_BASE_ADDRESS_IO_MASK : PCI_BASE_ADDRESS_MEM_MASK;
 }
 
-static uint32_t pci_bar_get(pcidevaddr_t dev, int bar_num)
+uint32_t pci_bar_get(pcidevaddr_t dev, int bar_num)
 {
 	return pci_config_readl(dev, PCI_BASE_ADDRESS_0 + bar_num * 4);
 }

lib/pci.h +3 -0

index fc0940adc299..7acbbdf2eb16 100644
@@ -15,6 +15,7 @@ enum {
 	PCIDEVADDR_INVALID = 0xffff,
 };
 
+extern bool pci_probe(void);
 extern void pci_print(void);
 extern bool pci_dev_exists(pcidevaddr_t dev);
 extern pcidevaddr_t pci_find_dev(uint16_t vendor_id, uint16_t device_id);

@@ -34,6 +35,8 @@ extern pcidevaddr_t pci_find_dev(uint16_t vendor_id, uint16_t device_id);
 extern phys_addr_t pci_bar_get_addr(pcidevaddr_t dev, int bar_num);
 extern void pci_bar_set_addr(pcidevaddr_t dev, int bar_num, phys_addr_t addr);
 extern phys_addr_t pci_bar_size(pcidevaddr_t dev, int bar_num);
+extern uint32_t pci_bar_get(pcidevaddr_t dev, int bar_num);
+extern uint32_t pci_bar_mask(uint32_t bar);
 extern bool pci_bar_is64(pcidevaddr_t dev, int bar_num);
 extern bool pci_bar_is_memory(pcidevaddr_t dev, int bar_num);
 extern bool pci_bar_is_valid(pcidevaddr_t dev, int bar_num);

Signature · 1 line

1.8.3.1

Quoted 5 lines

lib/pci-host-generic.c new file +5 -0

+static bool pci_alloc_resource(u64 *addr, u32 bar, u64 size)
+{
+	struct pci_host_bridge *host = pci_host_bridge;
+	struct pci_addr_space *as = &host->addr_space[0];
+	u32 mask;

lib/pci-host-generic.c line 182

Quoted 10 lines

lib/pci-host-generic.c new file +10 -0

+	int i;
+
+	for (i = 0; i < host->nr_addr_spaces; i++) {
+		if (as->type == pci_bar_type(bar))
+			break;
+		as++;
+	}
+	if (i >= host->nr_addr_spaces) {
+		printf("No PCI resource matching a device found\n");
+		return false;

So this is the new code with v8, and it needs more work. First, this print message doesn't explain the problem. It should point out which device and which bar failed to have resources allocated and why.

For the which, I think it should print the device number and function number in the lspci way %02x.%02x.%1x (bus.dev.fn) followed by the vendor-id and device-id, %04x:%04x. For the bar you'll want to refactor pci_dev_print to pull a pci_bar_print out of it that you can use here.

But... what about the why? Is it really necessary to fail this allocation? How does Linux handle it with the same guest config?

So I went a head and improved the print message in order to try to understand, and I think I do. The virtio-net-pci device that gets added by default wanted a MEM64/p region. We fail because of the '/p'. I don't know PCI, but shouldn't we be masking off that PCI_BASE_ADDRESS_MEM_PREFETCH bit in this check?

Finally, I'd set *addr to some invalid address, zero? when returning false here.

lib/pci-host-generic.c line 194

Quoted 12 lines

lib/pci-host-generic.c new file +12 -0

+	}
+
+	mask = pci_bar_mask(bar);
+	size = ALIGN(size, ~mask + 1);
+	assert(as->allocated + size <= as->size);
+
+	*addr = as->pci_start + as->allocated;
+	as->allocated += size;
+
+	return true;
+}
+

Thanks,
drew

Quoted 21 lines · Andrew Jones

Quoted 5 lines

lib/pci-host-generic.c new file +5 -0

+static bool pci_alloc_resource(u64 *addr, u32 bar, u64 size)
+{
+	struct pci_host_bridge *host = pci_host_bridge;
+	struct pci_addr_space *as = &host->addr_space[0];
+	u32 mask;

lib/pci-host-generic.c line 182

Quoted 10 lines

lib/pci-host-generic.c new file +10 -0

+	int i;
+
+	for (i = 0; i < host->nr_addr_spaces; i++) {
+		if (as->type == pci_bar_type(bar))
+			break;
+		as++;
+	}
+	if (i >= host->nr_addr_spaces) {
+		printf("No PCI resource matching a device found\n");
+		return false;

So this is the new code with v8, and it needs more work. First, this print message doesn't explain the problem. It should point out which device and which bar failed to have resources allocated and why.

Quoted 8 lines

For the which, I think it should print the device number and function number in the lspci way %02x.%02x.%1x (bus.dev.fn) followed by the vendor-id and device-id, %04x:%04x. For the bar you'll want to refactor pci_dev_print to pull a pci_bar_print out of it that you can use here.

But... what about the why? Is it really necessary to fail this allocation? How does Linux handle it with the same guest config?

Just to check we are on the same page - by "fail" you mean here the failure to assign a resource, not bailing out due to a mis- configuration, right?

Quoted 5 lines

So I went a head and improved the print message in order to try to understand, and I think I do. The virtio-net-pci device that gets added by default wanted a MEM64/p region. We fail because of the '/p'. I don't know PCI, but shouldn't we be masking off that PCI_BASE_ADDRESS_MEM_PREFETCH bit in this check?

I guess, if we could mask /p we would not have it in th first place. I recall some ugly side effects in case prefetch memory mismatch, but can not tell for sure.

@Thomas, may be you know the answer here?

Quoted 41 lines · Alexander Gordeev

Quoted 21 lines · Andrew Jones

Quoted 5 lines

lib/pci-host-generic.c new file +5 -0

+static bool pci_alloc_resource(u64 *addr, u32 bar, u64 size)
+{
+	struct pci_host_bridge *host = pci_host_bridge;
+	struct pci_addr_space *as = &host->addr_space[0];
+	u32 mask;

lib/pci-host-generic.c line 182

Quoted 10 lines

lib/pci-host-generic.c new file +10 -0

+	int i;
+
+	for (i = 0; i < host->nr_addr_spaces; i++) {
+		if (as->type == pci_bar_type(bar))
+			break;
+		as++;
+	}
+	if (i >= host->nr_addr_spaces) {
+		printf("No PCI resource matching a device found\n");
+		return false;

So this is the new code with v8, and it needs more work. First, this print message doesn't explain the problem. It should point out which device and which bar failed to have resources allocated and why.

Quoted 8 lines

For the which, I think it should print the device number and function number in the lspci way %02x.%02x.%1x (bus.dev.fn) followed by the vendor-id and device-id, %04x:%04x. For the bar you'll want to refactor pci_dev_print to pull a pci_bar_print out of it that you can use here.

But... what about the why? Is it really necessary to fail this allocation? How does Linux handle it with the same guest config?

Just to check we are on the same page - by "fail" you mean here the failure to assign a resource, not bailing out due to a mis- configuration, right?

Quoted 5 lines

So I went a head and improved the print message in order to try to understand, and I think I do. The virtio-net-pci device that gets added by default wanted a MEM64/p region. We fail because of the '/p'. I don't know PCI, but shouldn't we be masking off that PCI_BASE_ADDRESS_MEM_PREFETCH bit in this check?

Quoted 5 lines

I guess, if we could mask /p we would not have it in th first place. I recall some ugly side effects in case prefetch memory mismatch, but can not tell for sure.

@Thomas, may be you know the answer here?

Well, you can not map a device with a non-prefetchable BAR into an MMIO region that is treated as prefetchable from the CPU. According to the PCI LB specification:

"Prefetchable memory has the following characteristics:
* There are no side effects of a read operation. The read operation cannot be destructive to either the data or any other state information. For example, a FIFO that advances to the next data when read would not be prefetchable. Similarly, a location that cleared a status bit when read would not be prefetchable.
* When read, the device is required to return all bytes regardless of the byte enables (four or eight depending upon the width of the data transfer (refer to Section 3.8.1)).
* Bridges are permitted to merge writes into this range (refer to

    Section 3.2.6)."

But you can do it the other way round, you can map a BAR that is marked as prefetchable into an MMIO region that is treated as non-prefetchable (e.g. non-cachable) by the CPU.

Hope that helps,
Thomas

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Quoted 27 lines · Andrew Jones

Quoted 5 lines

lib/pci-host-generic.c new file +5 -0

+static bool pci_alloc_resource(u64 *addr, u32 bar, u64 size)
+{
+	struct pci_host_bridge *host = pci_host_bridge;
+	struct pci_addr_space *as = &host->addr_space[0];
+	u32 mask;

lib/pci-host-generic.c line 182

Quoted 10 lines

lib/pci-host-generic.c new file +10 -0

+	int i;
+
+	for (i = 0; i < host->nr_addr_spaces; i++) {
+		if (as->type == pci_bar_type(bar))
+			break;
+		as++;
+	}
+	if (i >= host->nr_addr_spaces) {
+		printf("No PCI resource matching a device found\n");
+		return false;

So this is the new code with v8, and it needs more work. First, this print message doesn't explain the problem. It should point out which device and which bar failed to have resources allocated and why.

For the which, I think it should print the device number and function number in the lspci way %02x.%02x.%1x (bus.dev.fn) followed by the vendor-id and device-id, %04x:%04x. For the bar you'll want to refactor pci_dev_print to pull a pci_bar_print out of it that you can use here.

Quoted 2 lines

But... what about the why? Is it really necessary to fail this allocation? How does Linux handle it with the same guest config?

Just to check we are on the same page - by "fail" you mean here the failure to assign a resource, not bailing out due to a mis- configuration, right?

Quoted 6 lines

So I went a head and improved the print message in order to try to understand, and I think I do. The virtio-net-pci device that gets added by default wanted a MEM64/p region. We fail because of the '/p'. I don't know PCI, but shouldn't we be masking off that PCI_BASE_ADDRESS_MEM_PREFETCH bit in this check?

I guess, if we could mask /p we would not have it in th first place. I recall some ugly side effects in case prefetch memory mismatch, but can not tell for sure.

@Thomas, may be you know the answer here?

Quoted 2 lines

Finally, I'd set *addr to some invalid address, zero? when returning false here.

Er.. should not we preserve input data in case a function failed? Anyway, zero may be a legitimate address in PIO space and I can not think of any better choice.

Quoted 15 lines

lib/pci-host-generic.c line 194

Quoted 13 lines

lib/pci-host-generic.c new file +12 -0

+	}
+
+	mask = pci_bar_mask(bar);
+	size = ALIGN(size, ~mask + 1);
+	assert(as->allocated + size <= as->size);
+
+	*addr = as->pci_start + as->allocated;
+	as->allocated += size;
+
+	return true;
+}
+

Thanks,
drew

--
To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Suggested-by Andrew Jones <[email protected]>

Cc Thomas Huth <[email protected]>

Cc Andrew Jones <[email protected]>

Cc Peter Xu <[email protected]>

Reviewed-by Andrew Jones <[email protected]>

Signed-off-by Alexander Gordeev <[email protected]>

---
 arm/Makefile.common |  6 +++++-
 arm/pci-test.c      | 27 +++++++++++++++++++++++++++
 arm/run             |  7 ++++++-
 arm/unittests.cfg   |  4 ++++
 lib/arm/asm/pci.h   |  1 +
 lib/arm64/asm/pci.h |  1 +
 6 files changed, 44 insertions(+), 2 deletions(-)
 create mode 100644 arm/pci-test.c
 create mode 100644 lib/arm/asm/pci.h
 create mode 100644 lib/arm64/asm/pci.h

arm/Makefile.common +5 -1

index ccb554d9251a..f37b5c2a3de4 100644
@@ -11,7 +11,8 @@ endif
 
 tests-common = \
 	$(TEST_DIR)/selftest.flat \
-	$(TEST_DIR)/spinlock-test.flat
+	$(TEST_DIR)/spinlock-test.flat \
+	$(TEST_DIR)/pci-test.flat
 
 all: test_cases
 

@@ -33,6 +34,9 @@ include scripts/asm-offsets.mak
 cflatobjs += lib/util.o
 cflatobjs += lib/alloc.o
 cflatobjs += lib/devicetree.o
+cflatobjs += lib/pci.o
+cflatobjs += lib/pci-host-generic.o
+cflatobjs += lib/pci-testdev.o
 cflatobjs += lib/virtio.o
 cflatobjs += lib/virtio-mmio.o
 cflatobjs += lib/chr-testdev.o

arm/pci-test.c new file +27 -0

new file mode 100644
index 000000000000..10a367de5357
@@ -0,0 +1,27 @@
+/*
+ * PCI bus operation test
+ *
+ * Copyright (C) 2016, Red Hat Inc, Alexander Gordeev <[email protected]>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.
+ */
+#include <libcflat.h>
+#include <pci.h>
+
+#define NR_TESTS (PCI_TESTDEV_NUM_BARS * PCI_TESTDEV_NUM_TESTS)
+
+int main(void)
+{
+	int ret;
+
+	if (!pci_probe())
+		report_abort("PCI bus probing failed\n");
+
+	pci_print();
+
+	ret = pci_testdev();
+	report("PCI test device passed %d/%d tests",
+		ret >= NR_TESTS, ret > 0 ? ret : 0, NR_TESTS);
+
+	return report_summary();
+}

arm/run +6 -1

index a2f35ef6a7e6..1ee6231599d6 100755
@@ -67,8 +67,13 @@ fi
 chr_testdev='-device virtio-serial-device'
 chr_testdev+=' -device virtconsole,chardev=ctd -chardev testdev,id=ctd'
 
+pci_testdev=
+if $qemu $M -device '?' 2>&1 | grep pci-testdev > /dev/null; then
+	pci_testdev="-device pci-testdev"
+fi
+
 M+=",accel=$ACCEL"
-command="$qemu $M -cpu $processor $chr_testdev"
+command="$qemu $M -cpu $processor $chr_testdev $pci_testdev"
 command+=" -display none -serial stdio -kernel"
 command="$(timeout_cmd) $command"
 echo $command "$@"

arm/unittests.cfg +4 -0

index 3f6fa45c587e..ae32a42a91c3 100644
@@ -54,3 +54,7 @@ file = selftest.flat
 smp = $MAX_SMP
 extra_params = -append 'smp'
 groups = selftest
+
+[pci-test]
+file = pci-test.flat
+groups = pci

lib/arm/asm/pci.h new file +0 -0

new file mode 100644
index 000000000000..01ecafe2b6d8

lib/arm64/asm/pci.h new file +1 -0

new file mode 100644
index 000000000000..f70ef560e2ab
@@ -0,0 +1 @@
+#include "../../arm/asm/pci.h"

Signature · 6 lines

1.8.3.1

--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to [email protected]
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Hi Andrew et al,

This is 8th version of PCI support. The only noticeable change since v7 is no failure in case no requested memory type is found for a BAR.

Sources are avalable at:
https://github.com/a-gordeev/kvm-unit-tests.git pci-testdev-v7

Quoted 7 lines · Alexander Gordeev

Hi Andrew et al,

This is 8th version of PCI support. The only noticeable change since v7 is no failure in case no requested memory type is found for a BAR.

Sources are avalable at:
https://github.com/a-gordeev/kvm-unit-tests.git pci-testdev-v7

                                                              ^^ v8

I tested this with/without the attached patch and works for me.

Thanks,
drew

lib/pci-host-generic.c +0 -0

index 96f4b4b37d34e..04e9681190308 100644
@@ -165,20 +165,35 @@ static struct pci_host_bridge *pci_dt_probe(void)
 	return host;
 }
 

lib/pci-host-generic.c +10 -1

+
+	size = pci_bar_size(dev, bar_num);
+	if (!size)
+		return false;
+
+	bar = pci_bar_get(dev, bar_num);
+	type = pci_bar_type(bar);
+	if (type & PCI_BASE_ADDRESS_MEM_TYPE_MASK)
+		type &= ~PCI_BASE_ADDRESS_MEM_PREFETCH;
 
 	for (i = 0; i < host->nr_addr_spaces; i++) {
-		if (as->type == pci_bar_type(bar))
+		if (as->type == type)
 			break;

lib/pci-host-generic.c +2 -8

 		return false;
 	}
 

@@ -216,15 +231,9 @@ bool pci_probe(void)
 		cmd = PCI_COMMAND_SERR;
 
 		for (i = 0; i < 6; i++) {
-			u64 addr, size;
-			u32 bar;
+			u64 addr;
 
-			size = pci_bar_size(dev, i);
-			if (!size)
-				continue;
-
-			bar = pci_bar_get(dev, i);
-			if (pci_alloc_resource(&addr, bar, size)) {
+			if (pci_alloc_resource(dev, i, &addr)) {
 				pci_bar_set_addr(dev, i, addr);
 
 				if (pci_bar_is_memory(dev, i))

lib/pci.c +56 -45

index 2dbbba4281442..6bd54cbac1bbd 100644
@@ -127,53 +127,31 @@ bool pci_bar_is64(pcidevaddr_t dev, int bar_num)
 		      PCI_BASE_ADDRESS_MEM_TYPE_64;
 }
 
-static void pci_dev_print(pcidevaddr_t dev)
+void pci_bar_print(pcidevaddr_t dev, int bar_num)
 {
-	uint16_t vendor_id = pci_config_readw(dev, PCI_VENDOR_ID);
-	uint16_t device_id = pci_config_readw(dev, PCI_DEVICE_ID);
-	uint8_t header = pci_config_readb(dev, PCI_HEADER_TYPE);
-	uint8_t progif = pci_config_readb(dev, PCI_CLASS_PROG);
-	uint8_t subclass = pci_config_readb(dev, PCI_CLASS_DEVICE);
-	uint8_t class = pci_config_readb(dev, PCI_CLASS_DEVICE + 1);
-	int i;
+	phys_addr_t size, start, end;
+	uint32_t bar;
 
-	printf("dev %2d fn %d vendor_id %04x device_id %04x type %02x "
-	       "progif %02x class %02x subclass %02x\n",
-	       dev / 8, dev % 8, vendor_id, device_id, header,
-	       progif, class, subclass);
-
-	if ((header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL)
+	size = pci_bar_size(dev, bar_num);
+	if (!size)
 		return;
 
-	for (i = 0; i < 6; i++) {
-		phys_addr_t size, start, end;
-		uint32_t bar;
-
-		size = pci_bar_size(dev, i);
-		if (!size)
-			continue;
-
-		start = pci_bar_get_addr(dev, i);
-		end = start + size - 1;
-
-		if (pci_bar_is64(dev, i)) {
-			printf("\tBAR#%d,%d [%" PRIx64 "-%" PRIx64 " ",
-			       i, i + 1, start, end);
-			i++;
-		} else {
-			printf("\tBAR#%d    [%02x-%02x ",
-			       i, (uint32_t)start, (uint32_t)end);
-		}
-
-		bar = pci_bar_get(dev, i);
-
-		if (bar & PCI_BASE_ADDRESS_SPACE_IO) {
-			printf("PIO]\n");
-			continue;
-		}
-
+	bar = pci_bar_get(dev, bar_num);
+	start = pci_bar_get_addr(dev, bar_num);
+	end = start + size - 1;
+
+	if (pci_bar_is64(dev, bar_num)) {
+		printf("BAR#%d,%d [%" PRIx64 "-%" PRIx64 " ",
+		       bar_num, bar_num + 1, start, end);
+	} else {
+		printf("BAR#%d [%02x-%02x ",
+		       bar_num, (uint32_t)start, (uint32_t)end);
+	}
+
+	if (bar & PCI_BASE_ADDRESS_SPACE_IO) {
+		printf("PIO");
+	} else {
 		printf("MEM");
-
 		switch (bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK) {
 		case PCI_BASE_ADDRESS_MEM_TYPE_32:
 			printf("32");

@@ -187,11 +165,44 @@ static void pci_dev_print(pcidevaddr_t dev)
 		default:
 			assert(0);
 		}
+	}
 
-		if (bar & PCI_BASE_ADDRESS_MEM_PREFETCH)
-			printf("/p");
+	if (bar & PCI_BASE_ADDRESS_MEM_PREFETCH)
+		printf("/p");
 
-		printf("]\n");
+	printf("]");
+}
+
+void pci_dev_print_id(pcidevaddr_t dev)
+{
+	printf("00.%02x.%1x %04x:%04x", dev / 8, dev % 8,
+		pci_config_readw(dev, PCI_VENDOR_ID),
+		pci_config_readw(dev, PCI_DEVICE_ID));
+}
+
+static void pci_dev_print(pcidevaddr_t dev)
+{
+	uint8_t header = pci_config_readb(dev, PCI_HEADER_TYPE);
+	uint8_t progif = pci_config_readb(dev, PCI_CLASS_PROG);
+	uint8_t subclass = pci_config_readb(dev, PCI_CLASS_DEVICE);
+	uint8_t class = pci_config_readb(dev, PCI_CLASS_DEVICE + 1);
+	int i;
+
+	pci_dev_print_id(dev);
+	printf(" type %02x progif %02x class %02x subclass %02x\n",
+	       header, progif, class, subclass);
+
+	if ((header & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_NORMAL)
+		return;
+
+	for (i = 0; i < 6; i++) {
+		if (pci_bar_size(dev, i)) {
+			printf("\t");
+			pci_bar_print(dev, i);
+			printf("\n");
+		}
+		if (pci_bar_is64(dev, i))
+			i++;
 	}
 }
 

lib/pci.h +2 -0

index 40e11a892783c..30f5381106108 100644
@@ -40,6 +40,8 @@ extern uint32_t pci_bar_mask(uint32_t bar);
 extern bool pci_bar_is64(pcidevaddr_t dev, int bar_num);
 extern bool pci_bar_is_memory(pcidevaddr_t dev, int bar_num);
 extern bool pci_bar_is_valid(pcidevaddr_t dev, int bar_num);
+extern void pci_bar_print(pcidevaddr_t dev, int bar_num);
+extern void pci_dev_print_id(pcidevaddr_t dev);
 
 int pci_testdev(void);