Skip to contentSkip to conversation

(Longhaul 1/5) PCI: Protect bus master DMA from Longhaul by rw semaphores

PATCH In review Rafał Bilski 5 participants 5 messages Jun 28, 2006 PCI

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 (Longhaul 1/5) PCI: Protect bus master DMA from Longhaul by rw semaphores

This patch will allow Longhaul cpufreq driver to change frequency
without breaking BMDMA. In order to work properly it needs:
- adding rw_semaphore to pci_device and bus structures - this is patch below,
- Longhaul should find host bridge and lock write on bus before frequency change,
- device driver support - device should lock read its bus before starting DMA transfer. I have curently implemented this for ide layer (tested with via82cxxx), libata (not tested, but this is similar code to ide) and VIA Rhine network card driver.
I don't know if this is acceptable infrastructure, but I hope it is
less horrible then last. Is this infrastructure at all?

This patch is adding 2 functions to pci structures: "acquire" and
"release". Driver calls "acquire" before starting transfer, with PCI_ACQUIRE_EXCLUSIVE if it needs exclusive access to device (bus is always PCI_ACQUIRE_SHARED) or with PCI_ACQUIRE_SHARED if device have own hardware queue. Ide and libata are using PCI_ACQUIRE_EXCLUSIVE. VIA Rhine driver is using PCI_ACQUIRE_SHARED.
Note: "acquire" may sleep. But in Linux 2.6 we can use
workqueues and I'm using them.

---

drivers/pci/probe.c +68 -0
diff -uprN -X linux-2.6.17-vanilla/Documentation/dontdiff linux-2.6.17-vanilla/drivers/pci/probe.c linux-2.6.17/drivers/pci/probe.c
@@ -9,6 +9,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/cpumask.h>
+#include <linux/rwsem.h>
 #include "pci.h"
 
 #define CARDBUS_LATENCY_TIMER	176	/* secondary latency timer */
@@ -315,6 +316,34 @@ void __devinit pci_read_bridge_bases(str
 	}
 }
 
+/**
+ * Generic bus locking
+ */
+
+#ifdef CONFIG_X86_LONGHAUL
+static void pci_bus_acquire(struct pci_bus *bus, int flags)
+{
+	if (bus->parent && bus->parent->acquire)
+		bus->parent->acquire(bus->parent, PCI_ACQUIRE_SHARED);
+	if (flags & PCI_ACQUIRE_EXCLUSIVE) {
+		down_write(&bus->access_sem);
+	} else {
+		down_read(&bus->access_sem);
+	}
+}
+
+static void pci_bus_release(struct pci_bus *bus, int flags)
+{
+	if (flags & PCI_ACQUIRE_EXCLUSIVE) {
+		up_write(&bus->access_sem);
+	} else {
+		up_read(&bus->access_sem);
+	}
+	if (bus->parent && bus->parent->release)
+		bus->parent->release(bus->parent, PCI_ACQUIRE_SHARED);
+}
+#endif
+
 static struct pci_bus * __devinit pci_alloc_bus(void)
 {
 	struct pci_bus *b;
@@ -324,6 +353,12 @@ static struct pci_bus * __devinit pci_al
 		INIT_LIST_HEAD(&b->node);
 		INIT_LIST_HEAD(&b->children);
 		INIT_LIST_HEAD(&b->devices);
+
+#ifdef CONFIG_X86_LONGHAUL
+		init_rwsem(&b->access_sem);
+		b->acquire = &pci_bus_acquire;
+	    	b->release = &pci_bus_release;
+#endif
 	}
 	return b;
 }
@@ -622,6 +657,34 @@ static void pci_read_irq(struct pci_dev 
 }
 
 /**
+ * Generic device locking
+ */
+
+#ifdef CONFIG_X86_LONGHAUL
+static void pci_generic_acquire(struct pci_dev *dev, int flags)
+{
+	if (dev->bus && dev->bus->acquire)
+		dev->bus->acquire(dev->bus, PCI_ACQUIRE_SHARED);
+	if (flags & PCI_ACQUIRE_EXCLUSIVE) {
+		down_write(&dev->access_sem);
+	} else {
+		down_read(&dev->access_sem);
+	}
+}
+
+static void pci_generic_release(struct pci_dev *dev, int flags)
+{
+	if (flags & PCI_ACQUIRE_EXCLUSIVE) {
+		up_write(&dev->access_sem);
+	} else {
+		up_read(&dev->access_sem);
+	}
+	if (dev->bus && dev->bus->release)
+		dev->bus->release(dev->bus, PCI_ACQUIRE_SHARED);
+}
+#endif
+
+/**
  * pci_setup_device - fill in class and map information of a device
  * @dev: the device structure to fill
  *
@@ -638,6 +701,11 @@ static int pci_setup_device(struct pci_d
 	sprintf(pci_name(dev), "%04x:%02x:%02x.%d", pci_domain_nr(dev->bus),
 		dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn));
 
+#ifdef CONFIG_X86_LONGHAUL
+	init_rwsem(&dev->access_sem);
+	dev->acquire = &pci_generic_acquire;
+	dev->release = &pci_generic_release;
+#endif
 	pci_read_config_dword(dev, PCI_CLASS_REVISION, &class);
 	class >>= 8;				    /* upper 3 bytes */
 	dev->class = class;
include/linux/pci.h +13 -1
diff -uprN -X linux-2.6.17-vanilla/Documentation/dontdiff linux-2.6.17-vanilla/include/linux/pci.h linux-2.6.17/include/linux/pci.h
@@ -52,6 +52,7 @@
 #include <linux/list.h>
 #include <linux/errno.h>
 #include <linux/device.h>
+#include <linux/rwsem.h>
 
 /* File state for mmap()s on /proc/bus/pci/X/Y */
 enum pci_mmap_state {
@@ -169,6 +170,13 @@ struct pci_dev {
 	struct bin_attribute *rom_attr; /* attribute descriptor for sysfs ROM entry */
 	int rom_attr_enabled;		/* has display of the rom attribute been enabled? */
 	struct bin_attribute *res_attr[DEVICE_COUNT_RESOURCE]; /* sysfs file for resources */
+#ifdef CONFIG_X86_LONGHAUL
+	struct rw_semaphore	access_sem;	/* Access control to device */
+	void (*acquire) (struct pci_dev *dev, int flags);
+	void (*release) (struct pci_dev *dev, int flags);
+#define PCI_ACQUIRE_SHARED		0
+#define PCI_ACQUIRE_EXCLUSIVE		(1 << 0)
+#endif
 };
 
 #define pci_dev_g(n) list_entry(n, struct pci_dev, global_list)
@@ -226,7 +234,6 @@ struct pci_bus {
 	struct pci_dev	*self;		/* bridge device as seen by parent */
 	struct resource	*resource[PCI_BUS_NUM_RESOURCES];
 					/* address space routed to this bus */
-
 	struct pci_ops	*ops;		/* configuration access functions */
 	void		*sysdata;	/* hook for sys-specific extension */
 	struct proc_dir_entry *procdir;	/* directory entry in /proc/bus/pci */
@@ -244,6 +251,11 @@ struct pci_bus {
 	struct class_device	class_dev;
 	struct bin_attribute	*legacy_io; /* legacy I/O for this bus */
 	struct bin_attribute	*legacy_mem; /* legacy mem */
+#ifdef CONFIG_X86_LONGHAUL
+	struct rw_semaphore	access_sem;	/* Access control to bus */
+	void (*acquire) (struct pci_bus *bus, int flags);
+	void (*release) (struct pci_bus *bus, int flags);
+#endif
 };
 
 #define pci_bus_b(n)	list_entry(n, struct pci_bus, node)

----------------------------------------------------------------------
PS. Fajny portal... >>> http://link.interia.pl/f196a

#1 lore raw
Quoted 6 lines · Rafa? Bilski

This patch will allow Longhaul cpufreq driver to change frequency
without breaking BMDMA. In order to work properly it needs:
- adding rw_semaphore to pci_device and bus structures - this is patch below,
- Longhaul should find host bridge and lock write on bus before frequency change,

Eeek! You mean the longhaul driver can change the frequency of the PCI bus? Oh, that's a recipe for disaster...

Quoted 6 lines

- device driver support - device should lock read its bus before starting DMA transfer. I have curently implemented this for ide layer (tested with via82cxxx), libata (not tested, but this is similar code to ide) and VIA Rhine network card driver.
I don't know if this is acceptable infrastructure, but I hope it is
less horrible then last. Is this infrastructure at all?

No, it's a hack :)

No, this is not acceptable. What exactly do you want to do here? Make sure the PCI drivers are not doing DMA when the longhaul driver wants to change the pci bus speed?

How often will this bus change happen?

Does it really save battery?

Will all PCI devices work properly at different speeds from what they originally thought they were running at?

And what about PCI devices that always do DMA? (think USB controllers, they can easily saturate the PCI bus all the time).

Why not just suspend all PCI devices make the bus change, and then resume them? That would require no PCI core, or driver changes.

thanks,

greg k-h

#2 lore raw

Ar Mer, 2006-06-28 am 10:34 -0700, ysgrifennodd Greg KH:

Quoted 2 lines

Eeek! You mean the longhaul driver can change the frequency of the PCI bus? Oh, that's a recipe for disaster...

Not as I understand the docs, and that would be unfixable. Some C3 setups do however appear to "fall off the bus" during transitions which means if BMDMA is active things get confused.

I am still not clear if this is just cache corruption through us not listening or whether we genuinely need to halt. In the former case flushing and disabling the CPU caches ought to be sufficient.

Alan

#3 lore raw
Quoted 2 lines · Alan Cox

I am still not clear if this is just cache corruption through us not listening or whether we genuinely need to halt.

Especially buggy steppings of C3s would stop snooping while in 'hlt', so it wouldn't surprise me if they stopped while doing a speed transition.

Quoted 3 lines

In the former case
flushing and disabling the CPU caches ought to be sufficient.

Sounds reasonable, unless that causes problems of its own. (My general experience with Via CPUs is that doing anything even slightly unusual will result in strange behaviour or outright buggyness.)

J

#4 lore raw
In reply to Greg KH
Quoted 10 lines · Greg KH
Quoted 6 lines · Rafa? Bilski

This patch will allow Longhaul cpufreq driver to change frequency
without breaking BMDMA. In order to work properly it needs:
- adding rw_semaphore to pci_device and bus structures - this is patch below,
- Longhaul should find host bridge and lock write on bus before frequency change,

Eeek! You mean the longhaul driver can change the frequency of the PCI bus? Oh, that's a recipe for disaster...

It *can* scale the FSB on some CPUs, but the Linux driver doesn't do that as there a lot of systems that it just doesn't work on. (Probably any that have FSB+PCI speed tied together). So we just scale the multiplier and the voltage (Though currently in the driver, voltage scaling is missing, so we never save any power, and just run at the maximum voltage the whole time).

The situation is : It needs there to be no bus mastering occuring at the time of a CPU speed transition. Though I'm unable to find the part that mentions this in the specs I have right now.

The really bizarre thing is that all this was working once. Circa 2.5.late/2.6.reallyearly I had reliable scaling of both voltage and speed on a C3. Over time the driver just seemed to fall apart. Perhaps we just got lucky before, and some change occured that results changed behaviour of pci bus mastering for
some devices.

Quoted 7 lines

No, this is not acceptable. What exactly do you want to do here? Make sure the PCI drivers are not doing DMA when the longhaul driver wants to change the pci bus speed?

How often will this bus change happen?

Does it really save battery?

99% of C3s aren't in battery powered devices, but it can save a considerable amount of power (~75% on some of the earlier chips iirc).

Thankfully their newer cpu's aren't affected by this brain damage, but there's a _lot_ of the older CPUs out there (all those EPIA boards etc).

Dave

Signature · 1 line
http://www.codemonkey.org.uk