Skip to contentSkip to conversation

x86 calgary: add fallback dma_ops]]

RFC PATCH In review Alexis Bruemmer 3 participants 5 messages May 8, 2008 – May 11, 2008 x86

lore
Share & tools
lore

Alexis Bruemmer

author
#0 lore raw

RFC PATCH 1/1 x86 calgary: add fallback dma_ops]]

Currently the calgary code can give drivers addresses above 4GB which is very bad for hardware that is only 32bit DMA addressable. This patch "teaches" calgary to fallback to the appropriate dma_ops when it encounters a device/bus which is not behind the Calgary/CalIOC2. I believe there is a better way to do this and am open for ideas, but for now this certainly fixes the badness.

Signed-off-by Alexis Bruemmer <[email protected]>

---

arch/x86/kernel/pci-nommu.c +5 -0
Index: linux-2.6.26-rc1/arch/x86/kernel/pci-nommu.c
===================================================================
@@ -10,6 +10,7 @@
 #include <asm/gart.h>
 #include <asm/processor.h>
 #include <asm/dma.h>
+#include <asm/calgary.h>
 
 static int
 check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size)

@@ -92,6 +93,10 @@ const struct dma_mapping_ops nommu_dma_o
 
 void __init no_iommu_init(void)
 {
+#ifdef CONFIG_CALGARY_IOMMU
+	if (use_calgary && (end_pfn < MAX_DMA32_PFN))
+		fallback_dma_ops = &nommu_dma_ops;
+#endif
 	if (dma_ops)
 		return;
arch/x86/kernel/pci-swiotlb_64.c +7 -0
Index: linux-2.6.26-rc1/arch/x86/kernel/pci-swiotlb_64.c
===================================================================
@@ -8,6 +8,7 @@
 #include <asm/gart.h>
 #include <asm/swiotlb.h>
 #include <asm/dma.h>
+#include <asm/calgary.h>
 
 int swiotlb __read_mostly;
 

@@ -47,4 +48,10 @@ void __init pci_swiotlb_init(void)
 		swiotlb_init();
 		dma_ops = &swiotlb_dma_ops;
 	}
+#ifdef CONFIG_CALGARY_IOMMU
+	else if (use_calgary && (end_pfn > MAX_DMA32_PFN)) {
+		swiotlb_init();
+		fallback_dma_ops = &swiotlb_dma_ops;
+	}
+#endif
 }
include/asm-x86/calgary.h +1 -0
Index: linux-2.6.26-rc1/include/asm-x86/calgary.h
===================================================================
@@ -60,6 +60,7 @@ struct cal_chipset_ops {
 #define TCE_TABLE_SIZE_8M		7
 
 extern int use_calgary;
+extern const struct dma_mapping_ops* fallback_dma_ops;
 
 #ifdef CONFIG_CALGARY_IOMMU
 extern int calgary_iommu_init(void);
arch/x86/kernel/pci-calgary_64.c +18 -31

95 lines not on this page · raw message

#1 lore raw
Quoted 6 lines · On Thu, 08 May 2008 14:40:20 -0700 Alexis Bruemmer <[email protected]> wrote:

Currently the calgary code can give drivers addresses above 4GB which is very bad for hardware that is only 32bit DMA addressable. This patch "teaches" calgary to fallback to the appropriate dma_ops when it encounters a device/bus which is not behind the Calgary/CalIOC2. I believe there is a better way to do this and am open for ideas, but for now this certainly fixes the badness.

I'm not sure that I correctly understand what you want. You mean that the Calgary IOMMU code ignores device's dma_mask and gives addresses above 4GB or the Calgary IOMMU code wrongly handles devices that are not behind the Calgary?

If you refers to the former problem, other IOMMU implementations do something like this.

arch/x86/kernel/pci-calgary_64.c +18 -2
index e28ec49..2a977bc 100644
@@ -268,21 +268,37 @@ static unsigned long iommu_range_alloc(struct device *dev,
 {
 	unsigned long flags;
 	unsigned long offset;
+	unsigned long limit;
 	unsigned long boundary_size;
+	unsigned long start;
+	unsigned long mask;
 
 	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
 			      PAGE_SIZE) >> PAGE_SHIFT;
 
 	BUG_ON(npages == 0);
 
+	if (dev->dma_mask)
+		mask = *dev->dma_mask >> PAGE_SHIFT;
+	else
+		mask = 0xfffffffful >> PAGE_SHIFT;
+
+	limit = tbl->it_size;
+	start = tbl->it_hint;
+
+	if (limit > mask) {
+		limit = mask + 1;
+		start = tbl->it_hint & mask;
+	}
+
 	spin_lock_irqsave(&tbl->it_lock, flags);
 
-	offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
+	offset = iommu_area_alloc(tbl->it_map, limit, start,
 				  npages, 0, boundary_size, 0);
 	if (offset == ~0UL) {
 		tbl->chip_ops->tce_cache_blast(tbl);
 
-		offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
+		offset = iommu_area_alloc(tbl->it_map, limit, 0,
 					  npages, 0, boundary_size, 0);
 		if (offset == ~0UL) {
 			printk(KERN_WARNING "Calgary: IOMMU full.\n");
author
#2 lore raw
Quoted 10 lines · FUJITA Tomonori
Quoted 6 lines · On Thu, 08 May 2008 14:40:20 -0700 Alexis Bruemmer <[email protected]> wrote:

Currently the calgary code can give drivers addresses above 4GB which is very bad for hardware that is only 32bit DMA addressable. This patch "teaches" calgary to fallback to the appropriate dma_ops when it encounters a device/bus which is not behind the Calgary/CalIOC2. I believe there is a better way to do this and am open for ideas, but for now this certainly fixes the badness.

Quoted 4 lines

I'm not sure that I correctly understand what you want. You mean that the Calgary IOMMU code ignores device's dma_mask and gives addresses above 4GB or the Calgary IOMMU code wrongly handles devices that are not behind the Calgary?

The real issue is the latter-- the Calgary IOMMU code does not properly handle devices that are not behind the Calgary/CalIO2.

Quoted 49 lines

If you refers to the former problem, other IOMMU implementations do something like this.

arch/x86/kernel/pci-calgary_64.c +18 -2
index e28ec49..2a977bc 100644
@@ -268,21 +268,37 @@ static unsigned long iommu_range_alloc(struct device *dev,
 {
 	unsigned long flags;
 	unsigned long offset;
+	unsigned long limit;
 	unsigned long boundary_size;
+	unsigned long start;
+	unsigned long mask;
 
 	boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
 			      PAGE_SIZE) >> PAGE_SHIFT;
 
 	BUG_ON(npages == 0);
 
+	if (dev->dma_mask)
+		mask = *dev->dma_mask >> PAGE_SHIFT;
+	else
+		mask = 0xfffffffful >> PAGE_SHIFT;
+
+	limit = tbl->it_size;
+	start = tbl->it_hint;
+
+	if (limit > mask) {
+		limit = mask + 1;
+		start = tbl->it_hint & mask;
+	}
+
 	spin_lock_irqsave(&tbl->it_lock, flags);
 
-	offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,
+	offset = iommu_area_alloc(tbl->it_map, limit, start,
 				  npages, 0, boundary_size, 0);
 	if (offset == ~0UL) {
 		tbl->chip_ops->tce_cache_blast(tbl);
 
-		offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,
+		offset = iommu_area_alloc(tbl->it_map, limit, 0,
 					  npages, 0, boundary_size, 0);
 		if (offset == ~0UL) {
 			printk(KERN_WARNING "Calgary: IOMMU full.\n");
#3 lore raw
Quoted 11 lines · On Thu, 08 May 2008 16:41:51 -0700 Alexis Bruemmer <[email protected]> wrote:
Quoted 10 lines · FUJITA Tomonori
Quoted 6 lines · On Thu, 08 May 2008 14:40:20 -0700 Alexis Bruemmer <[email protected]> wrote:

Currently the calgary code can give drivers addresses above 4GB which is very bad for hardware that is only 32bit DMA addressable. This patch "teaches" calgary to fallback to the appropriate dma_ops when it encounters a device/bus which is not behind the Calgary/CalIOC2. I believe there is a better way to do this and am open for ideas, but for now this certainly fixes the badness.

Quoted 6 lines
Quoted 4 lines

I'm not sure that I correctly understand what you want. You mean that the Calgary IOMMU code ignores device's dma_mask and gives addresses above 4GB or the Calgary IOMMU code wrongly handles devices that are not behind the Calgary?

The real issue is the latter-- the Calgary IOMMU code does not properly handle devices that are not behind the Calgary/CalIO2.

Thanks, now I see why you use swiotlb for such devices in the case of end_pfn > MAX_DMA32_PFN and no_dma_ops works for them in the case of of end_pfn < MAX_DMA32_PFN.

Can we put a pointer to dma_ops in struct device (archdata) like POWER does? The way to setup and handle x86 IOMMUs seems to become hacky day by day.

#4 lore raw
Quoted 3 lines · FUJITA Tomonori

Can we put a pointer to dma_ops in struct device (archdata) like POWER does? The way to setup and handle x86 IOMMUs seems to become hacky day by day.

Per-device dma-ops will be useful for KVM (for pass-through device support) and IB as well.

Cheers,
Muli