Skip to contentSkip to conversation

Add APCI tables support for AMD IOMMU

PATCH In review Wei Wang2 4 participants 6 messages Feb 28, 2008

lore
Share & tools
lore

Wei Wang2

author
#0 lore raw

PATCH 2/2 Add APCI tables support for AMD IOMMU

Signature · 12 lines
AMD Saxony, Dresden, Germany
Operating System Research Center

Legal Information:
AMD Saxony Limited Liability Company & Co. KG
Sitz (Geschäftsanschrift):
   Wilschdorfer Landstr. 101, 01109 Dresden, Deutschland
Registergericht Dresden: HRA 4896
vertretungsberechtigter Komplementär:
   AMD Saxony LLC (Sitz Wilmington, Delaware, USA)
Geschäftsführer der AMD Saxony LLC:
   Dr. Hans-R. Deppe, Thomas McCoy
xen/drivers/acpi/tables.c +1 -0
diff -r f8624b023d67 xen/drivers/acpi/tables.c
@@ -60,6 +60,7 @@ static char *acpi_table_signatures[ACPI_
 	[ACPI_HPET] = "HPET",
 	[ACPI_MCFG] = "MCFG",
 	[ACPI_DMAR] = "DMAR",
+	[ACPI_IVRS] = "IVRS",
 };
 
 static char *mps_inti_flags_polarity[] = { "dfl", "high", "res", "low" };
xen/drivers/passthrough/amd/Makefile +1 -0
diff -r f8624b023d67 xen/drivers/passthrough/amd/Makefile
@@ -2,3 +2,4 @@ obj-y += iommu_init.o
 obj-y += iommu_init.o
 obj-y += iommu_map.o
 obj-y += pci_amd_iommu.o
+obj-y += iommu_acpi.o
xen/drivers/passthrough/amd/iommu_detect.c +19 -17
diff -r f8624b023d67 xen/drivers/passthrough/amd/iommu_detect.c
@@ -86,30 +86,24 @@ int __init get_iommu_capabilities(u8 bus
 int __init get_iommu_capabilities(u8 bus, u8 dev, u8 func, u8 cap_ptr,
             struct amd_iommu *iommu)
 {
-    u32 cap_header, cap_range;
+    u32 cap_header, cap_range, misc_info;
     u64 mmio_bar;
 
-#if HACK_BIOS_SETTINGS
-    /* remove it when BIOS available */
-    write_pci_config(bus, dev, func,
-        cap_ptr + PCI_CAP_MMIO_BAR_HIGH_OFFSET, 0x00000000);
-    write_pci_config(bus, dev, func,
-        cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET, 0x40000001);
-    /* remove it when BIOS available */
-#endif
-
     mmio_bar = (u64)read_pci_config(bus, dev, func,
-             cap_ptr + PCI_CAP_MMIO_BAR_HIGH_OFFSET) << 32;
+            cap_ptr + PCI_CAP_MMIO_BAR_HIGH_OFFSET) << 32;
     mmio_bar |= read_pci_config(bus, dev, func,
-            cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET) &
-            PCI_CAP_MMIO_BAR_LOW_MASK;
-    iommu->mmio_base_phys = (unsigned long)mmio_bar;
-
-    if ( (mmio_bar == 0) || ( (mmio_bar & 0x3FFF) != 0 ) ) {
+            cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET); 
+    iommu->mmio_base_phys = mmio_bar & (u64)~0x3FFF;
+
+    if ( (mmio_bar & 0x1) == 0 || iommu->mmio_base_phys == 0 )
+    {
         dprintk(XENLOG_ERR ,
                 "AMD IOMMU: Invalid MMIO_BAR = 0x%"PRIx64"\n", mmio_bar);
         return -ENODEV;
     }
+
+    iommu->bdf = (bus << 8) | PCI_DEVFN(dev, func);
+    iommu->cap_offset = cap_ptr;
 
     cap_header = read_pci_config(bus, dev, func, cap_ptr);
     iommu->revision = get_field_from_reg_u32(cap_header,
@@ -119,12 +113,15 @@ int __init get_iommu_capabilities(u8 bus
     iommu->ht_tunnel_support = get_field_from_reg_u32(cap_header,
                     PCI_CAP_HT_TUNNEL_MASK,
                     PCI_CAP_HT_TUNNEL_SHIFT);
-    iommu->not_present_cached = get_field_from_reg_u32(cap_header,
+    iommu->pte_not_present_cached = get_field_from_reg_u32(cap_header,
                     PCI_CAP_NP_CACHE_MASK,
                     PCI_CAP_NP_CACHE_SHIFT);
 
     cap_range = read_pci_config(bus, dev, func,
             cap_ptr + PCI_CAP_RANGE_OFFSET);
+    iommu->unit_id = get_field_from_reg_u32(cap_range,
+                PCI_CAP_UNIT_ID_MASK,
+                PCI_CAP_UNIT_ID_SHIFT);
     iommu->root_bus = get_field_from_reg_u32(cap_range,
                 PCI_CAP_BUS_NUMBER_MASK,
                 PCI_CAP_BUS_NUMBER_SHIFT);
@@ -135,6 +132,11 @@ int __init get_iommu_capabilities(u8 bus
                 PCI_CAP_LAST_DEVICE_MASK,
                 PCI_CAP_LAST_DEVICE_SHIFT);
 
+    misc_info = read_pci_config(bus, dev, func,
+            cap_ptr + PCI_MISC_INFO_OFFSET);
+    iommu->msi_number = get_field_from_reg_u32(misc_info,
+                PCI_CAP_MSI_NUMBER_MASK,
+                PCI_CAP_MSI_NUMBER_SHIFT);
     return 0;
 }
 
xen/drivers/passthrough/amd/iommu_init.c +41 -0

49 lines not on this page · raw message

xen/drivers/passthrough/amd/iommu_map.c +38 -4

57 lines not on this page · raw message

xen/drivers/passthrough/amd/pci_amd_iommu.c +112 -30

273 lines not on this page · raw message

xen/include/asm-x86/amd-iommu.h +28 -4
diff -r f8624b023d67 xen/include/asm-x86/amd-iommu.h
@@ -43,14 +43,25 @@ struct amd_iommu {
     struct list_head list;
     spinlock_t lock; /* protect iommu */
 
-    int iotlb_support;
-    int ht_tunnel_support;
-    int not_present_cached;
+    u16 bdf;
+    u8  cap_offset;
     u8  revision;
+    u8  unit_id;
+    u8  msi_number;
 
     u8  root_bus;
     u8  first_devfn;
     u8  last_devfn;
+
+    u8 pte_not_present_cached;
+    u8 ht_tunnel_support;
+    u8 iotlb_support;
+
+    u8 isochronous;
+    u8 coherent;
+    u8 res_pass_pw;
+    u8 pass_pw;
+    u8 ht_tunnel_enable;
 
     int last_downstream_bus;
     int downstream_bus_present[PCI_MAX_BUS_COUNT];
@@ -61,10 +72,23 @@ struct amd_iommu {
     struct table_struct dev_table;
     struct table_struct cmd_buffer;
     u32 cmd_buffer_tail;
+    struct table_struct event_log;
+    u32 event_log_head;
 
-    int exclusion_enabled;
+    int exclusion_enable;
+    int exclusion_allow_all;
     unsigned long exclusion_base;
     unsigned long exclusion_limit;
 };
 
+struct ivrs_mappings {
+    u16 dte_requestor_id;
+    u8 dte_sys_mgt_enable;
+    u8 dte_allow_exclusion;
+    u8 unity_map_enable;
+    u8 write_permission;
+    u8 read_permission;
+    unsigned long addr_range_start;
+    unsigned long addr_range_length;
+};
 #endif /* _ASM_X86_64_AMD_IOMMU_H */
xen/include/asm-x86/hvm/svm/amd-iommu-defs.h +6 -0
diff -r f8624b023d67 xen/include/asm-x86/hvm/svm/amd-iommu-defs.h
@@ -117,6 +117,12 @@
 #define PCI_CAP_FIRST_DEVICE_SHIFT	16
 #define PCI_CAP_LAST_DEVICE_MASK	0xFF000000
 #define PCI_CAP_LAST_DEVICE_SHIFT	24
+
+#define PCI_CAP_UNIT_ID_MASK    0x0000001F
+#define PCI_CAP_UNIT_ID_SHIFT   0
+#define PCI_MISC_INFO_OFFSET    0x10
+#define PCI_CAP_MSI_NUMBER_MASK     0x0000001F
+#define PCI_CAP_MSI_NUMBER_SHIFT    0
 
 /* Device Table */
 #define IOMMU_DEV_TABLE_BASE_LOW_OFFSET		0x00
xen/include/asm-x86/hvm/svm/amd-iommu-proto.h +21 -3
diff -r f8624b023d67 xen/include/asm-x86/hvm/svm/amd-iommu-proto.h
@@ -21,6 +21,7 @@
 #ifndef _ASM_X86_64_AMD_IOMMU_PROTO_H
 #define _ASM_X86_64_AMD_IOMMU_PROTO_H
 
+#include <xen/sched.h>
 #include <asm/amd-iommu.h>
 
 #define for_each_amd_iommu(amd_iommu) \
@@ -54,10 +55,12 @@ int amd_iommu_map_page(struct domain *d,
 int amd_iommu_map_page(struct domain *d, unsigned long gfn, unsigned long mfn);
 int amd_iommu_unmap_page(struct domain *d, unsigned long gfn);
 void *amd_iommu_get_vptr_from_page_table_entry(u32 *entry);
+int amd_iommu_reserve_domain_unity_map(struct domain *domain,
+        unsigned long phys_addr, unsigned long size, int iw, int ir);
 
 /* device table functions */
-void amd_iommu_set_dev_table_entry(u32 *dte,
-        u64 root_ptr, u16 domain_id, u8 paging_mode);
+void amd_iommu_set_dev_table_entry(u32 *dte, u64 root_ptr,
+        u16 domain_id, u8 sys_mgt, u8 dev_ex, u8 paging_mode);
 int amd_iommu_is_dte_page_translation_valid(u32 *entry);
 void invalidate_dev_table_entry(struct amd_iommu *iommu,
             u16 devic_id);
@@ -69,10 +72,13 @@ void flush_command_buffer(struct amd_iom
 /* iommu domain funtions */
 int amd_iommu_domain_init(struct domain *domain);
 void amd_iommu_setup_domain_device(struct domain *domain,
-    struct amd_iommu *iommu, int requestor_id);
+    struct amd_iommu *iommu, int bdf);
 
 /* find iommu for bdf */
 struct amd_iommu *find_iommu_for_device(int bus, int devfn);
+
+/* amd-iommu-acpi functions */
+int __init parse_ivrs_table(unsigned long phys_addr, unsigned long size);
 
 static inline u32 get_field_from_reg_u32(u32 reg_value, u32 mask, u32 shift)
 {
@@ -91,4 +97,16 @@ static inline u32 set_field_in_reg_u32(u
     return reg_value;
 }
 
+static inline u8 get_field_from_byte(u8 value, u8 mask, u8 shift)
+{
+    u8 field;
+    field = (value & mask) >> shift;
+    return field;
+}
+
+static inline unsigned long region_to_pages(unsigned long addr, unsigned long size)
+{
+    return (PAGE_ALIGN(addr + size) - (addr & PAGE_MASK)) >> PAGE_SHIFT;
+}
+
 #endif /* _ASM_X86_64_AMD_IOMMU_PROTO_H */
xen/include/xen/acpi.h +1 -0
diff -r f8624b023d67 xen/include/xen/acpi.h
@@ -368,6 +368,7 @@ enum acpi_table_id {
 	ACPI_HPET,
 	ACPI_MCFG,
 	ACPI_DMAR,
+	ACPI_IVRS,
 	ACPI_TABLE_COUNT
 };
 

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#1 lore raw

Hi there,

Some picky comments inline, most of them are formatting but one semantic question too. I've removed most of the diff lines but left a little context before each comment.

- cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET) &
- PCI_CAP_MMIO_BAR_LOW_MASK;
- iommu->mmio_base_phys = (unsigned long)mmio_bar;
-
- if ( (mmio_bar == 0) || ( (mmio_bar & 0x3FFF) != 0 ) ) {
+ cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET);

Tiny trailing whitespace here.

+static void __init register_iommu_exclusion_range(struct amd_iommu *iommu) +{
+ u64 addr_lo, addr_hi;
+ u32 entry;
+
+ addr_lo = iommu->exclusion_limit & DMA_32BIT_MASK;
+ addr_hi = iommu->exclusion_limit >> 32;

iommu->exclusion_limit is an unsigned long, so on x86_32 you are bit shifting exclusion_limit by a value equal to its width. I would guess that gcc probably does the right thing in this case, but I don't know if this is strictly allowed by the C standard?

A question leading on from that, I guess, is whether exclusion_limit ought to be a u64 like addr_lo and addr_hi?

+    spin_lock_irqsave(&hd->mapping_lock, flags);
+    for ( i = 0; i < npages; ++i )
+    {

+ pte = get_pte_from_page_tables(hd->root_table,
+ hd->paging_mode, phys_addr>>PAGE_SHIFT);
+ if ( pte == 0 )

Most of the Xen codebase uses NULL rather than 0 to indicate a null pointer. Your use seems to be consistent with the rest of this file, though, so maybe that's acceptable here...

+        /* allocate 'ivrs mappings' table */
+        /* note: the table has entries to accomodate all IOMMUs */
+        last_bus = 0;
+        for_each_amd_iommu (iommu)

+ if (iommu->last_downstream_bus > last_bus)

Needs spaces between the brackets and expression ;-)

int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn) {

+    int bdf = (bus << 8) | devfn;
+    int req_id;
+    req_id = ivrs_mappings[bdf].dte_requestor_id;

+
+ if (ivrs_mappings[req_id].unity_map_enable)

Bracket spacing again ;-)

Cheers,
Mark

Signature · 1 line
Push Me Pull You - Distributed SCM tool (http://www.cl.cam.ac.uk/~maw48/pmpu/)
#2 lore raw

To be honest I didn't look all that closely at these patches, beyond observing that they largely modify AMD-IOMMU-specific files. It would be nice to fix up the coding style -- I tend to do this myself only when I already have the offending files loaded into an Emacs buffer for some other reason.

-- Keir

Quoted 62 lines · Mark Williamson

Hi there,

Some picky comments inline, most of them are formatting but one semantic question too. I've removed most of the diff lines but left a little context before each comment.

xen/drivers/passthrough/amd/iommu_detect.c +1 -5
-            cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET) &
-            PCI_CAP_MMIO_BAR_LOW_MASK;
-    iommu->mmio_base_phys = (unsigned long)mmio_bar;
-
-    if ( (mmio_bar == 0) || ( (mmio_bar & 0x3FFF) != 0 ) ) {
+            cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET);

Tiny trailing whitespace here.

xen/drivers/passthrough/amd/iommu_init.c +7 -0
+static void __init register_iommu_exclusion_range(struct amd_iommu *iommu)
+{
+    u64 addr_lo, addr_hi;
+    u32 entry;
+
+    addr_lo = iommu->exclusion_limit & DMA_32BIT_MASK;
+    addr_hi = iommu->exclusion_limit >> 32;

iommu->exclusion_limit is an unsigned long, so on x86_32 you are bit shifting exclusion_limit by a value equal to its width. I would guess that gcc probably does the right thing in this case, but I don't know if this is strictly allowed by the C standard?

A question leading on from that, I guess, is whether exclusion_limit ought to be a u64 like addr_lo and addr_hi?

xen/drivers/passthrough/amd/iommu_map.c +6 -0
+    spin_lock_irqsave(&hd->mapping_lock, flags);
+    for ( i = 0; i < npages; ++i )
+    {
+        pte = get_pte_from_page_tables(hd->root_table,
+           hd->paging_mode, phys_addr>>PAGE_SHIFT);
+        if ( pte == 0 )

Most of the Xen codebase uses NULL rather than 0 to indicate a null pointer. Your use seems to be consistent with the rest of this file, though, so maybe that's acceptable here...

xen/drivers/passthrough/amd/pci_amd_iommu.c +5 -0
+        /* allocate 'ivrs mappings' table */
+        /* note: the table has entries to accomodate all IOMMUs */
+        last_bus = 0;
+        for_each_amd_iommu (iommu)
+           if (iommu->last_downstream_bus > last_bus)

Needs spaces between the brackets and expression ;-)

int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn) {

xen/drivers/passthrough/amd/pci_amd_iommu.c +5 -0
+    int bdf = (bus << 8) | devfn;
+    int req_id;
+    req_id = ivrs_mappings[bdf].dte_requestor_id;
+
+    if (ivrs_mappings[req_id].unity_map_enable)

Bracket spacing again ;-)

Cheers,
Mark

#3 lore raw
Quoted 5 lines

To be honest I didn't look all that closely at these patches, beyond observing that they largely modify AMD-IOMMU-specific files. It would be nice to fix up the coding style -- I tend to do this myself only when I already have the offending files loaded into an Emacs buffer for some other reason.

ok.

FWIW, my checker tree has a Xen coding style checker plus a writeup of some key points of the coding style as a document. Would you take a patch to add these to the tree for future contributors to refer to?

Cheers,
Mark

Quoted 65 lines

-- Keir

Quoted 62 lines · Mark Williamson

Hi there,

Some picky comments inline, most of them are formatting but one semantic question too. I've removed most of the diff lines but left a little context before each comment.

xen/drivers/passthrough/amd/iommu_detect.c +1 -5
-            cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET) &
-            PCI_CAP_MMIO_BAR_LOW_MASK;
-    iommu->mmio_base_phys = (unsigned long)mmio_bar;
-
-    if ( (mmio_bar == 0) || ( (mmio_bar & 0x3FFF) != 0 ) ) {
+            cap_ptr + PCI_CAP_MMIO_BAR_LOW_OFFSET);

Tiny trailing whitespace here.

+static void __init register_iommu_exclusion_range(struct amd_iommu *iommu) +{

xen/drivers/passthrough/amd/iommu_init.c +5 -0
+    u64 addr_lo, addr_hi;
+    u32 entry;
+
+    addr_lo = iommu->exclusion_limit & DMA_32BIT_MASK;
+    addr_hi = iommu->exclusion_limit >> 32;

iommu->exclusion_limit is an unsigned long, so on x86_32 you are bit shifting exclusion_limit by a value equal to its width. I would guess that gcc probably does the right thing in this case, but I don't know if this is strictly allowed by the C standard?

A question leading on from that, I guess, is whether exclusion_limit ought to be a u64 like addr_lo and addr_hi?

xen/drivers/passthrough/amd/iommu_map.c +6 -0
+    spin_lock_irqsave(&hd->mapping_lock, flags);
+    for ( i = 0; i < npages; ++i )
+    {
+        pte = get_pte_from_page_tables(hd->root_table,
+           hd->paging_mode, phys_addr>>PAGE_SHIFT);
+        if ( pte == 0 )

Most of the Xen codebase uses NULL rather than 0 to indicate a null pointer. Your use seems to be consistent with the rest of this file, though, so maybe that's acceptable here...

xen/drivers/passthrough/amd/pci_amd_iommu.c +5 -0
+        /* allocate 'ivrs mappings' table */
+        /* note: the table has entries to accomodate all IOMMUs */
+        last_bus = 0;
+        for_each_amd_iommu (iommu)
+           if (iommu->last_downstream_bus > last_bus)

Needs spaces between the brackets and expression ;-)

int amd_iommu_assign_device(struct domain *d, u8 bus, u8 devfn) {

xen/drivers/passthrough/amd/pci_amd_iommu.c +5 -0
+    int bdf = (bus << 8) | devfn;
+    int req_id;
+    req_id = ivrs_mappings[bdf].dte_requestor_id;
+
+    if (ivrs_mappings[req_id].unity_map_enable)

Bracket spacing again ;-)

Cheers,
Mark

Signature · 1 line
Push Me Pull You - Distributed SCM tool (http://www.cl.cam.ac.uk/~maw48/pmpu/)
#4 lore raw
Quoted 11 lines · Mark Williamson
Quoted 5 lines

To be honest I didn't look all that closely at these patches, beyond observing that they largely modify AMD-IOMMU-specific files. It would be nice to fix up the coding style -- I tend to do this myself only when I already have the offending files loaded into an Emacs buffer for some other reason.

ok.

FWIW, my checker tree has a Xen coding style checker plus a writeup of some key points of the coding style as a document. Would you take a patch to add these to the tree for future contributors to refer to?

I'd certainly strongly consider it.

-- Keir

#5 lore raw

RE: [amd iommu] [patch 2/2]Add APCI tables supportfor AMD IOMMU

Wei Wang just left for a much deserved vacation, although out of email contact. So, he'll certainly get to the refactoring when he gets back. I'll recommend that he fixup all of the AMD IOMMU code, referring to Mark's nice coding style checker + doc.
tom

Quoted 43 lines

-----Original Message-----
From: [email protected] [mailto:[email protected]] On Behalf Of Keir Fraser
Sent: Thursday, February 28, 2008 3:22 PM
To: Mark Williamson
Cc: Wang2, Wei; [email protected]
Subject: Re: [Xen-devel] [amd iommu] [patch 2/2]Add APCI tables supportfor AMD IOMMU

Quoted 1 line · Mark Williamson
Quoted 1 line

To be honest I didn't look all that closely at these

patches, beyond

Quoted 1 line
Quoted 1 line

observing that they largely modify AMD-IOMMU-specific

files. It would

Quoted 1 line
Quoted 1 line

be nice to fix up the coding style -- I tend to do this

myself only

Quoted 1 line
Quoted 1 line

when I already have the offending files loaded into an

Emacs buffer

Quoted 5 lines
Quoted 1 line

for some other reason.

ok.

FWIW, my checker tree has a Xen coding style checker plus a

writeup of

Quoted 1 line

some key points of the coding style as a document. Would

you take a

Quoted 1 line

patch to add these to the tree for future contributors to refer to?

I'd certainly strongly consider it.

-- Keir

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel