Fix >4G i386 PAE grant table interface
Subsystems (2)
PATCH 1/1 Fix >4G i386 PAE grant table interface
It has been discovered that i386 boxes with more than 4G of RAM would randomly crash. It was traced to the interface of blktap using gnttab_set_map_op.
It would pass in the 64 bit pte entry, but the gnttab_set_map_op would only take a 32 bit (on i386) unsigned long as a parameter. So we lose the top 32bits.
Luckily! The kernel/HV ABI used a uint64_t as the variable to pass the address. So this does *NOT* break the current kernel/HV ABI.
But after the HV grabs the 64 bit address from the guest, it too calls a function that uses a unsigned long (32bits on i386) to pass that address with. So the HV side also chops off the top 64 bits of the variable.
This patch updates both the linux-2.6-sparse tree and the xen HV to use uint64_t instead of unsigned long for those particular functions. This patch has been tested on RHEL5 Beta on a box with 12G i386.
linux-2.6-xen-sparse/include/xen/gnttab.h +2 -2
@@ -118,7 +118,7 @@ int gnttab_resume(void);int gnttab_resume (void );static inline void -gnttab_set_map_op (struct gnttab_map_grant_ref *map,unsigned long addr, +gnttab_set_map_op (struct gnttab_map_grant_ref *map,uint64_t addr,uint32_t flags,grant_ref_t ref,domid_t domid) {if (flags & GNTMAP_contains_pte)
@@ -134,7 +134,7 @@ gnttab_set_map_op(struct gnttab_map_gran }static inline void -gnttab_set_unmap_op (struct gnttab_unmap_grant_ref *unmap,unsigned long addr, +gnttab_set_unmap_op (struct gnttab_unmap_grant_ref *unmap,uint64_t addr,uint32_t flags,grant_handle_t handle) {if (flags & GNTMAP_contains_pte)
xen/arch/powerpc/mm.c +2 -2
@@ -43,14 +43,14 @@ unsigned long total_pages;unsigned long total_pages;int create_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags) +uint64_t addr,unsigned long frame,unsigned int flags) {panic ("%s called\n" , __func__);return 1 ; }int destroy_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags) +uint64_t addr,unsigned long frame,unsigned int flags) {panic ("%s called\n" , __func__);return 1 ;
xen/arch/x86/mm.c +7 -7
@@ -2379,7 +2379,7 @@ int do_mmu_update(static int create_grant_pte_mapping ( -unsigned long pte_addr,l1_pgentry_t nl1e,struct vcpu *v) +uint64_t pte_addr,l1_pgentry_t nl1e,struct vcpu *v) {int rc = GNTST_okay;void *va;
@@ -2403,7 +2403,7 @@ static int create_grant_pte_mapping( } va =map_domain_page (mfn); - va = (void *)((unsigned long )va + (pte_addr & ~PAGE_MASK )); + va = (void *)((unsigned long )va + ((unsigned long )pte_addr & ~PAGE_MASK )); page =mfn_to_page (mfn); type = page->u.inuse.type_info & PGT_type_mask;
@@ -2435,7 +2435,7 @@ static int create_grant_pte_mapping( }static int destroy_grant_pte_mapping ( -unsigned long addr,unsigned long frame,struct domain *d) +uint64_t addr,unsigned long frame,struct domain *d) {int rc = GNTST_okay;void *va;
@@ -2454,7 +2454,7 @@ static int destroy_grant_pte_mapping( } va =map_domain_page (mfn); - va = (void *)((unsigned long )va + (addr & ~PAGE_MASK )); + va = (void *)((unsigned long )va + ((unsigned long )addr & ~PAGE_MASK )); page =mfn_to_page (mfn); type = page->u.inuse.type_info & PGT_type_mask;
@@ -2475,7 +2475,7 @@ static int destroy_grant_pte_mapping(/* Check that the virtual address supplied is actually mapped to frame. */ if (unlikely ((l1e_get_intpte (ol1e) >>PAGE_SHIFT ) != frame) ) { -MEM_LOG ("PTE entry %lx for address %lx doesn't match frame %lx" , +MEM_LOG ("PTE entry %lx for address %llx doesn't match frame %lx" , (unsigned long )l1e_get_intpte (ol1e), addr, frame);put_page_type (page); rc = GNTST_general_error;
@@ -2575,7 +2575,7 @@ static int destroy_grant_va_mapping( }int create_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags) +uint64_t addr,unsigned long frame,unsigned int flags) {l1_pgentry_t pte =l1e_from_pfn (frame,GRANT_PTE_FLAGS );
@@ -2590,7 +2590,7 @@ int create_grant_host_mapping( }int destroy_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags) +uint64_t addr,unsigned long frame,unsigned int flags) {if ( flags & GNTMAP_contains_pte )return destroy_grant_pte_mapping (addr, frame, current->domain);
xen/include/asm-powerpc/grant_table.h +2 -2
@@ -30,9 +30,9 @@* must hold a reference to the page. */ int create_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags); +uint64_t addr,unsigned long frame,unsigned int flags);int destroy_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags); +uint64_t addr,unsigned long frame,unsigned int flags);#define gnttab_create_shared_page (d, t, i) \ do { \
xen/include/asm-x86/grant_table.h +2 -2
@@ -14,9 +14,9 @@* must hold a reference to the page. */ int create_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags); +uint64_t addr,unsigned long frame,unsigned int flags);int destroy_grant_host_mapping ( -unsigned long addr,unsigned long frame,unsigned int flags); +uint64_t addr,unsigned long frame,unsigned int flags);#define gnttab_create_shared_page (d, t, i) \ do { \
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Quoted 5 lines · Steven Rostedt
This patch updates both the linux-2.6-sparse tree and the xen HV to use
uint64_tinstead of unsigned long for those particular functions. This patch has been tested on RHEL5 Beta on a box with 12G i386.Signed-off-by Steven Rostedt <[email protected]>
Nasty bug. At least it affects only blktap. Thanks.
-- Keir
Quoted 8 lines
Quoted 1 line
Quoted 1 line
Steven Rostedt <[email protected]> 02.11.06 17:53 >>>
It has been discovered that i386 boxes with more than 4G of RAM would randomly crash. It was traced to the interface of blktap using
gnttab_set_map_op.It would pass in the 64 bit pte entry, but the
gnttab_set_map_opwould only take a 32 bit (on i386) unsigned long as a parameter. So we lose the top 32bits.
Could you use maddr_t here rather than uint64_t? For non-PAE i386 Linux, especially when using CONFIG_REGPARM, adding a useless argument slot seems wasteful...
Keir - will this go into 3.0.3?
Jan
Quoted 10 lines
Quoted 1 line
Quoted 1 line
Keir Fraser <[email protected]> 02.11.06 19:16 >>>
Quoted 5 lines · Steven Rostedt
This patch updates both the linux-2.6-sparse tree and the xen HV to use
uint64_tinstead of unsigned long for those particular functions. This patch has been tested on RHEL5 Beta on a box with 12G i386.Signed-off-by Steven Rostedt <[email protected]>Nasty bug. At least it affects only blktap. Thanks.
Looking at this it would seem to me that the second call to gnttab_set_unmap_op in blkltap.c is missing the GNTMAP_contains_pte flag, affecting auto_translated_physmap guests.
drivers/xen/blktap/blktap.c +4 -2
@@ -908,8 +908,10 @@ static void fast_flush_area(pending_req_return ; } -gnttab_set_unmap_op (&unmap[invcount], - ptep, GNTMAP_host_map, +gnttab_set_unmap_op (&unmap[invcount], ptep, + GNTMAP_host_map + | GNTMAP_application_map + | GNTMAP_contains_pte, khandle->user); invcount++; }
Quoted 17 lines
Quoted 1 line
Quoted 1 line
Steven Rostedt <[email protected]> 02.11.06 17:53 >>>
It has been discovered that i386 boxes with more than 4G of RAM would randomly crash. It was traced to the interface of blktap using
gnttab_set_map_op.It would pass in the 64 bit pte entry, but the
gnttab_set_map_opwould only take a 32 bit (on i386) unsigned long as a parameter. So we lose the top 32bits.Luckily! The kernel/HV ABI used a
uint64_tas the variable to pass the address. So this does *NOT* break the current kernel/HV ABI.But after the HV grabs the 64 bit address from the guest, it too calls a function that uses a unsigned long (32bits on i386) to pass that address with. So the HV side also chops off the top 64 bits of the variable.
Quoted 5 lines
This patch updates both the linux-2.6-sparse tree and the xen HV to use
uint64_tinstead of unsigned long for those particular functions. This patch has been tested on RHEL5 Beta on a box with 12G i386.Signed-off-by Steven Rostedt <[email protected]>
After integrating the kernel part of this patch in my tree, I found that almost the whole kernel got rebuilt. include/asm-{i386,x86_64}/mach-xen/asm/fixmap.h were needlessly including include/xen/gnttab.h. Removing this made necessary explicit inclusion of that header in tpm_xen.c, the build of which should not have succeeded on non-x86 architectures before.
drivers/char/tpm/tpm_xen.c +1 -0
@@ -41,6 +41,7 @@#include <xen/evtchn.h> #include <xen/interface/grant_table.h> #include <xen/interface/io/tpmif.h> +#include <xen/gnttab.h> #include <xen/xenbus.h> #include "tpm.h" #include "tpm_vtpm.h"
include/asm-i386/mach-xen/asm/fixmap.h +0 -1
@@ -27,7 +27,6 @@ extern unsigned long __FIXADDR_TOP;#include <asm/acpi.h> #include <asm/apicdef.h> #include <asm/page.h> -#include <xen/gnttab.h> #ifdef CONFIG_HIGHMEM #include <linux/threads.h> #include <asm/kmap_types.h>
include/asm-x86_64/mach-xen/asm/fixmap.h +0 -1
@@ -14,7 +14,6 @@#include <linux/config.h> #include <linux/kernel.h> #include <asm/apicdef.h> -#include <xen/gnttab.h> #include <asm/page.h> #include <asm/vsyscall.h> #include <asm/vsyscall32.h>
Quoted 9 lines · Jan Beulich
Quoted 4 lines
Quoted 1 line
Quoted 1 line
Steven Rostedt <[email protected]> 02.11.06 17:53 >>>
It has been discovered that i386 boxes with more than 4G of RAM would randomly crash. It was traced to the interface of blktap using
gnttab_set_map_op.It would pass in the 64 bit pte entry, but the
gnttab_set_map_opwould only take a 32 bit (on i386) unsigned long as a parameter. So we lose the top 32bits.Could you use
maddr_there rather thanuint64_t? For non-PAE i386 Linux, especially when usingCONFIG_REGPARM, adding a useless argument slot seems wasteful...
Actually, it makes no difference to me. In fact uint64_t was my third incarnation, since I wasn't sure what the best would be. I started with unsigned long long, then switched to u64, and then noticed that since host_addr is uint64_t, that seemed the proper thing to use.
So a maddr_t would work too.
Do you want to do the patch, or would you like me to send another patch that would do this change?
-- Steve
Quoted 2 lines
Do you want to do the patch, or would you like me to send another patch that would do this change?
Unless this was already committed, maybe Keir could do this on the fly?
Jan
Quoted 4 lines · Jan Beulich
Quoted 2 lines
Do you want to do the patch, or would you like me to send another patch that would do this change?
Unless this was already committed, maybe Keir could do this on the fly?
Already committed, but I'll consider patches.
-- Keir
Steven Rostedt
Acked-byQuoted 7 lines · Jan Beulich
Quoted 7 lines
Quoted 1 line
Quoted 1 line
Keir Fraser <[email protected]> 02.11.06 19:16 >>>
Quoted 4 lines · Steven Rostedt
This patch updates both the linux-2.6-sparse tree and the xen HV to use
uint64_tinstead of unsigned long for those particular functions. This patch has been tested on RHEL5 Beta on a box with 12G i386.
Quoted 7 lines
Quoted 2 lines
Quoted 1 line
Signed-off-by Steven Rostedt <[email protected]>Nasty bug. At least it affects only blktap. Thanks.
Looking at this it would seem to me that the second call to
gnttab_set_unmap_opinblkltap.cis missing theGNTMAP_contains_pteflag, affectingauto_translated_physmapguests.
It's much worst than that!
Without this patch, my patch is still broken. In the hypervisor this flag is checked, and will go down the wrong path when it is missing:
int destroy_grant_host_mapping(
u64 addr, unsigned long frame, unsigned int flags)
{
if ( flags & GNTMAP_contains_pte )
return destroy_grant_pte_mapping(addr, frame, current->domain);
return destroy_grant_va_mapping(addr, frame, current);
}You can see here that we call the va_mapping instead of the pte mapping. Which means that we once again truncate the top 32 bits of the address, since a va addr is expected to be "unsigned long" in size.
Also, this can be bad, if the va addr is really virtual, and doesn't match the actual offset shift (like handling high memory).
Quoted 3 lines
Signed-off-by Jan Beulich <[email protected]>
please apply.
-- Steve
Quoted 4 lines · Steven Rostedt
It's much worst than that!
Without this patch, my patch is still broken. In the hypervisor this flag is checked, and will go down the wrong path when it is missing:
The hypervisor remembers the flags for itself, so the kernel can only confuse itself. Still, it was a bug.
-- Keir
Quoted 10 lines · Keir Fraser
Quoted 4 lines · Steven Rostedt
It's much worst than that!
Without this patch, my patch is still broken. In the hypervisor this flag is checked, and will go down the wrong path when it is missing:
The hypervisor remembers the flags for itself, so the kernel can only confuse itself. Still, it was a bug.
Well the flag for the application map isn't the problem. But I do see missing the flag for contains pte is a problem.
Quoted 1 line
int create_grant_host_mapping(
u64 addr, unsigned long frame, unsigned int flags)
{
l1_pgentry_t pte = l1e_from_pfn(frame, GRANT_PTE_FLAGS);
if ( (flags & GNTMAP_application_map) )
l1e_add_flags(pte,_PAGE_USER);
if ( !(flags & GNTMAP_readonly) )
l1e_add_flags(pte,_PAGE_RW);
if ( flags & GNTMAP_contains_pte )
return create_grant_pte_mapping(addr, pte, current);
return create_grant_va_mapping(addr, pte, current);
}int destroy_grant_host_mapping(
u64 addr, unsigned long frame, unsigned int flags)
{
if ( flags & GNTMAP_contains_pte )
return destroy_grant_pte_mapping(addr, frame, current->domain);
return destroy_grant_va_mapping(addr, frame, current);
}So is there a difference between create_grant_pte_mapping and create_grant_va_mapping. As well as destroy_grant_pte_mapping and destroy_grant_va_mapping. So calling pte create, and then va destroy on the same mapping is not a bug?
-- Steve
Quoted 4 lines · Steven Rostedt
So is there a difference between
create_grant_pte_mappingandcreate_grant_va_mapping. As well asdestroy_grant_pte_mappinganddestroy_grant_va_mapping. So calling pte create, and then va destroy on the same mapping is not a bug?
That would be a bug, if it were possible, which it's not. 'flags' is not a parameter to the gnttab_unmap operation. Xen remembers the flags from the original map operation.
-- Keir
Quoted 4 lines · Keir Fraser
On 4/11/06 2:25 pm, "Steven Rostedt" <[email protected]> wrote:
Quoted 9 lines
Quoted 4 lines
So is there a difference between
create_grant_pte_mappingandcreate_grant_va_mapping. As well asdestroy_grant_pte_mappinganddestroy_grant_va_mapping. So calling pte create, and then va destroy on the same mapping is not a bug?That would be a bug, if it were possible, which it's not. 'flags' is not a parameter to the
gnttab_unmapoperation. Xen remembers the flags from the original map operation.
OK, took me some time to find what you mean:
__gnttab_unmap_grant_ref(
struct gnttab_unmap_grant_ref *op)
{
[...]
map = &ld->grant_table->maptrack[op->handle];
[...]
dom = map->domid;
ref = map->ref;
flags = map->flags;[...]
if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) )
{
if ( (rc = destroy_grant_host_mapping(op->host_addr,
frame, flags)) < 0 )
goto unmap_out;OK, but it can be a problem on the kernel side because of the Xen auto translate physmap feature. But not as bad as I thought.
But it's still good to be consistent.
-- Steve
Quoted 9 lines
Quoted 1 line
Quoted 1 line
Keir Fraser <[email protected]> 03.11.06 21:43 >>>
Quoted 4 lines · Jan Beulich
Quoted 2 lines
Do you want to do the patch, or would you like me to send another patch that would do this change?
Unless this was already committed, maybe Keir could do this on the fly?
Already committed, but I'll consider patches.
Don't use a 64-bit addr parameter to gnttab_set_{,un}map_op() when the upper 32 bits will never be used (i.e. i386 non-PAE).
include/xen/gnttab.h +2 -2
@@ -118,7 +118,7 @@ int gnttab_suspend(void);int gnttab_resume (void );static inline void -gnttab_set_map_op (struct gnttab_map_grant_ref *map,uint64_t addr, +gnttab_set_map_op (struct gnttab_map_grant_ref *map,maddr_t addr,uint32_t flags,grant_ref_t ref,domid_t domid) {if (flags & GNTMAP_contains_pte)
@@ -134,7 +134,7 @@ gnttab_set_map_op(struct gnttab_map_gran }static inline void -gnttab_set_unmap_op (struct gnttab_unmap_grant_ref *unmap,uint64_t addr, +gnttab_set_unmap_op (struct gnttab_unmap_grant_ref *unmap,maddr_t addr,uint32_t flags,grant_handle_t handle) {if (flags & GNTMAP_contains_pte)