i386 Vmi mmu changes
Subsystems (2)
RFC PATCH 19/24 i386 Vmi mmu changes
MMU code movement. Unfortunately, this one is a little bit more complicated than the rest. We have to override the default accessors that directly write to page table entries. Because of the 2/3-level PAE split in Linux, this turned out to be really ugly at first, but by allowing the sub-arch layer to override the definitions and keeping the native definitions in place, the code becomes much cleaner.
The assembly changes are because pgtable.h is not needed in these files, and now generates bad input to the assembler due to raw C function prototypes.
The change to process.c is somewhat different, but included here because it makes use of the flush_deferred_cpu_state() subarch function, which is used to synchronize all CPU state, including segmentation and page tables. This is required immediately before attempting to reload the %fs and %gs segments.
The most recent addition was the sub-arch map_linear_pg/range functions, which serve the purpose of notifying the hypervisor about remapped page tables. In early VMI initialization, the linearly mapped memory range is communicated to the hypervisor by calling map_linear_range(), which is sufficient for allowing VA->PA conversion during MMU calls in most cases. It is not sufficient if the kernel is compiled with page tables in high memory, since the page tables may then be mapped in at will. In this case, the kernel must notify the hypervisor of these mappings when they change. This is the purpose of the changes to the pte_offset_map functions.
This need more comments.
arch/i386/kernel/efi_stub.S +0 -1
@@ -8,7 +8,6 @@ #include <linux/config.h> #include <linux/linkage.h> #include <asm/page.h> -#include <asm/pgtable.h> /* * efi_call_phys(void *, ...) is a function with variable parameters.
arch/i386/kernel/head.S +0 -1
@@ -13,7 +13,6 @@ #include <linux/linkage.h> #include <asm/segment.h> #include <asm/page.h> -#include <asm/pgtable.h> #include <asm/desc.h> #include <asm/cache.h> #include <asm/thread_info.h>
arch/i386/kernel/process.c +13 -12
@@ -662,18 +662,6 @@ struct task_struct fastcall * __switch_tload_TLS (next, cpu);/* -* Restore %fs and %gs if needed. -* -* Glibc normally makes %fs be zero, and %gs is one of -* the TLS segments. -*/ -if (unlikely (prev->fs | next->fs)) -loadsegment (fs, next->fs); - -if (prev->gs | next->gs) -loadsegment (gs, next->gs); - -/* * Restore IOPL if needed. */ if (unlikely (prev->iopl != next->iopl))
@@ -696,6 +684,19 @@ struct task_struct fastcall * __switch_thandle_io_bitmap (next, tss);disable_tsc (prev_p, next_p); +flush_deferred_cpu_state (); + +/* +* Restore %fs and %gs if needed. +* +* Glibc normally makes %fs be zero, and %gs is one of +* the TLS segments. +*/ +if (unlikely (prev->fs | next->fs)) +loadsegment (fs, next->fs); + +if (prev->gs | next->gs) +loadsegment (gs, next->gs);return prev_p; }
arch/i386/kernel/smp.c +1 -0
@@ -346,6 +346,7 @@ out:static void flush_tlb_others (cpumask_t cpumask,struct mm_struct *mm,unsigned long va) { +flush_deferred_cpu_state ();/* * A couple of (to be removed) sanity checks: *
arch/i386/kernel/setup.c +1 -0
@@ -1555,6 +1555,7 @@ void __init setup_arch(char **cmdline_p)parse_cmdline_early (cmdline_p); max_low_pfn =setup_memory (); +mach_map_linear_range (__PAGE_OFFSET, max_low_pfn,0 );/* * NOTE: before this point _nobody_ is allowed to allocate
arch/i386/mm/fault.c +7 -0
@@ -552,6 +552,13 @@ vmalloc_fault: goto no_context;set_pmd (pmd, *pmd_k); +/* +* Needed. We have just updated this root with a copy of +* the kernel pmd. To return without flushing would +* introduce a fault loop. +*/ +update_mmu_cache (NULL , pmd, pmd_k->pmd); + pte_k =pte_offset_kernel (pmd_k, address);if (!pte_present (*pte_k)) goto no_context;
arch/i386/mm/init.c +2 -1
@@ -69,7 +69,6 @@ static pmd_t * __init one_md_table_init( pud =pud_offset (pgd,0 ); pmd_table =pmd_offset (pud,0 );#endif -return pmd_table; }
@@ -81,6 +80,7 @@ static pte_t * __init one_page_table_ini {if (pmd_none (*pmd)) {pte_t *page_table = (pte_t *)alloc_bootmem_low_pages (PAGE_SIZE ); +mach_setup_pte (__pa (page_table) >>PAGE_SHIFT );set_pmd (pmd,__pmd (__pa (page_table) | _PAGE_TABLE));if (page_table !=pte_offset_kernel (pmd,0 ))BUG ();
@@ -382,6 +382,7 @@ static void __init pagetable_init (void)*/ set_pgd (&pgd_base[0 ], pgd_base[USER_PTRS_PER_PGD ]);#endif + }#ifdef CONFIG_SOFTWARE_SUSPEND
arch/i386/mm/pgtable.c +26 -5
@@ -240,20 +240,37 @@ pgd_t *pgd_alloc(struct mm_struct *mm)int i;pgd_t *pgd =kmem_cache_alloc (pgd_cache,GFP_KERNEL ); -if (PTRS_PER_PMD ==1 || !pgd) +if (!pgd)return pgd; +if (PTRS_PER_PMD ==1 ) { +mach_setup_pgd (__pa (pgd) >>PAGE_SHIFT , +__pa (swapper_pg_dir) >>PAGE_SHIFT , +USER_PTRS_PER_PGD , +PTRS_PER_PGD -USER_PTRS_PER_PGD ); +return pgd; + } + +/* PAE mode will set up the pmds here */ +mach_setup_pgd (__pa (pgd) >>PAGE_SHIFT , +__pa (swapper_pg_dir) >>PAGE_SHIFT , +USER_PTRS_PER_PGD , +PTRS_PER_PGD -USER_PTRS_PER_PGD );for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) {pmd_t *pmd =kmem_cache_alloc (pmd_cache,GFP_KERNEL );if (!pmd) goto out_oom; +mach_setup_pmd (__pa (pmd) >>PAGE_SHIFT );set_pgd (&pgd[i],__pgd (1 +__pa (pmd))); }return pgd; out_oom: -for (i--; i >=0 ; i--) +for (i--; i >=0 ; i--) { +mach_release_pmd (pgd_val (pgd[i]) >>PAGE_SHIFT );kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); + } +mach_release_pgd (__pa (pgd) >>PAGE_SHIFT );kmem_cache_free (pgd_cache, pgd);return NULL ; }
@@ -263,9 +280,13 @@ void pgd_free(pgd_t *pgd)int i;/* in the PAE case user pgd entries are overwritten before usage */ -if (PTRS_PER_PMD >1 ) -for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) +if (PTRS_PER_PMD >1 ) { +for (i =0 ; i <USER_PTRS_PER_PGD ; ++i) { +mach_release_pmd (pgd_val (pgd[i]) >>PAGE_SHIFT );kmem_cache_free (pmd_cache, (void *)__va (pgd_val (pgd[i])-1 )); -/* in the non-PAE case, free_pgtables() clears user pgd entries */ + } + } +/* in the non-PAE case, clear_page_range() clears user pgd entries */ +mach_release_pgd (__pa (pgd) >>PAGE_SHIFT );kmem_cache_free (pgd_cache, pgd); }
Index: linux-2.6.16-rc5/arch/i386/mach-vmi/setup.c
===================================================================
arch/i386/mach-vmi/smpboot_hooks.c +1 -0
@@ -110,6 +110,7 @@ void __init smpboot_pre_start_secondary_ {if (vmi_hypervisor_found ()) { *(unsigned long *) trampoline_base =0xa5a5a5a5 ; +mach_map_linear_range (__PAGE_OFFSET, max_low_pfn,0 ); } }
fs/binfmt_elf.c +1 -0
@@ -951,6 +951,7 @@ static int load_elf_binary(struct linux_sys_close (elf_exec_fileno);set_binfmt (&elf_format); +arch_flush_predirty ();#ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES retval =arch_setup_additional_pages (bprm, executable_stack);
include/asm-i386/pgalloc.h +16 -4
@@ -5,14 +5,22 @@#include <asm/fixmap.h> #include <linux/threads.h> #include <linux/mm.h> /* for struct page */ +#include <mach_pgalloc.h> -#define pmd_populate_kernel (mm, pmd, pte) \ - set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))) +#define pmd_populate_kernel (mm, pmd, pte) \ +do { \ + mach_setup_pte(__pa(pte) >> PAGE_SHIFT); \ + set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))); \ +} while (0)#define pmd_populate (mm, pmd, pte) \ +do { \ + mach_setup_pte(page_to_pfn(pte)); \ set_pmd(pmd, __pmd(_PAGE_TABLE + \ ((unsigned long long)page_to_pfn(pte) << \ - (unsigned long long) PAGE_SHIFT))) + (unsigned long long) PAGE_SHIFT))); \ +} while (0) +/* * Allocate and free page tables. */
@@ -33,7 +41,11 @@ static inline void pte_free(struct page } -#define __pte_free_tlb (tlb,pte) tlb_remove_page((tlb),(pte)) +#define __pte_free_tlb (tlb,pte) \ +do { \ + tlb_remove_page((tlb),(pte)); \ + mach_release_pte(page_to_pfn(pte)); \ +} while (0)#ifdef CONFIG_X86_PAE /*
include/asm-i386/pgtable-2level.h +13 -11
@@ -8,17 +8,6 @@#define pgd_ERROR (e) \ printk("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -/* -* Certain architectures need to do special things when PTEs -* within a page table are directly modified. Thus, the following -* hook is made available. -*/ -#define set_pte (pteptr, pteval) (*(pteptr) = pteval) -#define set_pte_at (mm,addr,ptep,pteval) set_pte(ptep,pteval) -#define set_pte_atomic (pteptr, pteval) set_pte(pteptr,pteval) -#define set_pmd (pmdptr, pmdval) (*(pmdptr) = (pmdval)) - -#define ptep_get_and_clear (mm,addr,xp) __pte(xchg(&(xp)->pte_low, 0))#define pte_same (a, b) ((a).pte_low == (b).pte_low)#define pte_page (x) pfn_to_page(pte_pfn(x))#define pte_none (x) (!(x).pte_low)
@@ -61,4 +50,17 @@ static inline int pte_exec_kernel(pte_t#define __pte_to_swp_entry (pte) ((swp_entry_t) { (pte).pte_low })#define __swp_entry_to_pte (x) ((pte_t) { (x).val }) +/* +* Certain architectures need to do special things when PTEs +* within a page table are directly modified. Thus, the following +* hook is made available. +*/ +#ifndef __HAVE_SUBARCH_PTE_WRITE_FUNCTIONS +#define set_pte (pteptr, pteval) (*(pteptr) = pteval) +#define set_pmd (pmdptr, pmdval) (*(pmdptr) = (pmdval)) +#define ptep_get_and_clear (mm,addr,xp) __pte(xchg(&(xp)->pte_low, 0)) +#endif /* __HAVE_SUBARCH_PTE_WRITE_FUNCTIONS */ + +#define set_pte_atomic (pteptr, pteval) set_pte((pteptr), (pteval)) +#endif /* _I386_PGTABLE_2LEVEL_H */
include/asm-i386/pgtable-3level.h +49 -43
109 lines not on this page · raw message
include/asm-i386/pgtable.h +40 -27
120 lines not on this page · raw message
include/asm-i386/mach-default/mach_pgtable.h +18 -0
@@ -0,0 +1,18 @@ +#ifndef _MACH_PGTABLE_H +#define _MACH_PGTABLE_H + +/* +* The i386 doesn't have any external MMU info: the kernel page +* tables contain all the necessary information. +* +* Also, we only update the dirty/accessed state if we set +* the dirty bit by hand in the kernel, since the hardware +* will do the accessed bit for us, and we don't want to +* race with other CPU's that might be updating the dirty +* bit at the same time. +*/ +#define update_mmu_cache (vma,address,pte) do { } while (0) + +#define arch_flush_predirty () + +#endif /* _PGTABLE_OPS_H */
include/asm-i386/mach-default/mach_pgalloc.h +40 -0
@@ -0,0 +1,40 @@ +/* +* Copyright (C) 2005, VMware, Inc. +* +* All rights reserved. +* +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, but +* WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or +* NON INFRINGEMENT. See the GNU General Public License for more +* details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +* +* Send feedback to [email protected] +* +*/ + +#ifndef _MACH_PGALLOC_H +#define _MACH_PGALLOC_H + +#define mach_setup_pte (pfn) +#define mach_release_pte (pfn) + +#define mach_setup_pmd (pfn) +#define mach_release_pmd (pfn) + +#define mach_setup_pgd (pfn, root, base, pdirs) +#define mach_release_pgd (pfn) + +#define mach_map_linear_pt (num, ptep, pfn) +#define mach_map_linear_range (start, pages, pfn) + +#endif /* _MACH_PGALLOC_H */
include/asm-i386/mach-vmi/pgtable-2level-ops.h +64 -0
64 lines not on this page · raw message
include/asm-i386/mach-vmi/pgtable-3level-ops.h +44 -0
44 lines not on this page · raw message
include/asm-i386/mach-vmi/mach_pgalloc.h +69 -0
69 lines not on this page · raw message
include/asm-i386/mach-vmi/mach_pgtable.h +80 -0
80 lines not on this page · raw message
Quoted 6 lines · Zachary Amsden
MMU code movement. Unfortunately, this one is a little bit more complicated than the rest. We have to override the default accessors that directly write to page table entries. Because of the 2/3-level PAE split in Linux, this turned out to be really ugly at first, but by allowing the sub-arch layer to override the definitions and keeping the native definitions in place, the code becomes much cleaner.
This patch definitely needs to be split up more.
-Andi