Skip to contentSkip to conversation

IA64 various hugepage size - Overview

PATCH In review Zou Nan hai 2 participants 10 messages Apr 14, 2006

Subsystems (9)
lore
Share & tools
lore
#0 lore raw

cover IA64 various hugepage size - Overview

This series of patches add support to have different hugepage sizes for different tasks on IA64.

While still keep the kernel compatible to old applications.

With this patch, application can set it's huge page size via a prctl syscall.

Then different hugepage sized applications can co-work together.

Zou Nan hai

#1 lore raw

Re: [PATCH 1/8] IA64 various hugepage size - Add a variable to mm structure

Add a hugepage_shift member to struct mm to make huge page size a per mm structure.

arch/ia64/mm/hugetlbpage.c +6 -0
diff -Nraup a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
@@ -185,3 +185,9 @@ static int __init hugetlb_setup_sz(char 
 	return 1;
 }
 __setup("hugepagesz=", hugetlb_setup_sz);
+
+void hugepage_size_init(struct mm_struct *mm)
+{
+  mm->hugepage_shift = hpage_shift;
+}
+
arch/ia64/mm/init.c +0 -5
diff -Nraup a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
@@ -408,11 +408,6 @@ ia64_mmu_init (void *my_cpu_data)
 	ia64_set_pta(pta | (0 << 8) | (vmlpt_bits << 2) | VHPT_ENABLE_BIT);
 
 	ia64_tlb_init();
-
-#ifdef	CONFIG_HUGETLB_PAGE
-	ia64_set_rr(HPAGE_REGION_BASE, HPAGE_SHIFT << 2);
-	ia64_srlz_d();
-#endif
 }
 
 #ifdef CONFIG_VIRTUAL_MEM_MAP
include/asm-ia64/mmu_context.h +7 -5
diff -Nraup a/include/asm-ia64/mmu_context.h b/include/asm-ia64/mmu_context.h
@@ -127,13 +127,12 @@ destroy_context (struct mm_struct *mm)
 }
 
 static inline void
-reload_context (nv_mm_context_t context)
+reload_context (struct mm_struct *mm, nv_mm_context_t context)
 {
 	unsigned long rid;
 	unsigned long rid_incr = 0;
-	unsigned long rr0, rr1, rr2, rr3, rr4, old_rr4;
+	unsigned long rr0, rr1, rr2, rr3, rr4;
 
-	old_rr4 = ia64_get_rr(RGN_BASE(RGN_HPAGE));
 	rid = context << 3;	/* make space for encoding the region number */
 	rid_incr = 1 << 8;
 
@@ -144,7 +143,10 @@ reload_context (nv_mm_context_t context)
 	rr3 = rr0 + 3*rid_incr;
 	rr4 = rr0 + 4*rid_incr;
 #ifdef  CONFIG_HUGETLB_PAGE
-	rr4 = (rr4 & (~(0xfcUL))) | (old_rr4 & 0xfc);
+	{
+                unsigned long ps = mm->hugepage_shift << 2;
+                rr4 = ((rr4 & (~(0xfcUL))) | ps);
+        }
 
 #  if RGN_HPAGE != 4
 #    error "reload_context assumes RGN_HPAGE is 4"
@@ -171,7 +173,7 @@ activate_context (struct mm_struct *mm)
 		context = get_mmu_context(mm);
 		if (!cpu_isset(smp_processor_id(), mm->cpu_vm_mask))
 			cpu_set(smp_processor_id(), mm->cpu_vm_mask);
-		reload_context(context);
+		reload_context(mm, context);
 		/*
 		 * in the unlikely event of a TLB-flush by another thread,
 		 * redo the load.
include/asm-ia64/page.h +1 -0
diff -Nraup a/include/asm-ia64/page.h b/include/asm-ia64/page.h
@@ -59,6 +59,7 @@
 # define ARCH_HAS_HUGEPAGE_ONLY_RANGE
 # define ARCH_HAS_PREPARE_HUGEPAGE_RANGE
 # define ARCH_HAS_HUGETLB_FREE_PGD_RANGE
+# define ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
 #endif /* CONFIG_HUGETLB_PAGE */
 
 #ifdef __ASSEMBLY__
include/linux/mm.h +6 -0
diff -Nraup a/include/linux/mm.h b/include/linux/mm.h
@@ -1059,5 +1059,11 @@ void drop_slab(void);
 extern int randomize_va_space;
 #endif
 
+#ifndef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+#define hugepage_size_init(mm)
+#else
+extern void hugepage_size_init(struct mm_struct *mm);
+#endif
+
 #endif /* __KERNEL__ */
 #endif /* _LINUX_MM_H */
include/linux/sched.h +3 -0
diff -Nraup a/include/linux/sched.h b/include/linux/sched.h
@@ -350,6 +350,9 @@ struct mm_struct {
 	/* aio bits */
 	rwlock_t		ioctx_list_lock;
 	struct kioctx		*ioctx_list;
+#ifdef  ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+        unsigned int            hugepage_shift;
+#endif
 };
 
 struct sighand_struct {
kernel/fork.c +1 -0
diff -Nraup a/kernel/fork.c b/kernel/fork.c
@@ -333,6 +333,7 @@ static struct mm_struct * mm_init(struct
 	mm->ioctx_list = NULL;
 	mm->free_area_cache = TASK_UNMAPPED_BASE;
 	mm->cached_hole_size = ~0UL;
+	hugepage_size_init(mm);
 
 	if (likely(!mm_alloc_pgd(mm))) {
 		mm->def_flags = 0;
#2 lore raw

Re: [PATCH 2/8] IA64 various hugepage size - Add the is_valid_hpage_size function

Add a function is_valid_hpage_size

arch/ia64/mm/hugetlbpage.c +16 -14
diff -Nraup a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
@@ -159,28 +159,15 @@ unsigned long hugetlb_get_unmapped_area(
 
 static int __init hugetlb_setup_sz(char *str)
 {
-	u64 tr_pages;
 	unsigned long long size;
 
-	if (ia64_pal_vm_page_size(&tr_pages, NULL) != 0)
-		/*
-		 * shouldn't happen, but just in case.
-		 */
-		tr_pages = 0x15557000UL;
-
 	size = memparse(str, &str);
-	if (*str || (size & (size-1)) || !(tr_pages & size) ||
-		size <= PAGE_SIZE ||
-		size >= (1UL << PAGE_SHIFT << MAX_ORDER)) {
+	if (*str || !is_valid_hpage_size(size)) {
 		printk(KERN_WARNING "Invalid huge page size specified\n");
 		return 1;
 	}
 
 	hpage_shift = __ffs(size);
-	/*
-	 * boot cpu already executed ia64_mmu_init, and has HPAGE_SHIFT_DEFAULT
-	 * override here with new page shift.
-	 */
 	ia64_set_rr(HPAGE_REGION_BASE, hpage_shift << 2);
 	return 1;
 }
@@ -191,3 +178,18 @@ void hugepage_size_init(struct mm_struct
   mm->hugepage_shift = hpage_shift;
 }
 
+int is_valid_hpage_size(unsigned long long size)
+{
+        u64 tr_pages;
+        if (ia64_pal_vm_page_size(&tr_pages, NULL) != 0)
+                /*
+                 * shouldn't happen, but just in case.
+                 */
+                tr_pages = 0x15557000UL;
+        if((size & (size-1)) || !(tr_pages & size) ||
+                size <= PAGE_SIZE ||
+                size >= (1UL << PAGE_SHIFT << MAX_ORDER))
+                return 0;
+        return 1;
+}
+
include/linux/mm.h +5 -0
diff -Nraup a/include/linux/mm.h b/include/linux/mm.h
@@ -1061,8 +1061,13 @@ extern int randomize_va_space;
 
 #ifndef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
 #define hugepage_size_init(mm)
+static inline int is_valid_hpage_size(unsigned long long size)
+{
+	return 1;
+}
 #else
 extern void hugepage_size_init(struct mm_struct *mm);
+extern int is_valid_hpage_size(unsigned long long size);
 #endif
 
 #endif /* __KERNEL__ */
#3 lore raw

Re: [PATCH 3/8] IA64 various hugepage size - Add a mount option to hugetlbfs

Add a mount option page_size= to hugetlbfs.
The s_blocksize_bits in hugetlb super block will indicate the huge page size which attached to this fs.

fs/hugetlbfs/inode.c +22 -3
diff -Nraup a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
@@ -63,6 +63,14 @@ static int hugetlbfs_file_mmap(struct fi
 	loff_t len, vma_len;
 	int ret;
 
+#ifdef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+        struct super_block *sb = inode->i_sb;
+        unsigned long hpage_shift = sb->s_blocksize_bits;
+
+        if (hpage_shift != vma->vm_mm->hugepage_shift)
+                return -EINVAL;
+#endif
+
 	if (vma->vm_pgoff & (HPAGE_SIZE / PAGE_SIZE - 1))
 		return -EINVAL;
 
@@ -632,7 +640,17 @@ hugetlbfs_parse_options(char *options, s
 			size &= HPAGE_MASK;
 			pconfig->nr_blocks = (size >> HPAGE_SHIFT);
 			value = rest;
-		} else if (!strcmp(opt,"nr_inodes")) {
+		}
+#ifdef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+                else if (!strcmp(opt, "page_size")) {
+                        unsigned long long size = memparse(value, &rest);
+                        if (!is_valid_hpage_size(size))
+                                return -EINVAL;
+                        pconfig->page_shift = __ffs(size);
+                        value = rest;
+                }
+#endif
+		else if (!strcmp(opt,"nr_inodes")) {
 			pconfig->nr_inodes = memparse(value, &rest);
 			value = rest;
 		} else
@@ -658,6 +676,7 @@ hugetlbfs_fill_super(struct super_block 
 	config.uid = current->fsuid;
 	config.gid = current->fsgid;
 	config.mode = 0755;
+	config.page_shift = HPAGE_SHIFT;
 	ret = hugetlbfs_parse_options(data, &config);
 
 	if (ret)
@@ -673,8 +692,8 @@ hugetlbfs_fill_super(struct super_block 
 	sbinfo->max_inodes = config.nr_inodes;
 	sbinfo->free_inodes = config.nr_inodes;
 	sb->s_maxbytes = MAX_LFS_FILESIZE;
-	sb->s_blocksize = HPAGE_SIZE;
-	sb->s_blocksize_bits = HPAGE_SHIFT;
+	sb->s_blocksize = 1UL << config.page_shift;
+	sb->s_blocksize_bits = config.page_shift;
 	sb->s_magic = HUGETLBFS_MAGIC;
 	sb->s_op = &hugetlbfs_ops;
 	sb->s_time_gran = 1;
include/linux/hugetlb.h +1 -0
diff -Nraup a/include/linux/hugetlb.h b/include/linux/hugetlb.h
@@ -124,6 +124,7 @@ struct hugetlbfs_config {
 	uid_t   uid;
 	gid_t   gid;
 	umode_t mode;
+	int     page_shift;
 	long	nr_blocks;
 	long	nr_inodes;
 };
#4 lore raw

Re: [PATCH 4/8] IA64 various hugepage size - modify HPAGE related macros

Modify HPAGE related macros to mm related.
Also change some variable in hugetlb.c into array.
Since those variable will not be accessed in hot path. the impact to performance should be limited.

arch/i386/mm/hugetlbpage.c +7 -7
diff -Nraup a/arch/i386/mm/hugetlbpage.c b/arch/i386/mm/hugetlbpage.c
@@ -107,7 +107,7 @@ follow_huge_pmd(struct mm_struct *mm, un
 
 	page = pte_page(*(pte_t *)pmd);
 	if (page)
-		page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT);
+		page += ((address & ~HPAGE_MASK(mm)) >> PAGE_SHIFT);
 	return page;
 }
 #endif
@@ -131,7 +131,7 @@ static unsigned long hugetlb_get_unmappe
 	}
 
 full_search:
-	addr = ALIGN(start_addr, HPAGE_SIZE);
+	addr = ALIGN(start_addr, HPAGE_SIZE(mm));
 
 	for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
 		/* At this point:  (!vma || addr < vma->vm_end). */
@@ -153,7 +153,7 @@ full_search:
 		}
 		if (addr + mm->cached_hole_size < vma->vm_start)
 		        mm->cached_hole_size = vma->vm_start - addr;
-		addr = ALIGN(vma->vm_end, HPAGE_SIZE);
+		addr = ALIGN(vma->vm_end, HPAGE_SIZE(mm));
 	}
 }
 
@@ -181,7 +181,7 @@ try_again:
 		goto fail;
 
 	/* either no address requested or cant fit in requested address hole */
-	addr = (mm->free_area_cache - len) & HPAGE_MASK;
+	addr = (mm->free_area_cache - len) & HPAGE_MASK(mm);
 	do {
 		/*
 		 * Lookup failure means no vma is above this address,
@@ -212,7 +212,7 @@ try_again:
 		        largest_hole = vma->vm_start - addr;
 
 		/* try just below the current vma->vm_start */
-		addr = (vma->vm_start - len) & HPAGE_MASK;
+		addr = (vma->vm_start - len) & HPAGE_MASK(mm);
 	} while (len <= vma->vm_start);
 
 fail:
@@ -253,13 +253,13 @@ hugetlb_get_unmapped_area(struct file *f
 	struct mm_struct *mm = current->mm;
 	struct vm_area_struct *vma;
 
-	if (len & ~HPAGE_MASK)
+	if (len & ~HPAGE_MASK(mm))
 		return -EINVAL;
 	if (len > TASK_SIZE)
 		return -ENOMEM;
 
 	if (addr) {
-		addr = ALIGN(addr, HPAGE_SIZE);
+		addr = ALIGN(addr, HPAGE_SIZE(mm));
 		vma = find_vma(mm, addr);
 		if (TASK_SIZE - len >= addr &&
 		    (!vma || addr + len <= vma->vm_start))
arch/ia64/mm/hugetlbpage.c +18 -17
diff -Nraup a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
@@ -22,12 +22,12 @@
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
 
-unsigned int hpage_shift=HPAGE_SHIFT_DEFAULT;
+unsigned int init_hpage_shift=HPAGE_SHIFT_DEFAULT;
 
 pte_t *
 huge_pte_alloc (struct mm_struct *mm, unsigned long addr)
 {
-	unsigned long taddr = htlbpage_to_page(addr);
+	unsigned long taddr = htlbpage_to_page(mm, addr);
 	pgd_t *pgd;
 	pud_t *pud;
 	pmd_t *pmd;
@@ -46,7 +46,7 @@ huge_pte_alloc (struct mm_struct *mm, un
 pte_t *
 huge_pte_offset (struct mm_struct *mm, unsigned long addr)
 {
-	unsigned long taddr = htlbpage_to_page(addr);
+	unsigned long taddr = htlbpage_to_page(mm, addr);
 	pgd_t *pgd;
 	pud_t *pud;
 	pmd_t *pmd;
@@ -73,9 +73,9 @@ huge_pte_offset (struct mm_struct *mm, u
  */
 int prepare_hugepage_range(unsigned long addr, unsigned long len)
 {
-	if (len & ~HPAGE_MASK)
+	if (len & ~HPAGE_MASK(current->mm))
 		return -EINVAL;
-	if (addr & ~HPAGE_MASK)
+	if (addr & ~HPAGE_MASK(current->mm))
 		return -EINVAL;
 	if (REGION_NUMBER(addr) != RGN_HPAGE)
 		return -EINVAL;
@@ -95,7 +95,7 @@ struct page *follow_huge_addr(struct mm_
 	if (!ptep || pte_none(*ptep))
 		return NULL;
 	page = pte_page(*ptep);
-	page += ((addr & ~HPAGE_MASK) >> PAGE_SHIFT);
+	page += ((addr & ~HPAGE_MASK(mm)) >> PAGE_SHIFT);
 	return page;
 }
 int pmd_huge(pmd_t pmd)
@@ -122,13 +122,14 @@ void hugetlb_free_pgd_range(struct mmu_g
 	 * If floor and ceiling are also in the hugetlb region, they
 	 * must likewise be scaled down; but if outside, left unchanged.
 	 */
+	struct mm_struct *mm = (*tlb)->mm;
 
-	addr = htlbpage_to_page(addr);
-	end  = htlbpage_to_page(end);
+	addr = htlbpage_to_page(mm, addr);
+	end  = htlbpage_to_page(mm, end);
 	if (REGION_NUMBER(floor) = RGN_HPAGE)
-		floor = htlbpage_to_page(floor);
+		floor = htlbpage_to_page(mm, floor);
 	if (REGION_NUMBER(ceiling) = RGN_HPAGE)
-		ceiling = htlbpage_to_page(ceiling);
+		ceiling = htlbpage_to_page(mm, ceiling);
 
 	free_pgd_range(tlb, addr, end, floor, ceiling);
 }
@@ -137,23 +138,24 @@ unsigned long hugetlb_get_unmapped_area(
 		unsigned long pgoff, unsigned long flags)
 {
 	struct vm_area_struct *vmm;
+	struct mm_struct *mm = current->mm;
 
 	if (len > RGN_MAP_LIMIT)
 		return -ENOMEM;
-	if (len & ~HPAGE_MASK)
+	if (len & ~HPAGE_MASK(mm))
 		return -EINVAL;
 	/* This code assumes that RGN_HPAGE != 0. */
-	if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE - 1)))
+	if ((REGION_NUMBER(addr) != RGN_HPAGE) || (addr & (HPAGE_SIZE(mm) - 1)))
 		addr = HPAGE_REGION_BASE;
 	else
-		addr = ALIGN(addr, HPAGE_SIZE);
+		addr = ALIGN(addr, HPAGE_SIZE(mm));
 	for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) {
 		/* At this point:  (!vmm || addr < vmm->vm_end). */
 		if (REGION_OFFSET(addr) + len > RGN_MAP_LIMIT)
 			return -ENOMEM;
 		if (!vmm || (addr + len) <= vmm->vm_start)
 			return addr;
-		addr = ALIGN(vmm->vm_end, HPAGE_SIZE);
+		addr = ALIGN(vmm->vm_end, HPAGE_SIZE(mm));
 	}
 }
 
@@ -167,15 +169,14 @@ static int __init hugetlb_setup_sz(char 
 		return 1;
 	}
 
-	hpage_shift = __ffs(size);
-	ia64_set_rr(HPAGE_REGION_BASE, hpage_shift << 2);
+	init_hpage_shift = __ffs(size);
 	return 1;
 }
 __setup("hugepagesz=", hugetlb_setup_sz);
 
 void hugepage_size_init(struct mm_struct *mm)
 {
-  mm->hugepage_shift = hpage_shift;
+  mm->hugepage_shift = init_hpage_shift;
 }
 
 int is_valid_hpage_size(unsigned long long size)
arch/powerpc/mm/hash_utils_64.c +3 -3
diff -Nraup a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
@@ -94,7 +94,7 @@ int mmu_linear_psize = MMU_PAGE_4K;
 int mmu_virtual_psize = MMU_PAGE_4K;
 #ifdef CONFIG_HUGETLB_PAGE
 int mmu_huge_psize = MMU_PAGE_16M;
-unsigned int HPAGE_SHIFT;
+unsigned int hpage_shift;
 #endif
 
 /* There are definitions of page sizes arrays to be used when none
@@ -337,9 +337,9 @@ static void __init htab_init_page_sizes(
 	/* Calculate HPAGE_SHIFT and sanity check it */
 	if (mmu_psize_defs[mmu_huge_psize].shift > MIN_HUGEPTE_SHIFT &&
 	    mmu_psize_defs[mmu_huge_psize].shift < SID_SHIFT)
-		HPAGE_SHIFT = mmu_psize_defs[mmu_huge_psize].shift;
+		hpage_shift = mmu_psize_defs[mmu_huge_psize].shift;
 	else
-		HPAGE_SHIFT = 0; /* No huge pages dude ! */
+		hpage_shift = 0; /* No huge pages dude ! */
 #endif /* CONFIG_HUGETLB_PAGE */
 }
 
arch/powerpc/mm/hugetlbpage.c +11 -11
diff -Nraup a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
@@ -40,7 +40,7 @@ pte_t *huge_pte_offset(struct mm_struct 
 
 	BUG_ON(! in_hugepage_area(mm->context, addr));
 
-	addr &= HPAGE_MASK;
+	addr &= HPAGE_MASK(mm);
 
 	pg = pgd_offset(mm, addr);
 	if (!pgd_none(*pg)) {
@@ -81,7 +81,7 @@ pte_t *huge_pte_alloc(struct mm_struct *
 
 	BUG_ON(! in_hugepage_area(mm->context, addr));
 
-	addr &= HPAGE_MASK;
+	addr &= HPAGE_MASK(mm);
 
 	pg = pgd_offset(mm, addr);
 	pu = pud_alloc(mm, pg, addr);
@@ -115,7 +115,7 @@ void set_huge_pte_at(struct mm_struct *m
 		 */
 		unsigned long old = pte_update(ptep, ~0UL);
 		if (old & _PAGE_HASHPTE)
-			hpte_update(mm, addr & HPAGE_MASK, ptep, old, 1);
+			hpte_update(mm, addr & HPAGE_MASK(mm), ptep, old, 1);
 		flush_tlb_pending();
 	}
 	*ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
@@ -127,7 +127,7 @@ pte_t huge_ptep_get_and_clear(struct mm_
 	unsigned long old = pte_update(ptep, ~0UL);
 
 	if (old & _PAGE_HASHPTE)
-		hpte_update(mm, addr & HPAGE_MASK, ptep, old, 1);
+		hpte_update(mm, addr & HPAGE_MASK(mm), ptep, old, 1);
 	*ptep = __pte(0);
 
 	return __pte(old);
@@ -331,7 +331,7 @@ follow_huge_addr(struct mm_struct *mm, u
 	ptep = huge_pte_offset(mm, address);
 	page = pte_page(*ptep);
 	if (page)
-		page += (address % HPAGE_SIZE) / PAGE_SIZE;
+		page += (address % HPAGE_SIZE(mm)) / PAGE_SIZE;
 
 	return page;
 }
@@ -562,7 +562,7 @@ static unsigned long htlb_get_low_area(u
 
 		if (!vma || (addr + len) <= vma->vm_start)
 			return addr;
-		addr = ALIGN(vma->vm_end, HPAGE_SIZE);
+		addr = ALIGN(vma->vm_end, HPAGE_SIZE(current->mm));
 		/* Depending on segmask this might not be a confirmed
 		 * hugepage region, so the ALIGN could have skipped
 		 * some VMAs */
@@ -589,7 +589,7 @@ static unsigned long htlb_get_high_area(
 
 		if (!vma || (addr + len) <= vma->vm_start)
 			return addr;
-		addr = ALIGN(vma->vm_end, HPAGE_SIZE);
+		addr = ALIGN(vma->vm_end, HPAGE_SIZE(current->mm));
 		/* Depending on segmask this might not be a confirmed
 		 * hugepage region, so the ALIGN could have skipped
 		 * some VMAs */
@@ -606,7 +606,7 @@ unsigned long hugetlb_get_unmapped_area(
 	int lastshift;
 	u16 areamask, curareas;
 
-	if (HPAGE_SHIFT = 0)
+	if (HPAGE_SHIFT(current->mm) = 0)
 		return -EINVAL;
 	if (len & ~HPAGE_MASK)
 		return -EINVAL;
@@ -700,7 +700,7 @@ static unsigned int hash_huge_page_do_la
 	/* page is dirty */
 	if (!test_bit(PG_arch_1, &page->flags) && !PageReserved(page)) {
 		if (trap = 0x400) {
-			for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++)
+			for (i = 0; i < (HPAGE_SIZE(current->mm) / PAGE_SIZE); i++)
 				__flush_dcache_icache(page_address(page+i));
 			set_bit(PG_arch_1, &page->flags);
 		} else {
@@ -774,7 +774,7 @@ int hash_huge_page(struct mm_struct *mm,
 		/* There MIGHT be an HPTE for this pte */
 		unsigned long hash, slot;
 
-		hash = hpt_hash(va, HPAGE_SHIFT);
+		hash = hpt_hash(va, HPAGE_SHIFT(mm));
 		if (old_pte & _PAGE_F_SECOND)
 			hash = ~hash;
 		slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
@@ -786,7 +786,7 @@ int hash_huge_page(struct mm_struct *mm,
 	}
 
 	if (likely(!(old_pte & _PAGE_HASHPTE))) {
-		unsigned long hash = hpt_hash(va, HPAGE_SHIFT);
+		unsigned long hash = hpt_hash(va, HPAGE_SHIFT(mm));
 		unsigned long hpte_group;
 
 		pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT;
arch/sparc64/kernel/sun4v_tlb_miss.S +1 -1
diff -Nraup a/arch/sparc64/kernel/sun4v_tlb_miss.S b/arch/sparc64/kernel/sun4v_tlb_miss.S
@@ -182,7 +182,7 @@ sun4v_tsb_miss_common:
 	cmp	%g5, -1
 	be,pt	%xcc, 80f
 	 nop
-	COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7)
+	COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT(0), %g2, %g7)
 
 	/* That clobbered %g2, reload it.  */
 	ldxa	[%g0] ASI_SCRATCHPAD, %g2
arch/sparc64/kernel/tsb.S +1 -1
diff -Nraup a/arch/sparc64/kernel/tsb.S b/arch/sparc64/kernel/tsb.S
@@ -76,7 +76,7 @@ tsb_miss_page_table_walk:
 	mov		512, %g7
 	andn		%g5, 0x7, %g5
 	sllx		%g7, %g6, %g7
-	srlx		%g4, HPAGE_SHIFT, %g6
+	srlx		%g4, HPAGE_SHIFT(0), %g6
 	sub		%g7, 1, %g7
 	and		%g6, %g7, %g6
 	sllx		%g6, 4, %g6
arch/sparc64/mm/fault.c +1 -1
diff -Nraup a/arch/sparc64/mm/fault.c b/arch/sparc64/mm/fault.c
@@ -415,7 +415,7 @@ good_area:
 
 	mm_rss = get_mm_rss(mm);
 #ifdef CONFIG_HUGETLB_PAGE
-	mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
+	mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE(mm) / PAGE_SIZE));
 #endif
 	if (unlikely(mm_rss >
 		     mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
arch/sparc64/mm/hugetlbpage.c +9 -9
diff -Nraup a/arch/sparc64/mm/hugetlbpage.c b/arch/sparc64/mm/hugetlbpage.c
@@ -54,7 +54,7 @@ static unsigned long hugetlb_get_unmappe
 	task_size -= len;
 
 full_search:
-	addr = ALIGN(addr, HPAGE_SIZE);
+	addr = ALIGN(addr, HPAGE_SIZE(mm));
 
 	for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
 		/* At this point:  (!vma || addr < vma->vm_end). */
@@ -81,7 +81,7 @@ full_search:
 		if (addr + mm->cached_hole_size < vma->vm_start)
 		        mm->cached_hole_size = vma->vm_start - addr;
 
-		addr = ALIGN(vma->vm_end, HPAGE_SIZE);
+		addr = ALIGN(vma->vm_end, HPAGE_SIZE(mm));
 	}
 }
 
@@ -105,7 +105,7 @@ hugetlb_get_unmapped_area_topdown(struct
  	}
 
 	/* either no address requested or can't fit in requested address hole */
-	addr = mm->free_area_cache & HPAGE_MASK;
+	addr = mm->free_area_cache & HPAGE_MASK(mm);
 
 	/* make sure it can fit in the remaining address space */
 	if (likely(addr > len)) {
@@ -119,7 +119,7 @@ hugetlb_get_unmapped_area_topdown(struct
 	if (unlikely(mm->mmap_base < len))
 		goto bottomup;
 
-	addr = (mm->mmap_base-len) & HPAGE_MASK;
+	addr = (mm->mmap_base-len) & HPAGE_MASK(mm);
 
 	do {
 		/*
@@ -138,7 +138,7 @@ hugetlb_get_unmapped_area_topdown(struct
  		        mm->cached_hole_size = vma->vm_start - addr;
 
 		/* try just below the current vma->vm_start */
-		addr = (vma->vm_start-len) & HPAGE_MASK;
+		addr = (vma->vm_start-len) & HPAGE_MASK(mm);
 	} while (likely(len < vma->vm_start));
 
 bottomup:
@@ -171,13 +171,13 @@ hugetlb_get_unmapped_area(struct file *f
 	if (test_thread_flag(TIF_32BIT))
 		task_size = STACK_TOP32;
 
-	if (len & ~HPAGE_MASK)
+	if (len & ~HPAGE_MASK(mm))
 		return -EINVAL;
 	if (len > task_size)
 		return -ENOMEM;
 
 	if (addr) {
-		addr = ALIGN(addr, HPAGE_SIZE);
+		addr = ALIGN(addr, HPAGE_SIZE(mm));
 		vma = find_vma(mm, addr);
 		if (task_size - len >= addr &&
 		    (!vma || addr + len <= vma->vm_start))
@@ -203,7 +203,7 @@ pte_t *huge_pte_alloc(struct mm_struct *
 	 * all of the sub-ptes for the hugepage range.  So we have
 	 * to give it the first such sub-pte.
 	 */
-	addr &= HPAGE_MASK;
+	addr &= HPAGE_MASK(mm);
 
 	pgd = pgd_offset(mm, addr);
 	pud = pud_alloc(mm, pgd, addr);
@@ -222,7 +222,7 @@ pte_t *huge_pte_offset(struct mm_struct 
 	pmd_t *pmd;
 	pte_t *pte = NULL;
 
-	addr &= HPAGE_MASK;
+	addr &= HPAGE_MASK(mm);
 
 	pgd = pgd_offset(mm, addr);
 	if (!pgd_none(*pgd)) {
arch/sparc64/mm/init.c +1 -1
diff -Nraup a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
@@ -325,7 +325,7 @@ void update_mmu_cache(struct vm_area_str
 		    (tlb_type != hypervisor &&
 		     (pte_val(pte) & _PAGE_SZALL_4U) = _PAGE_SZHUGE_4U)) {
 			tsb_index = MM_TSB_HUGE;
-			tsb_hash_shift = HPAGE_SHIFT;
+			tsb_hash_shift = HPAGE_SHIFT(mm);
 		}
 	}
 #endif
fs/hugetlbfs/inode.c +33 -28

191 lines not on this page · raw message

include/asm-i386/page.h +10 -5
diff -Nraup a/include/asm-i386/page.h b/include/asm-i386/page.h
@@ -52,21 +52,26 @@ typedef struct { unsigned long long pgpr
 #define pmd_val(x)	((x).pmd)
 #define pte_val(x)	((x).pte_low | ((unsigned long long)(x).pte_high << 32))
 #define __pmd(x) ((pmd_t) { (x) } )
-#define HPAGE_SHIFT	21
+#define HPAGE_SHIFT(mm)	21
 #else
 typedef struct { unsigned long pte_low; } pte_t;
 typedef struct { unsigned long pgd; } pgd_t;
 typedef struct { unsigned long pgprot; } pgprot_t;
 #define boot_pte_t pte_t /* or would you rather have a typedef */
 #define pte_val(x)	((x).pte_low)
-#define HPAGE_SHIFT	22
+#define HPAGE_SHIFT(mm)	22
 #endif
 #define PTE_MASK	PAGE_MASK
 
 #ifdef CONFIG_HUGETLB_PAGE
-#define HPAGE_SIZE	((1UL) << HPAGE_SHIFT)
-#define HPAGE_MASK	(~(HPAGE_SIZE - 1))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
+#define HPAGE_SIZE(mm)	((1UL) << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)	(~(HPAGE_SIZE(mm) - 1))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
+
 #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
 #endif
 
include/asm-ia64/page.h +12 -7
diff -Nraup a/include/asm-ia64/page.h b/include/asm-ia64/page.h
@@ -50,10 +50,11 @@
 
 #ifdef CONFIG_HUGETLB_PAGE
 # define HPAGE_REGION_BASE	RGN_BASE(RGN_HPAGE)
-# define HPAGE_SHIFT		hpage_shift
+# define HPAGE_SHIFT(mm)	(mm->hugepage_shift)
 # define HPAGE_SHIFT_DEFAULT	28	/* check ia64 SDM for architecture supported size */
-# define HPAGE_SIZE		(__IA64_UL_CONST(1) << HPAGE_SHIFT)
-# define HPAGE_MASK		(~(HPAGE_SIZE - 1))
+# define HPAGE_SIZE(mm)		(__IA64_UL_CONST(1) << HPAGE_SHIFT(mm))
+# define HPAGE_MASK(mm)		(~(HPAGE_SIZE(mm) - 1))
+# define HUGETLB_PAGE_ORDER(mm) ((mm->hugepage_shift) - PAGE_SHIFT)
 
 # define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
 # define ARCH_HAS_HUGEPAGE_ONLY_RANGE
@@ -154,13 +155,17 @@ typedef union ia64_va {
 #define REGION_OFFSET(x)	({ia64_va _v; _v.l = (long) (x); _v.f.off;})
 
 #ifdef CONFIG_HUGETLB_PAGE
-# define htlbpage_to_page(x)	(((unsigned long) REGION_NUMBER(x) << 61)			\
-				 | (REGION_OFFSET(x) >> (HPAGE_SHIFT-PAGE_SHIFT)))
-# define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
+# define htlbpage_to_page(mm,x)	(((unsigned long) REGION_NUMBER(x) << 61)			\
+				 | (REGION_OFFSET(x) >> (mm->hugepage_shift-PAGE_SHIFT)))
+# define HUGETLB_INIT_PAGE_SHIFT (init_hpage_shift)
+# define HUGETLB_INIT_PAGE_ORDER (init_hpage_shift - PAGE_SHIFT)
+# define HUGETLB_INIT_PAGE_SIZE  (__IA64_UL_CONST(1)<<init_hpage_shift)
+# define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
+
 # define is_hugepage_only_range(mm, addr, len)		\
 	 (REGION_NUMBER(addr) = RGN_HPAGE ||	\
 	  REGION_NUMBER((addr)+(len)-1) = RGN_HPAGE)
-extern unsigned int hpage_shift;
+extern unsigned int init_hpage_shift;
 #endif
 
 static __inline__ int
include/asm-parisc/page.h +9 -4
diff -Nraup a/include/asm-parisc/page.h b/include/asm-parisc/page.h
@@ -158,10 +158,15 @@ extern int npmem_ranges;
 #endif /* CONFIG_DISCONTIGMEM */
 
 #ifdef CONFIG_HUGETLB_PAGE
-#define HPAGE_SHIFT		22	/* 4MB (is this fixed?) */
-#define HPAGE_SIZE      	((1UL) << HPAGE_SHIFT)
-#define HPAGE_MASK		(~(HPAGE_SIZE - 1))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
+#define HPAGE_SHIFT(mm)		22	/* 4MB (is this fixed?) */
+#define HPAGE_SIZE(mm)      	((1UL) << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)		(~(HPAGE_SIZE(mm) - 1))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
 #endif
 
 #define virt_addr_valid(kaddr)	pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
include/asm-powerpc/page_64.h +10 -6
diff -Nraup a/include/asm-powerpc/page_64.h b/include/asm-powerpc/page_64.h
@@ -78,14 +78,18 @@ extern u64 ppc64_pft_size;
 
 /* Large pages size */
 #ifdef CONFIG_HUGETLB_PAGE
-extern unsigned int HPAGE_SHIFT;
+#define HPAGE_SHIFT(mm) hpage_shift;
 #else
-#define HPAGE_SHIFT PAGE_SHIFT
+#define HPAGE_SHIFT(mm) PAGE_SHIFT
 #endif
-#define HPAGE_SIZE		((1UL) << HPAGE_SHIFT)
-#define HPAGE_MASK		(~(HPAGE_SIZE - 1))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
-
+#define HPAGE_SIZE(mm)		((1UL) << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)		(~(HPAGE_SIZE(mm) - 1))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
 #endif /* __ASSEMBLY__ */
 
 #ifdef CONFIG_HUGETLB_PAGE
include/asm-sh/page.h +10 -5
diff -Nraup a/include/asm-sh/page.h b/include/asm-sh/page.h
@@ -22,15 +22,20 @@
 #define PTE_MASK	PAGE_MASK
 
 #if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
-#define HPAGE_SHIFT	16
+#define HPAGE_SHIFT(mm)	16
 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_1MB)
-#define HPAGE_SHIFT	20
+#define HPAGE_SHIFT(mm)	20
 #endif
 
 #ifdef CONFIG_HUGETLB_PAGE
-#define HPAGE_SIZE		(1UL << HPAGE_SHIFT)
-#define HPAGE_MASK		(~(HPAGE_SIZE-1))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT-PAGE_SHIFT)
+#define HPAGE_SIZE(mm)		(1UL << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)		(~(HPAGE_SIZE(mm)-1))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
 #define ARCH_HAS_SETCLEAR_HUGE_PTE
 #endif
 
include/asm-sh64/page.h +11 -6
diff -Nraup a/include/asm-sh64/page.h b/include/asm-sh64/page.h
@@ -30,17 +30,22 @@
 #define PTE_MASK	PAGE_MASK
 
 #if defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
-#define HPAGE_SHIFT	16
+#define HPAGE_SHIFT(mm)	16
 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_1MB)
-#define HPAGE_SHIFT	20
+#define HPAGE_SHIFT(mm)	20
 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_512MB)
-#define HPAGE_SHIFT	29
+#define HPAGE_SHIFT(mm)	29
 #endif
 
 #ifdef CONFIG_HUGETLB_PAGE
-#define HPAGE_SIZE		(1UL << HPAGE_SHIFT)
-#define HPAGE_MASK		(~(HPAGE_SIZE-1))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT-PAGE_SHIFT)
+#define HPAGE_SIZE(mm)		(1UL << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)		(~(HPAGE_SIZE(mm)-1))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
 #define ARCH_HAS_SETCLEAR_HUGE_PTE
 #endif
 
include/asm-sparc64/page.h +12 -6
diff -Nraup a/include/asm-sparc64/page.h b/include/asm-sparc64/page.h
@@ -31,17 +31,23 @@
 #ifdef __KERNEL__
 
 #if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
-#define HPAGE_SHIFT		22
+#define HPAGE_SHIFT(mm)		22
 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K)
-#define HPAGE_SHIFT		19
+#define HPAGE_SHIFT(mm)		19
 #elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K)
-#define HPAGE_SHIFT		16
+#define HPAGE_SHIFT(mm)		16
 #endif
 
 #ifdef CONFIG_HUGETLB_PAGE
-#define HPAGE_SIZE		(_AC(1,UL) << HPAGE_SHIFT)
-#define HPAGE_MASK		(~(HPAGE_SIZE - 1UL))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
+#define HPAGE_SIZE(mm)		(_AC(1,UL) << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)		(~(HPAGE_SIZE(mm) - 1UL))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
+
 #define ARCH_HAS_SETCLEAR_HUGE_PTE
 #define ARCH_HAS_HUGETLB_PREFAULT_HOOK
 #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
include/asm-x86_64/page.h +9 -4
diff -Nraup a/include/asm-x86_64/page.h b/include/asm-x86_64/page.h
@@ -36,10 +36,15 @@
 #define LARGE_PAGE_MASK (~(LARGE_PAGE_SIZE-1))
 #define LARGE_PAGE_SIZE (1UL << PMD_SHIFT)
 
-#define HPAGE_SHIFT PMD_SHIFT
-#define HPAGE_SIZE	((1UL) << HPAGE_SHIFT)
-#define HPAGE_MASK	(~(HPAGE_SIZE - 1))
-#define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
+#define HPAGE_SHIFT(mm) PMD_SHIFT
+#define HPAGE_SIZE(mm)  ((1UL) << HPAGE_SHIFT(mm))
+#define HPAGE_MASK(mm)  (~(HPAGE_SIZE(mm) - 1))
+#define HUGETLB_PAGE_ORDER(mm) (HPAGE_SHIFT(mm) - PAGE_SHIFT)
+
+#define HUGETLB_INIT_PAGE_SHIFT (HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_ORDER (HPAGE_SHIFT(0) - PAGE_SHIFT)
+#define HUGETLB_INIT_PAGE_SIZE  (1UL<<HPAGE_SHIFT(0))
+#define HUGETLB_INIT_PAGE_MASK  (~(HUGETLB_INIT_PAGE_SIZE - 1))
 
 #ifdef __KERNEL__
 #ifndef __ASSEMBLY__
include/linux/hugetlb.h +9 -9
diff -Nraup a/include/linux/hugetlb.h b/include/linux/hugetlb.h
@@ -59,9 +59,9 @@ void hugetlb_free_pgd_range(struct mmu_g
  */
 static inline int prepare_hugepage_range(unsigned long addr, unsigned long len)
 {
-	if (len & ~HPAGE_MASK)
+	if (len & ~HPAGE_MASK(current->mm))
 		return -EINVAL;
-	if (addr & ~HPAGE_MASK)
+	if (addr & ~HPAGE_MASK(current->mm))
 		return -EINVAL;
 	return 0;
 }
@@ -113,8 +113,8 @@ static inline unsigned long hugetlb_tota
 #define hugetlb_change_protection(vma, address, end, newprot)
 
 #ifndef HPAGE_MASK
-#define HPAGE_MASK	PAGE_MASK		/* Keep the compiler happy */
-#define HPAGE_SIZE	PAGE_SIZE
+#define HPAGE_MASK(mm)	PAGE_MASK		/* Keep the compiler happy */
+#define HPAGE_SIZE(mm)	PAGE_SIZE
 #endif
 
 #endif /* !CONFIG_HUGETLB_PAGE */
@@ -124,7 +124,7 @@ struct hugetlbfs_config {
 	uid_t   uid;
 	gid_t   gid;
 	umode_t mode;
-	int     page_shift;
+	long	page_shift;
 	long	nr_blocks;
 	long	nr_inodes;
 };
@@ -157,11 +157,11 @@ static inline struct hugetlbfs_sb_info *
 
 extern const struct file_operations hugetlbfs_file_operations;
 extern struct vm_operations_struct hugetlb_vm_ops;
-struct file *hugetlb_zero_setup(size_t);
+struct file *hugetlb_zero_setup(size_t, int);
 int hugetlb_extend_reservation(struct hugetlbfs_inode_info *info,
-			       unsigned long atleast_hpages);
+			       unsigned long atleast_hpages, int order);
 void hugetlb_truncate_reservation(struct hugetlbfs_inode_info *info,
-				  unsigned long atmost_hpages);
+				  unsigned long atmost_hpages, int order);
 int hugetlb_get_quota(struct address_space *mapping);
 void hugetlb_put_quota(struct address_space *mapping);
 
@@ -178,7 +178,7 @@ static inline void set_file_hugepages(st
 
 #define is_file_hugepages(file)		0
 #define set_file_hugepages(file)	BUG()
-#define hugetlb_zero_setup(size)	ERR_PTR(-ENOSYS)
+#define hugetlb_zero_setup(size, int)	ERR_PTR(-ENOSYS)
 
 #endif /* !CONFIG_HUGETLBFS */
 
include/linux/mm.h +1 -1
diff -Nraup a/include/linux/mm.h b/include/linux/mm.h
@@ -1063,7 +1063,7 @@ extern int randomize_va_space;
 #define hugepage_size_init(mm)
 static inline int is_valid_hpage_size(unsigned long long size)
 {
-	return 1;
+	return (size = HPAGE_SIZE(0));
 }
 #else
 extern void hugepage_size_init(struct mm_struct *mm);
ipc/shm.c +4 -2
diff -Nraup a/ipc/shm.c b/ipc/shm.c
@@ -217,7 +217,8 @@ static int newseg (key_t key, int shmflg
 
 	if (shmflg & SHM_HUGETLB) {
 		/* hugetlb_zero_setup takes care of mlock user accounting */
-		file = hugetlb_zero_setup(size);
+		file = hugetlb_zero_setup(size,
+			HUGETLB_PAGE_ORDER(current->mm));
 		shp->mlock_user = current->user;
 	} else {
 		int acctflag = VM_ACCOUNT;
@@ -411,7 +412,8 @@ static void shm_get_stat(unsigned long *
 
 		if (is_file_hugepages(shp->shm_file)) {
 			struct address_space *mapping = inode->i_mapping;
-			*rss += (HPAGE_SIZE/PAGE_SIZE)*mapping->nrpages;
+			struct super_block *sb = inode->i_sb;
+			*rss += (sb->s_blocksize/PAGE_SIZE)*mapping->nrpages;
 		} else {
 			struct shmem_inode_info *info = SHMEM_I(inode);
 			spin_lock(&info->lock);
mm/hugetlb.c +104 -92

476 lines not on this page · raw message

mm/memory.c +1 -1
diff -Nraup a/mm/memory.c b/mm/memory.c
@@ -824,7 +824,7 @@ unsigned long unmap_vmas(struct mmu_gath
 			if (unlikely(is_vm_hugetlb_page(vma))) {
 				unmap_hugepage_range(vma, start, end);
 				zap_work -= (end - start) /
-						(HPAGE_SIZE / PAGE_SIZE);
+					(HPAGE_SIZE(vma->vm_mm) / PAGE_SIZE);
 				start = end;
 			} else
 				start = unmap_page_range(*tlbp, vma,
mm/mempolicy.c +3 -3
diff -Nraup a/mm/mempolicy.c b/mm/mempolicy.c
@@ -1170,7 +1170,7 @@ struct zonelist *huge_zonelist(struct vm
 	if (pol->policy = MPOL_INTERLEAVE) {
 		unsigned nid;
 
-		nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
+		nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT(vma->vm_mm));
 		return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER);
 	}
 	return zonelist_policy(GFP_HIGHUSER, pol);
@@ -1772,8 +1772,8 @@ static void check_huge_range(struct vm_a
 	unsigned long addr;
 	struct page *page;
 
-	for (addr = start; addr < end; addr += HPAGE_SIZE) {
-		pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
+	for (addr = start; addr < end; addr += HPAGE_SIZE(vma->vm_mm)) {
+		pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK(vma->vm_mm));
 		pte_t pte;
 
 		if (!ptep)
mm/mmap.c +1 -1
diff -Nraup a/mm/mmap.c b/mm/mmap.c
@@ -1693,7 +1693,7 @@ int split_vma(struct mm_struct * mm, str
 	struct mempolicy *pol;
 	struct vm_area_struct *new;
 
-	if (is_vm_hugetlb_page(vma) && (addr & ~HPAGE_MASK))
+	if (is_vm_hugetlb_page(vma) && (addr & ~HPAGE_MASK(mm)))
 		return -EINVAL;
 
 	if (mm->map_count >= sysctl_max_map_count)
#5 lore raw

Re: [PATCH 5/8] IA64 various hugepage size - mount more hugetlb fs for SHM

Mount more hugetlbfs for SHM to support various hugepage size in SHM.

fs/hugetlbfs/inode.c +36 -14
diff -Nraup a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
@@ -759,7 +759,7 @@ static struct file_system_type hugetlbfs
 	.kill_sb	= kill_litter_super,
 };
 
-static struct vfsmount *hugetlbfs_vfsmount;
+static struct vfsmount *hugetlbfs_vfsmount[MAX_ORDER];
 
 static int can_do_hugetlb_shm(void)
 {
@@ -784,7 +784,7 @@ struct file *hugetlb_zero_setup(size_t s
 	if (!user_shm_lock(size, current->user))
 		return ERR_PTR(-ENOMEM);
 
-	root = hugetlbfs_vfsmount->mnt_root;
+	root = hugetlbfs_vfsmount[order]->mnt_root;
 	snprintf(buf, 16, "%u", atomic_inc_return(&counter));
 	quick_string.name = buf;
 	quick_string.len = strlen(quick_string.name);
@@ -812,7 +812,7 @@ struct file *hugetlb_zero_setup(size_t s
 	d_instantiate(dentry, inode);
 	inode->i_size = size;
 	inode->i_nlink = 0;
-	file->f_vfsmnt = mntget(hugetlbfs_vfsmount);
+	file->f_vfsmnt = mntget(hugetlbfs_vfsmount[order]);
 	file->f_dentry = dentry;
 	file->f_mapping = inode->i_mapping;
 	file->f_op = &hugetlbfs_file_operations;
@@ -834,27 +834,49 @@ static int __init init_hugetlbfs_fs(void
 {
 	int error;
 	struct vfsmount *vfsmount;
-
 	hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
-					sizeof(struct hugetlbfs_inode_info),
-					0, 0, init_once, NULL);
+			sizeof(struct hugetlbfs_inode_info),
+			0, 0, init_once, NULL);
 	if (hugetlbfs_inode_cachep = NULL)
 		return -ENOMEM;
 
 	error = register_filesystem(&hugetlbfs_fs_type);
 	if (error)
 		goto out;
-
-	vfsmount = kern_mount(&hugetlbfs_fs_type);
-
-	if (!IS_ERR(vfsmount)) {
-		hugetlbfs_vfsmount = vfsmount;
-		return 0;
+#ifdef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+	{
+		int order;
+		char fsname[64], data[64];
+		for (order = 0; order < MAX_ORDER; order++) {
+			if (is_valid_hpage_size(1UL<<(PAGE_SHIFT+order))) {
+				sprintf(fsname, "%s%d", hugetlbfs_fs_type.name, order);
+				sprintf(data, "page_size=%ld", (1UL<<(PAGE_SHIFT+order)));
+				vfsmount = do_kern_mount(hugetlbfs_fs_type.name, 0,
+						fsname, data);
+
+				if (!IS_ERR(vfsmount))
+					hugetlbfs_vfsmount[order] = vfsmount;
+				else {
+					error = PTR_ERR(vfsmount);
+					goto out;
+				}
+			}
+		}
 	}
+#else
+        vfsmount = kern_mount(&hugetlbfs_fs_type);
 
-	error = PTR_ERR(vfsmount);
+        if (!IS_ERR(vfsmount)) {
+                hugetlbfs_vfsmount[HUGETLB_INIT_PAGE_ORDER] = vfsmount;
+                return 0;
+        }
+
+        error = PTR_ERR(vfsmount);
+        goto out;
+#endif
+	return 0;
 
- out:
+out:
 	if (error)
 		kmem_cache_destroy(hugetlbfs_inode_cachep);
 	return error;
#6 lore raw

Re: [PATCH 6/8] IA64 various hugepage size - introduce prctl options to set/get hugepage size

Introduce 2 prctl option to set/get hugepage size.

arch/ia64/mm/hugetlbpage.c +22 -0
diff -Nraup a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
@@ -194,3 +194,25 @@ int is_valid_hpage_size(unsigned long lo
         return 1;
 }
 
+int set_hugepagesize(struct task_struct *task,unsigned long size)
+{
+	struct mm_struct *mm = task->mm;
+	struct vm_area_struct *vma;
+	unsigned int shift;
+	if (!is_valid_hpage_size(size))
+		return -EINVAL;
+	shift = __ffs(size);
+	if (shift = mm->hugepage_shift)
+		return 0;
+	down_write(&mm->mmap_sem);
+	for (vma = mm->mmap; vma ; vma = vma->vm_next) {
+		if (vma->vm_flags & VM_HUGETLB) {
+			up_write(&mm->mmap_sem);
+			return -EPERM;
+		}
+	}
+	mm->hugepage_shift = shift;
+	activate_mm(mm, mm);
+	up_write(&mm->mmap_sem);
+	return 0;
+}
include/asm-ia64/page.h +2 -0
diff -Nraup a/include/asm-ia64/page.h b/include/asm-ia64/page.h
@@ -166,6 +166,8 @@ typedef union ia64_va {
 	 (REGION_NUMBER(addr) = RGN_HPAGE ||	\
 	  REGION_NUMBER((addr)+(len)-1) = RGN_HPAGE)
 extern unsigned int init_hpage_shift;
+struct task_struct;
+extern int set_hugepagesize(struct task_struct *,unsigned long);
 #endif
 
 static __inline__ int
include/asm-ia64/processor.h +8 -0
diff -Nraup a/include/asm-ia64/processor.h b/include/asm-ia64/processor.h
@@ -211,6 +211,14 @@ typedef struct {
 		 (int __user *) (addr));							\
 })
 
+#ifdef CONFIG_HUGETLB_PAGE
+#define	GET_HUGEPAGESIZE(task,addr) \
+({	put_user((1UL<<(task)->mm->hugepage_shift), \
+		(unsigned long __user *)(addr)); \
+})
+#define SET_HUGEPAGESIZE(task,size) set_hugepagesize(task,size)
+#endif
+
 #ifdef CONFIG_IA32_SUPPORT
 struct desc_struct {
 	unsigned int a, b;
include/linux/prctl.h +4 -0
diff -Nraup a/include/linux/prctl.h b/include/linux/prctl.h
@@ -52,4 +52,8 @@
 #define PR_SET_NAME    15		/* Set process name */
 #define PR_GET_NAME    16		/* Get process name */
 
+/* Get/set task huge page size (if meaningful) */
+#define PR_SET_HUGEPAGE_SIZE	17
+#define PR_GET_HUGEPAGE_SIZE	18
+
 #endif /* _LINUX_PRCTL_H */
kernel/sys.c +12 -0
diff -Nraup a/kernel/sys.c b/kernel/sys.c
@@ -57,6 +57,12 @@
 #ifndef GET_FPEXC_CTL
 # define GET_FPEXC_CTL(a,b)	(-EINVAL)
 #endif
+#ifndef GET_HUGEPAGESIZE
+# define GET_HUGEPAGESIZE(a,b)  (-EINVAL)
+#endif
+#ifndef SET_HUGEPAGESIZE
+# define SET_HUGEPAGESIZE(a,b)  (-EINVAL)
+#endif
 
 /*
  * this is where the system-wide overflow UID and GID are defined, for
@@ -2057,6 +2063,12 @@ asmlinkage long sys_prctl(int option, un
 				return -EFAULT;
 			return 0;
 		}
+		case PR_SET_HUGEPAGE_SIZE:
+			error = SET_HUGEPAGESIZE(current, arg2);
+			break;
+		case PR_GET_HUGEPAGE_SIZE:
+			error = GET_HUGEPAGESIZE(current, arg2);
+			break;
 		default:
 			error = -EINVAL;
 			break;
#7 lore raw

Re: [PATCH 7/8] IA64 various hugepage size - Add proc control to reserve and free

Add proc control to reserve and free various sized hugepages

cat /proc/hugetlb/hugepages will show huge pages information

echo "size nr" > /proc/hugetlb/hugepages will modify the page number for size.

e.g
echo "64k 1000" > /proc/hugetlb/hugepages will allocate 1000 pages of 64k page.

echo "64k 0" > /proc/hugetlb/hugepages will free all the 64k pages

mm/hugetlb.c +60 -32
diff -Nraup a/mm/hugetlb.c b/mm/hugetlb.c
@@ -15,6 +15,9 @@
 #include <linux/cpuset.h>
 #include <linux/mutex.h>
 
+#include <linux/proc_fs.h>
+#include <linux/seq_file.h>
+
 #include <asm/page.h>
 #include <asm/pgtable.h>
 
@@ -259,38 +262,6 @@ void hugetlb_truncate_reservation(struct
 	spin_unlock(&hugetlb_lock);
 }
 
-static int __init hugetlb_init(void)
-{
-	unsigned long i, j;
-
-#ifndef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
-	if (HPAGE_SHIFT(0) = 0)
-		return 0;
-#endif
-
-	for (i = 0; i < MAX_NUMNODES; ++i)
-		for (j = 0; j < MAX_ORDER; ++j)
-			INIT_LIST_HEAD(&hugepage_freelists[i][j]);
-
-	for (i = 0; i < max_huge_pages; ++i) {
-		if (!alloc_fresh_huge_page(HUGETLB_INIT_PAGE_ORDER))
-			break;
-	}
-	max_huge_pages = free_huge_pages[HUGETLB_INIT_PAGE_ORDER] nr_huge_pages[HUGETLB_INIT_PAGE_ORDER] = i;
-	printk("Total HugeTLB memory allocated, %ld\n", 
-		free_huge_pages[HUGETLB_INIT_PAGE_ORDER]);
-	return 0;
-}
-module_init(hugetlb_init);
-
-static int __init hugetlb_setup(char *s)
-{
-	if (sscanf(s, "%lu", &max_huge_pages) <= 0)
-		max_huge_pages = 0;
-	return 1;
-}
-__setup("hugepages=", hugetlb_setup);
-
 #ifdef CONFIG_SYSCTL
 static void update_and_free_page(struct page *page, int order)
 {
@@ -364,8 +335,120 @@ int hugetlb_sysctl_handler(struct ctl_ta
 		HUGETLB_INIT_PAGE_ORDER);
 	return 0;
 }
+#ifdef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+static char *
+bitvector_process(char *p, unsigned long vector)
+{
+	int i,j;
+	const char *units[]={ "", "K", "M", "G", "T" };
+
+	for (i=0, j=0; i < 64; i++ , j=i/10) {
+		if (vector & 0x1) {
+			p += sprintf(p, "%d%s ", 1 << (i-j*10), units[j]);
+		}
+		vector >>= 1;
+	}
+	return p;
+}
+
+static int proc_hugetlb_pagesizes_show(struct seq_file *s, void *p)
+{
+	int order;
+	char buf[32];
+	for (order = 0; order < MAX_ORDER; order++) {
+		if (!is_valid_hpage_size(1UL<<(order+PAGE_SHIFT)))
+			continue;
+		bitvector_process(buf, (1UL<<(order+PAGE_SHIFT)));
+		seq_printf(s, "%s\t%ld\t%ld\t%ld\n",
+				buf, nr_huge_pages[order],
+				free_huge_pages[order],
+				reserved_huge_pages[order]);
+	}
+
+	return 0;
+}
+
+static ssize_t proc_hugetlb_pagesizes_write(struct file *file,
+	const char __user *buffer, size_t length, loff_t *ppos)
+{
+	char buf[64];
+	unsigned long nr, order;
+	unsigned long long size;
+	char *rest;
+	if (!buffer || length >= sizeof(buf))
+		return -EINVAL;
+	if (copy_from_user(buf, buffer, length))
+		return -EFAULT;
+	buf[length] = 0;
+
+	size = memparse(buf, &rest);
+	order = __ffs(size) - PAGE_SHIFT;
+	if (order >= MAX_ORDER || !is_valid_hpage_size(size))
+		return -EINVAL;
+	if (sscanf(rest, "%ld", &nr) != 1)
+		return -EINVAL;
+	set_max_huge_pages(nr, order);
+	return length;
+}
+
+static int proc_hugetlb_pagesizes_open(struct inode *inode, struct file

*file)
+{
+ return single_open(file, proc_hugetlb_pagesizes_show, NULL);
+}
+
+static struct file_operations proc_hugetlb_pagesizes_operations = {

+        .open           = proc_hugetlb_pagesizes_open,
+        .read           = seq_read,
+        .write          = proc_hugetlb_pagesizes_write,
+        .llseek         = seq_lseek,
+        .release        = single_release,

+};
+#endif /* ARCH_HAS_VARIABLE_HUGEPAGE_SIZE */
#endif /* CONFIG_SYSCTL */

+static int __init hugetlb_init(void)
+{
+ unsigned long i, j;
+ struct proc_dir_entry *pde;
+
+#ifndef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+ if (HPAGE_SHIFT(0) = 0)
+ return 0;
+#endif
+
+ for (i = 0; i < MAX_NUMNODES; ++i)
+ for (j = 0; j < MAX_ORDER; ++j)
+ INIT_LIST_HEAD(&hugepage_freelists[i][j]);
+
+ for (i = 0; i < max_huge_pages; ++i) {
+ if (!alloc_fresh_huge_page(HUGETLB_INIT_PAGE_ORDER))
+ break;
+ }
+ max_huge_pages = free_huge_pages[HUGETLB_INIT_PAGE_ORDER] nr_huge_pages[HUGETLB_INIT_PAGE_ORDER] = i;
+ printk("Total HugeTLB memory allocated, %ld\n",
+ free_huge_pages[HUGETLB_INIT_PAGE_ORDER]);
+
+#ifdef ARCH_HAS_VARIABLE_HUGEPAGE_SIZE
+ if (!proc_mkdir("hugetlb", NULL))
+ return 1;
+ pde = create_proc_entry("hugetlb/hugepages", 0, NULL);
+ if (!pde)
+ return 1;
+ pde->proc_fops = &proc_hugetlb_pagesizes_operations;
+#endif
+ return 0;
+}
+module_init(hugetlb_init);
+
+static int __init hugetlb_setup(char *s)
+{
+ if (sscanf(s, "%lu", &max_huge_pages) <= 0)
+ max_huge_pages = 0;
+ return 1;
+}
+__setup("hugepages=", hugetlb_setup);
+
+
int hugetlb_report_meminfo(char *buf)
{
return sprintf(buf,

#8 lore raw

Re: [PATCH 8/8] IA64 various hugepage size - Modify kernel document

Modify the kernel document about hugetlb

Documentation/vm/hugetlbpage.txt +4 -1
diff -Nraup a/Documentation/vm/hugetlbpage.txt b/Documentation/vm/hugetlbpage.txt
@@ -68,9 +68,12 @@ If the user applications are going to re
 call, then it is required that system administrator mount a file system of
 type hugetlbfs:
 
-	mount none /mnt/huge -t hugetlbfs <uid=value> <gid=value> <mode=value>
+	mount none /mnt/huge -t hugetlbfs -o <uid=value> <gid=value> <mode=value>
 		 <size=value> <nr_inodes=value>
 
+on IA64 there is another mount option page_size=value which could
+mount a hugetlbfs with given huge page size;
+
 This command mounts a (pseudo) filesystem of type hugetlbfs on the directory
 /mnt/huge.  Any files created on /mnt/huge uses hugepages.  The uid and gid
 options sets the owner and group of the root of the file system.  By default
#9 lore raw

Re: [PATCH 8/8] IA64 various hugepage size - Modify kernel document

Quoted 8 lines · On 14 Apr 2006 09:00:18 +0800 Zou Nan hai wrote:

Modify the kernel document about hugetlb

diff -Nraup a/Documentation/vm/hugetlbpage.txt b/Documentation/vm/hugetlbpage.txt

Documentation/vm/hugetlbpage.txt If the user applications are going to re · line 76

Quoted 10 lines
Documentation/vm/hugetlbpage.txt +4 -1
@@ -68,9 +68,12 @@ If the user applications are going to re
 call, then it is required that system administrator mount a file system of
 type hugetlbfs:
 
-	mount none /mnt/huge -t hugetlbfs <uid=value> <gid=value> <mode=value>
+	mount none /mnt/huge -t hugetlbfs -o <uid=value> <gid=value> <mode=value>
 		 <size=value> <nr_inodes=value>
 
+on IA64 there is another mount option page_size=value which could
+mount a hugetlbfs with given huge page size;
+

Typos. How about:
On IA64 there is another mount option (page_size=value) which can be used to mount a hugetlbfs with the given huge page size.

Documentation/vm/hugetlbpage.txt If the user applications are going to re · line 79

Quoted 3 lines
Documentation/vm/hugetlbpage.txt +0 -0
 This command mounts a (pseudo) filesystem of type hugetlbfs on the directory
 /mnt/huge.  Any files created on /mnt/huge uses hugepages.  The uid and gid
 options sets the owner and group of the root of the file system.  By default

---
~Randy