Add support for page alloc w/ custom cache attributes

94 lines not on this page · raw message

new file mode 100644
index 0000000..609939d
@@ -0,0 +1,57 @@
+/*
+ * arch/arm/include/asm/attrib_alloc.h
+ *
+ * Page allocator with custom cache attributes
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __ARCH_ARM_ATTRIB_ALLOC_H
+#define __ARCH_ARM_ATTRIB_ALLOC_H
+
+#include <linux/types.h>
+#include <asm/page.h>
+
+#ifdef CONFIG_ARM_ATTRIB_ALLOCATOR
+struct page *arm_attrib_alloc_pages_node(int nid, gfp_t gfp,
+					 unsigned int order, pgprot_t prot);
+
+void arm_attrib_free_pages(struct page *page, unsigned int order);
+#else
+static inline struct page *arm_attrib_alloc_pages_node(int, gfp_t,
+						       unsigned int, pgprot_t)
+{
+	return NULL;
+}
+
+static inline arm_attrib_free_pages(struct page *, unsigned int)
+{
+}
+#endif
+
+static inline struct page *arm_attrib_alloc_pages(gfp_t gfp,
+					  unsigned int order, pgprot_t prot)
+{
+	return arm_attrib_alloc_pages_node(-1, gfp, order, prot);
+}
+
+#define arm_attrib_alloc_page(gfp, prot)	\
+	arm_attrib_alloc_pages((gfp), 0, (prot))
+
+#define arm_attrib_free_page(page)	arm_attrib_free_pages((page), 0)
+
+#endif /* __ARCH_ARM_ATTRIB_ALLOC_H */

index feb988a..d8c464c 100644
@@ -9,7 +9,19 @@
 #define PKMAP_NR(virt)		(((virt) - PKMAP_BASE) >> PAGE_SHIFT)
 #define PKMAP_ADDR(nr)		(PKMAP_BASE + ((nr) << PAGE_SHIFT))
 
+#ifndef CONFIG_ARM_ATTRIB_ALLOCATOR
 #define kmap_prot		PAGE_KERNEL
+#else
+static inline pgprot_t kmap_prot_of(struct page *page)
+{
+	if (PageUncached(page)) {
+		return __pgprot_modify(PAGE_KERNEL, L_PTE_MT_MASK,
+				       page->private);
+	} else
+		return PAGE_KERNEL;
+}
+#define kmap_prot		kmap_prot_of(page)
+#endif
 
 #define flush_cache_kmaps() \
 	do { \

122 lines not on this page · raw message

index 9e7742f..fa94be5 100644
@@ -485,6 +485,12 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
 {
 	unsigned long paddr;
 
+	if (PageUncached(page) &&
+	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
+	      page_private(page) == L_PTE_MT_WRITEALLOC ||
+	      page_private(page) == L_PTE_MT_DEV_CACHED))
+		return;
+
 	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 
 	paddr = page_to_phys(page) + off;

@@ -500,7 +506,15 @@ EXPORT_SYMBOL(___dma_page_cpu_to_dev);
 void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
 	size_t size, enum dma_data_direction dir)
 {
-	unsigned long paddr = page_to_phys(page) + off;
+	unsigned long paddr;
+
+	if (PageUncached(page) &&
+	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
+	      page_private(page) == L_PTE_MT_WRITEALLOC ||
+	      page_private(page) == L_PTE_MT_DEV_CACHED))
+		return;
+
+	paddr = page_to_phys(page) + off;
 
 	/* FIXME: non-speculating: not required */
 	/* don't bother invalidating if DMA to device */

index c6844cb..33e900a 100644
@@ -153,6 +153,15 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 
 void __flush_dcache_page(struct address_space *mapping, struct page *page)
 {
+	/* PageUncached is used to indicate that cache attribute flags
+	 * are stored with the page; cache maintenance may still be
+	 * required if the page is mapped inner-cacheable, outer
+	 * non-cacheable */
+	if (PageUncached(page) &&
+	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
+	      page_private(page) == L_PTE_MT_WRITEALLOC ||
+	      page_private(page) == L_PTE_MT_DEV_CACHED))
+		return;
 	/*
 	 * Writeback any data associated with the kernel mapping of this
 	 * page.  This ensures that data in the physical page is mutually

index 6ab2440..b25ebee 100644
@@ -109,6 +109,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
 {
 	unsigned int idx;
 	unsigned long vaddr;
+	struct page *page = pfn_to_page(pfn);
 
 	pagefault_disable();
 

@@ -117,7 +118,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
 #ifdef CONFIG_DEBUG_HIGHMEM
 	BUG_ON(!pte_none(*(TOP_PTE(vaddr))));
 #endif
-	set_pte_ext(TOP_PTE(vaddr), pfn_pte(pfn, kmap_prot), 0);
+	set_pte_ext(TOP_PTE(vaddr), mk_pte(page, kmap_prot), 0);
 	local_flush_tlb_kernel_page(vaddr);
 
 	return (void *)vaddr;

index ab68cf1..4739ffe 100644
@@ -184,6 +184,7 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
 #define L_PTE_MT_DEV_NONSHARED	(0x0c << 2)	/* 1100 */
 #define L_PTE_MT_DEV_WC		(0x09 << 2)	/* 1001 */
 #define L_PTE_MT_DEV_CACHED	(0x0b << 2)	/* 1011 */
+#define L_PTE_MT_INNER_WB	(0x05 << 2)	/* 0101 (armv6, armv7) */
 #define L_PTE_MT_MASK		(0x0f << 2)
 
 #ifndef __ASSEMBLY__

@@ -321,6 +322,8 @@ static inline pte_t pte_mkspecial(pte_t pte) { return pte; }
 #define pgprot_dmacoherent(prot) \
 	__pgprot_modify(prot, L_PTE_MT_MASK|L_PTE_EXEC, L_PTE_MT_UNCACHED)
 #endif
+#define pgprot_inner_writeback(prot) \
+	__pgprot_modify(prot, L_PTE_MT_MASK, L_PTE_MT_INNER_WB)
 
 #define pmd_none(pmd)		(!pmd_val(pmd))
 #define pmd_present(pmd)	(pmd_val(pmd))

index fa94be5..5de376a 100644
@@ -488,16 +488,20 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
 	if (PageUncached(page) &&
 	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
 	      page_private(page) == L_PTE_MT_WRITEALLOC ||
-	      page_private(page) == L_PTE_MT_DEV_CACHED))
+	      page_private(page) == L_PTE_MT_DEV_CACHED ||
+	      page_private(page) == L_PTE_MT_INNER_WB))
 		return;
 
 	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 
 	paddr = page_to_phys(page) + off;
-	if (dir == DMA_FROM_DEVICE) {
-		outer_inv_range(paddr, paddr + size);
-	} else {
-		outer_clean_range(paddr, paddr + size);
+
+	if (!PageUncached(page) || (page_private(page) != L_PTE_MT_INNER_WB)) {
+		if (dir == DMA_FROM_DEVICE) {
+			outer_inv_range(paddr, paddr + size);
+		} else {
+			outer_clean_range(paddr, paddr + size);
+		}
 	}
 	/* FIXME: non-speculating: flush on bidirectional mappings? */
 }

@@ -511,15 +515,18 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
 	if (PageUncached(page) &&
 	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
 	      page_private(page) == L_PTE_MT_WRITEALLOC ||
-	      page_private(page) == L_PTE_MT_DEV_CACHED))
+	      page_private(page) == L_PTE_MT_DEV_CACHED ||
+	      page_private(page) == L_PTE_MT_INNER_WB))
 		return;
 
 	paddr = page_to_phys(page) + off;
 
 	/* FIXME: non-speculating: not required */
 	/* don't bother invalidating if DMA to device */
-	if (dir != DMA_TO_DEVICE)
-		outer_inv_range(paddr, paddr + size);
+	if (!PageUncached(page) || (page_private(page) != L_PTE_MT_INNER_WB)) {
+		if (dir != DMA_TO_DEVICE)
+			outer_inv_range(paddr, paddr + size);
+	}
 
 	dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
 }

index 33e900a..4f6f115 100644
@@ -160,7 +160,8 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
 	if (PageUncached(page) &&
 	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
 	      page_private(page) == L_PTE_MT_WRITEALLOC ||
-	      page_private(page) == L_PTE_MT_DEV_CACHED))
+	      page_private(page) == L_PTE_MT_DEV_CACHED ||
+	      page_private(page) == L_PTE_MT_INNER_WB))
 		return;
 	/*
 	 * Writeback any data associated with the kernel mapping of this