remove page->virtual

Hot off the presses. Seems to work.

The patch removes page->virtual for all architectures which do not define WANT_PAGE_VIRTUAL. Hash for it instead.

Possibly we could define WANT_PAGE_VIRTUAL for CONFIG_HIHGMEM4G, but it seems unlikely.

A lot of the pressure went off kmap() and page_address() as a result of the move to kmap_atomic(). That should be the preferred way to address CPU load in the set_page_address() and page_address() hashing and locking.

If kmap_atomic is not usable then the next best approach is for users to cache the result of kmap() in a local rather than calling page_address() repeatedly.

One heavy user of kmap() and page_address() is the ext2 directory code.

On a 7G Quad PIII, running four concurrent instances of

while true
do
find /usr/src/linux > /dev/null
done

on ext2 with everything cached, profiling shows that the new hashed set_page_address() and page_address() implementations consume 0.4% and
1.3% of CPU time respectively. I think that's OK. (Plus the tested code
was doing an unneeded lookup in set_page_address(), for debug purposes)

c0141684 865      1.31499     page_address
c013851c 871      1.32411     kmem_cache_alloc
c015a068 1038     1.57799     dget_locked
c014ddfc 1220     1.85467     vfs_getattr
c014dd80 1320     2.00669     generic_fillattr
c0144f88 1359     2.06598     sys_chdir
c0159e60 1384     2.10398     dput
c014e334 1385     2.1055      cp_new_stat64
c01090c0 1741     2.6467      system_call
c0145b6a 1753     2.66494     .text.lock.open
c0151114 1786     2.71511     path_release
c0155d3c 2172     3.30192     filldir64
c0187580 2473     3.7595      ext2_readdir
c01eb950 2562     3.8948      atomic_dec_and_lock
c01eb690 2814     4.2779      __generic_copy_to_user
c015aba4 3194     4.85558     __d_lookup
c015207c 3567     5.42262     path_lookup
c01515dc 3775     5.73883     link_path_walk
c01eb99e 3811     5.79355     .text.lock.dec_and_lock
c01546b3 5847     8.88872     .text.lock.namei
c01884f2 6914     10.5108     .text.lock.dir

 include/linux/mm.h |   48 ++++++++----------
 init/main.c        |    1 
 kernel/ksyms.c     |    3 +
 mm/highmem.c       |  141 ++++++++++++++++++++++++++++++++++++++++++++++++++---
 mm/page_alloc.c    |    5 +
 5 files changed, 164 insertions(+), 34 deletions(-)

@@ -176,7 +176,7 @@ struct page {
 	 * Architectures with slow multiplication can define
 	 * WANT_PAGE_VIRTUAL in asm/page.h
 	 */
-#if defined(CONFIG_HIGHMEM) || defined(WANT_PAGE_VIRTUAL)
+#if defined(WANT_PAGE_VIRTUAL)
 	void *virtual;			/* Kernel virtual address (NULL if
 					   not kmapped, ie. highmem) */
 #endif /* CONFIG_HIGMEM || WANT_PAGE_VIRTUAL */

@@ -289,38 +289,34 @@ static inline void set_page_zone(struct 
 	page->flags |= zone_num << ZONE_SHIFT;
 }
 
-/*
- * In order to avoid #ifdefs within C code itself, we define
- * set_page_address to a noop for non-highmem machines, where
- * the field isn't useful.
- * The same is true for page_address() in arch-dependent code.
- */
-#if defined(CONFIG_HIGHMEM) || defined(WANT_PAGE_VIRTUAL)
+#define lowmem_page_address(page)					\
+	__va( ( ((page) - page_zone(page)->zone_mem_map)		\
+			+ page_zone(page)->zone_start_pfn) << PAGE_SHIFT)
+
+#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
+#define HASHED_PAGE_VIRTUAL
+#endif
 
+#if defined(WANT_PAGE_VIRTUAL)
+#define page_address(page) ((page)->virtual)
 #define set_page_address(page, address)			\
 	do {						\
 		(page)->virtual = (address);		\
 	} while(0)
+#define page_address_init()  do { } while(0)
+#endif
 
-#else /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */
-#define set_page_address(page, address)  do { } while(0)
-#endif /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */
+#if defined(HASHED_PAGE_VIRTUAL)
+void *page_address(struct page *page);
+void set_page_address(struct page *page, void *virtual);
+void page_address_init(void);
+#endif
 
-/*
- * Permanent address of a page. Obviously must never be
- * called on a highmem page.
- */
-#if defined(CONFIG_HIGHMEM) || defined(WANT_PAGE_VIRTUAL)
-
-#define page_address(page) ((page)->virtual)
-
-#else /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */
-
-#define page_address(page)						\
-	__va( ( ((page) - page_zone(page)->zone_mem_map)		\
-			+ page_zone(page)->zone_start_pfn) << PAGE_SHIFT)
-
-#endif /* CONFIG_HIGHMEM || WANT_PAGE_VIRTUAL */
+#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
+#define page_address(page) lowmem_page_address(page)
+#define set_page_address(page, address)  do { } while(0)
+#define page_address_init()  do { } while(0)
+#endif
 
 /*
  * Return true if this page is mapped into pagetables.  Subtle: test pte.direct

@@ -436,6 +436,7 @@ asmlinkage void __init start_kernel(void
 		initrd_start = 0;
 	}
 #endif
+	page_address_init();
 	mem_init();
 	kmem_cache_sizes_init();
 	pidhash_init();

@@ -133,6 +133,9 @@ EXPORT_SYMBOL(highmem_start_page);
 EXPORT_SYMBOL(kmap_prot);
 EXPORT_SYMBOL(kmap_pte);
 #endif
+#ifdef HASHED_PAGE_VIRTUAL
+EXPORT_SYMBOL(page_address);
+#endif
 EXPORT_SYMBOL(get_user_pages);
 
 /* filesystem internal functions */

@@ -917,12 +917,15 @@ void __init free_area_init_core(pg_data_
 			set_page_count(page, 0);
 			SetPageReserved(page);
 			INIT_LIST_HEAD(&page->list);
+#ifdef WANT_PAGE_VIRTUAL
 			if (j != ZONE_HIGHMEM)
 				/*
 				 * The shift left won't overflow because the
 				 * ZONE_NORMAL is below 4G.
 				 */
-				set_page_address(page, __va(zone_start_pfn << PAGE_SHIFT));
+				set_page_address(page,
+					__va(zone_start_pfn << PAGE_SHIFT));
+#endif
 			zone_start_pfn++;
 		}
 

.

Looks yummy. I'll take it for a spin tonight on my benchmark-o-matic. Clearing some more air in ZONE_NORMAL is always welcome here.

What's going on here? fs stuff is really hurting. At any rate, the overhead of the address calculation and hashtable lookup is microscopic according to this, and I want space.

Cheers,
Bill
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/

Ta.

c0182f90 5949 14.5552 ext2_readdir

lock_kernel()

c014f65c 5790 14.1662 path_lookup

Confused. oprofile claims read_lock(&current->fs->lock) but it's surely dcache_lock.

c01e6e80 4275 10.4595 atomic_dec_and_lock

dput/iput

c014eba4 2264     5.53924     link_path_walk          
c0158050 1988     4.86397     __d_lookup              
c01426e8 1903     4.656       sys_chdir               
c01e6bc0 1735     4.24496     __generic_copy_to_user  
c015319c 1344     3.28831     filldir64               
c014e6b4 1205     2.94823     path_release            
c0109070 1065     2.6057      system_call             
c014b934 929      2.27295     cp_new_stat64           
c014b380 822      2.01116     generic_fillattr        
c0157250 712      1.74202     dput                    
c014b3fc 622      1.52182     vfs_getattr             
c0157468 556      1.36034     dget_locked