page accounting

brokenpatch Andrew Morton 3 participants 4 messages Apr 10, 2002 – Apr 11, 2002

Subsystems (11)

Memory management MM core Page allocator SPARC + UltraSPARC (sparc/sparc64) +7 more subsystems

lore

Share & tools

lore

Andrew Morton

author

2002-04-10 11:02 UTC #0 lore raw

The patch implements per-CPU accounting of the global number of locked, diry and pagecache pages.

It also busts the headers up a bit, breaking the various SetPageFoo() macros into <linux/page-flags.h>. The page accounting declarations are in <linux/page-accounting.h> The intent here is that these headers be included directly in the .c files which need them. But at this stage they're just pulled into mm.h.

The locked and dirty page accounting is needed for making writeback and throttling decisions in the address_space-based writeback code.

Problem is, I just converted it to use the new per-CPU code and it broke. The numbers aren't correct. It worked fine with open-coded per-CPU accumulators. Rusty, can you spot the error?

=====================================

arch/sparc64/kernel/sys_sunos32.c renamed from arch/sparc64/kernel/sys_sunos32.c~dallocbase-35-page_accounting +1 -1

@@ -157,7 +157,7 @@ asmlinkage int sunos_brk(u32 baddr)
 	 * fool it, but this should catch most mistakes.
 	 */
 	freepages = atomic_read(&buffermem_pages) >> PAGE_SHIFT;
-	freepages += atomic_read(&page_cache_size);
+	freepages += get_page_cache_size();
 	freepages >>= 1;
 	freepages += nr_free_pages();
 	freepages += nr_swap_pages;

arch/sparc/kernel/sys_sunos.c renamed from arch/sparc/kernel/sys_sunos.c~dallocbase-35-page_accounting +1 -1

@@ -193,7 +193,7 @@ asmlinkage int sunos_brk(unsigned long b
 	 * fool it, but this should catch most mistakes.
 	 */
 	freepages = atomic_read(&buffermem_pages) >> PAGE_SHIFT;
-	freepages += atomic_read(&page_cache_size);
+	freepages += get_page_cache_size();
 	freepages >>= 1;
 	freepages += nr_free_pages();
 	freepages += nr_swap_pages;

drivers/md/lvm-snap.c renamed from drivers/md/lvm-snap.c~dallocbase-35-page_accounting +2 -1

@@ -34,6 +34,7 @@
  *                 o pv number is returned in new uint * arg
  *                 o -1 returned on error
  *                 lvm_snapshot_fill_COW_table has a return value too.
+ *    25/02/2002 - s/LockPage/SetPageLocked/ - [email protected]
  *
  */

@@ -451,7 +452,7 @@ int lvm_snapshot_alloc_iobuf_pages(struc
 			goto out;
 
 		iobuf->maplist[i] = page;
-		LockPage(page);
+		SetPageLocked(page);
 		iobuf->nr_pages++;
 	}
 	iobuf->offset = 0;

fs/proc/proc_misc.c renamed from fs/proc/proc_misc.c~dallocbase-35-page_accounting +10 -3

@@ -129,14 +129,16 @@ static int meminfo_read_proc(char *page,
 	struct sysinfo i;
 	int len;
 	int pg_size ;
+	struct page_state ps;
 
+	get_page_state(&ps);
 /*
  * display in kilobytes.
  */
 #define K(x) ((x) << (PAGE_SHIFT - 10))
 	si_meminfo(&i);
 	si_swapinfo(&i);
-	pg_size = atomic_read(&page_cache_size) - i.bufferram ;
+	pg_size = get_page_cache_size() - i.bufferram ;
 
 	/*
 	 * Tagged format, for easy grepping and expansion.

@@ -155,7 +157,9 @@ static int meminfo_read_proc(char *page,
 		"LowTotal:     %8lu kB\n"
 		"LowFree:      %8lu kB\n"
 		"SwapTotal:    %8lu kB\n"
-		"SwapFree:     %8lu kB\n",
+		"SwapFree:     %8lu kB\n"
+		"Dirty:        %8lu kB\n"
+		"Locked:       %8lu kB\n",
 		K(i.totalram),
 		K(i.freeram),
 		K(i.sharedram),

@@ -169,7 +173,10 @@ static int meminfo_read_proc(char *page,
 		K(i.totalram-i.totalhigh),
 		K(i.freeram-i.freehigh),
 		K(i.totalswap),
-		K(i.freeswap));
+		K(i.freeswap),
+		K(ps.nr_dirty),
+		K(ps.nr_locked)
+		);
 
 	return proc_calc_metrics(page, start, off, count, eof, len);
 #undef K

include/linux/mm.h renamed from include/linux/mm.h~dallocbase-35-page_accounting +19 ~~-61~~

123 lines not on this page · raw message

include/linux/pagemap.h renamed from include/linux/pagemap.h~dallocbase-35-page_accounting +2 -3

@@ -41,8 +41,6 @@ static inline struct page *page_cache_al
  */
 #define page_cache_entry(x)	virt_to_page(x)
 
-extern atomic_t page_cache_size; /* # of pages currently in the page cache */
-
 extern struct page * find_get_page(struct address_space *mapping,
 				unsigned long index);
 extern struct page * find_lock_page(struct address_space *mapping,

@@ -61,6 +59,7 @@ extern int add_to_page_cache(struct page
 		struct address_space *mapping, unsigned long index);
 extern int add_to_page_cache_unique(struct page *page,
 		struct address_space *mapping, unsigned long index);
+
 static inline void ___add_to_page_cache(struct page *page,
 		struct address_space *mapping, unsigned long index)
 {

@@ -69,7 +68,7 @@ static inline void ___add_to_page_cache(
 	page->index = index;
 
 	mapping->nrpages++;
-	atomic_inc(&page_cache_size);
+	inc_page_state(nr_pagecache);
 }
 
 extern void FASTCALL(lock_page(struct page *page));

include/linux/swap.h renamed from include/linux/swap.h~dallocbase-35-page_accounting +0 -1

@@ -100,7 +100,6 @@ extern unsigned int nr_free_buffer_pages
 extern int nr_active_pages;
 extern int nr_inactive_pages;
 extern atomic_t nr_async_pages;
-extern atomic_t page_cache_size;
 extern atomic_t buffermem_pages;
 extern spinlock_t pagecache_lock;
 extern void __remove_inode_page(struct page *);

mm/filemap.c renamed from mm/filemap.c~dallocbase-35-page_accounting +7 -8

@@ -46,7 +46,6 @@
  * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <[email protected]>
  */
 
-atomic_t page_cache_size = ATOMIC_INIT(0);
 
 /*
  * Lock ordering:

@@ -74,7 +73,7 @@ void __remove_inode_page(struct page *pa
 	page->mapping = NULL;
 
 	mapping->nrpages--;
-	atomic_dec(&page_cache_size);
+	dec_page_state(nr_pagecache);
 }
 
 void remove_inode_page(struct page *page)

@@ -103,7 +102,7 @@ static inline int sync_page(struct page 
  */
 void set_page_dirty(struct page *page)
 {
-	if (!test_and_set_bit(PG_dirty, &page->flags)) {
+	if (!TestSetPageDirty(page)) {
 		struct address_space *mapping = page->mapping;
 
 		if (mapping) {

@@ -583,13 +582,13 @@ int filemap_fdatawait(struct address_spa
 static int __add_to_page_cache(struct page *page,
 		struct address_space *mapping, unsigned long offset)
 {
-	unsigned long flags;
-
 	page_cache_get(page);
 	if (radix_tree_insert(&mapping->page_tree, offset, page) < 0)
 		goto nomem;
-	flags = page->flags & ~(1 << PG_uptodate | 1 << PG_error | 1 << PG_dirty | 1 << PG_referenced | 1 << PG_arch_1 | 1 << PG_checked);
-	page->flags = flags | (1 << PG_locked);
+	page->flags &= ~(1 << PG_uptodate | 1 << PG_error |
+			1 << PG_referenced | 1 << PG_arch_1 | 1 << PG_checked);
+	SetPageLocked(page);
+	ClearPageDirty(page);
 	___add_to_page_cache(page, mapping, offset);
 	return 0;
  nomem:

@@ -714,7 +713,7 @@ void unlock_page(struct page *page)
 	wait_queue_head_t *waitqueue = page_waitqueue(page);
 	clear_bit(PG_launder, &(page)->flags);
 	smp_mb__before_clear_bit();
-	if (!test_and_clear_bit(PG_locked, &(page)->flags))
+	if (!TestClearPageLocked(page))
 		BUG();
 	smp_mb__after_clear_bit(); 
 	if (waitqueue_active(waitqueue))

mm/mmap.c renamed from mm/mmap.c~dallocbase-35-page_accounting +1 -1

@@ -69,7 +69,7 @@ int vm_enough_memory(long pages)
 	    return 1;
 
 	/* The page cache contains buffer pages these days.. */
-	free = atomic_read(&page_cache_size);
+	free = get_page_cache_size();
 	free += nr_free_pages();
 	free += nr_swap_pages;

mm/page_alloc.c renamed from mm/page_alloc.c~dallocbase-35-page_accounting +36 -1

@@ -111,7 +111,8 @@ static void __free_pages_ok (struct page
 		BUG();
 	if (PageActive(page))
 		BUG();
-	page->flags &= ~((1<<PG_referenced) | (1<<PG_dirty));
+	ClearPageDirty(page);
+	page->flags &= ~(1<<PG_referenced);
 
 	if (current->flags & PF_FREE_PAGES)
 		goto local_freelist;

@@ -533,6 +534,40 @@ unsigned int nr_free_highpages (void)
 }
 #endif
 
+/*
+ * Accumulate the page_state information across all CPUs.
+ * The result is unavoidably approximate - it can change
+ * during and after execution of this function.
+ */
+struct page_state page_states __per_cpu_data;
+EXPORT_SYMBOL(page_states);
+
+void get_page_state(struct page_state *ret)
+{
+	int pcpu;
+
+	ret->nr_dirty = 0;
+	ret->nr_locked = 0;
+	ret->nr_pagecache = 0;
+
+	for (pcpu = 0; pcpu < smp_num_cpus; pcpu++) {
+		struct page_state *ps;
+
+		ps = &per_cpu(page_states, cpu_logical_map(pcpu));
+		ret->nr_dirty += ps->nr_dirty;
+		ret->nr_locked += ps->nr_locked;
+		ret->nr_pagecache += ps->nr_pagecache;
+	}
+}
+
+unsigned long get_page_cache_size(void)
+{
+	struct page_state ps;
+
+	get_page_state(&ps);
+	return ps.nr_pagecache;
+}
+
 #define K(x) ((x) << (PAGE_SHIFT-10))
 
 /*

mm/vmscan.c renamed from mm/vmscan.c~dallocbase-35-page_accounting +1 -1

@@ -409,7 +409,7 @@ static int shrink_cache(int nr_pages, zo
 			/*
 			 * It is not critical here to write it only if
 			 * the page is unmapped beause any direct writer
-			 * like O_DIRECT would set the PG_dirty bitflag
+			 * like O_DIRECT would set the page's dirty bitflag
 			 * on the phisical page after having successfully
 			 * pinned it and after the I/O to the page is finished,
 			 * so the direct writes to the page cannot get lost.

drivers/char/agp/agpgart_be.c renamed from drivers/char/agp/agpgart_be.c~dallocbase-35-page_accounting +2 -2

@@ -770,7 +770,7 @@ static unsigned long agp_generic_alloc_p
 		return 0;
 
 	get_page(page);
-	LockPage(page);
+	SetPageLocked(page);
 	atomic_inc(&agp_bridge.current_memory_agp);
 	return (unsigned long)page_address(page);
 }

@@ -2744,7 +2744,7 @@ static unsigned long ali_alloc_page(void
 		return 0;
 
 	get_page(page);
-	LockPage(page);
+	SetPageLocked(page);
 	atomic_inc(&agp_bridge.current_memory_agp);
 
 	global_cache_flush();

mm/swap_state.c renamed from mm/swap_state.c~dallocbase-35-page_accounting +6 -5

@@ -159,10 +159,11 @@ int move_to_swap_cache(struct page *page
 
 		/* Add it to the swap cache */
 		*pslot = page;
-		page->flags = ((page->flags & ~(1 << PG_uptodate | 1 << PG_error
-						| 1 << PG_dirty  | 1 << PG_referenced
-						| 1 << PG_arch_1 | 1 << PG_checked))
-			       | (1 << PG_locked));
+		page->flags &= ~(1 << PG_uptodate | 1 << PG_error
+				| 1 << PG_referenced | 1 << PG_arch_1
+				| 1 << PG_checked);
+		SetPageLocked(page);
+		ClearPageDirty(page);
 		___add_to_page_cache(page, &swapper_space, entry.val);
 	}

@@ -207,7 +208,7 @@ int move_from_swap_cache(struct page *pa
 		page->flags &= ~(1 << PG_uptodate | 1 << PG_error |
 				 1 << PG_referenced | 1 << PG_arch_1 |
 				 1 << PG_checked);
-		page->flags |= (1 << PG_dirty);
+		SetPageDirty(page);
 		___add_to_page_cache(page, mapping, index);
 	}

include/linux/page-flags.h new file +120 -0

120 lines not on this page · raw message

include/linux/page-accounting.h new file +33 -0

@@ -0,0 +1,33 @@
+/*
+ * Per-CPU page accounting.  Inclusion of this file requires
+ * that <linux/percpu.h> be included beforehand.
+ */
+
+#ifndef PAGE_ACCOUNTING_H
+#define PAGE_ACCOUNTING_H
+
+/*
+ * Dirty- and locked-page accounting.  One instance per CPU.
+ */
+struct page_state {
+	unsigned long nr_dirty;
+	unsigned long nr_locked;
+	unsigned long nr_pagecache;
+};
+
+extern struct page_state page_states;
+
+extern void get_page_state(struct page_state *ret);
+
+#define mod_page_state(member, delta)					\
+	do {								\
+		preempt_disable();					\
+		this_cpu(page_states).member += delta;			\
+		preempt_enable();					\
+	} while (0)
+
+#define inc_page_state(member)	mod_page_state(member, 1UL)
+#define dec_page_state(member)	mod_page_state(member, 0UL - 1)
+
+#endif		/* PAGE_ACCOUNTING_H */
+

Rusty Russell

2002-04-10 12:53 UTC #1 lore raw

In message <[email protected]> you write:

Quoted 4 lines

Problem is, I just converted it to use the new per-CPU code and it broke. The numbers aren't correct. It worked fine with open-coded per-CPU accumulators. Rusty, can you spot the error?

Can't see it. Dump each one (0 ... NR_CPUS-1) and see where the counters are actually going?

Rusty.
--
Anyone who quotes me in their sig is an idiot. -- Rusty Russell.

Suparna Bhattacharya

2002-04-11 15:15 UTC #2 lore raw

In reply to Andrew Morton

Quoted 12 lines · Andrew Morton

The patch implements per-CPU accounting of the global number of locked, diry and pagecache pages.
include/linux/mm.h renamed from include/linux/mm.h~dallocbase-35-page_accounting +6 -1
-#define PG_locked               0      /* Page is locked. Don't touch. */
+
+/*
+ * Don't use the *_dontuse flags.  Use the macros.  Otherwise
+ * you'll break locked- and dirty-page accounting.
+ */
+#define PG_locked_dontuse       0      /* Page is locked. Don't touch. */
 #define PG_error                1

include/linux/mm.h typedef struct page { · line 292

Quoted 10 lines

include/linux/mm.h renamed from include/linux/mm.h~dallocbase-35-page_accounting +2 -3

 #define PG_referenced           2
 #define PG_uptodate             3
-#define PG_dirty                4
+#define PG_dirty_dontuse        4
 #define PG_unused               5
 #define PG_lru                  6
 #define PG_active               7
-#define PG_slab                         8
-#define PG_skip                        10
+#define PG_slab                         8      /* kill me if needed: slab debug */

A little plea for mercy for this tiny bit :)
Could we keep PG_slab around unless its really causing trouble ? It makes life easier for some things we do ... Also its rather nice that today as a result of a wide usage of
slab infrastructure in the kernel, one can easily/directly interpret the contents of
almost any arbitrary memory location in the kernel, if its a slab page, as one knows what type of data it contains.

Quoted 9 lines

include/linux/mm.h renamed from include/linux/mm.h~dallocbase-35-page_accounting +1 -1

+#define PG_skip                        10      /* kill me now: obsolete */
 #define PG_highmem             11
 #define PG_checked             12      /* kill me in 2.5.<early>. */
 #define PG_arch_1              13
 #define PG_reserved            14
 #define PG_launder             15      /* written out by VM pressure.. */
-
 #define PG_private             16      /* Has something at ->private */

Andrew Morton

author

2002-04-11 19:06 UTC #3 lore raw

In reply to Andrew Morton

Quoted 4 lines · Suparna Bhattacharya

Quoted 2 lines
+#define PG_slab 8 /* kill me if needed: slab debug */
A little plea for mercy for this tiny bit :)

Sure, I'll update the comment.

Things are getting a *little* squeezy in the page->flags department. The zone takes eight, and of the remaining 24, I'm showing 18 used up. A couple of these can be recycled easily. I added two to support delayed allocate:

PG_disk_reserved: page has a disk-reservation
PG_space_reclaim: icky hack to avoid deadlocking when
writeback is forced to collapse outstanding reservations.

A number of the reasons for delalloc are going away now; we'll be able to do multipage bio-based writeback and readahead for the map-at-prepare_write buffer-based
filesystems. We'll see.

Updated patchset is at
http://www.zip.com.au/~akpm/linux/patches/2.5/2.5.8-pre3/
It hasn't quite recovered from the changed buffer/page relationship yet - a heavy dbench run on 1k blocksize ext2 dies after half an hour over a page which didn't come unlocked.

- Went back to open-coded per-CPU page accumulators. I stared at the assembly for some time and it looked OK, so I'm not sure what went wrong with the `percpu' version. I'll have another shot later.

- Split fs/fs-writeback.c out from fs/inode.c. These are all the functions related to sending bulk file data to storage. fs/inode.c contains the inode writeback code, the hashing, all the other stuff involved with manipulating the state of in-core inodes.

I think this is a reasonable splitup - it makes the diff more readable too...

- include/linux/writeback.h is for communication between fs/fs-writeback.c, mm/page-writeback.c and fs/inode.c

- Lots more changes to fs/buffer.c; many of them pointless cleanups and shuffling functions around and documenting stuff.

- Documenting the VM/fs locking ordering rules, slowly.

- The locking between __set_page_dirty_buffers(), try_to_free_buffers() and the functions which attach buffers to pages is coming together. This exclusion is needed to preserve the buffer-page relationship which has been proposed.

It would be a ton easier and cleaner if set_page_dirty was called under the page lock, but that's rather hard to arrange. Maybe that would be a better approach.

- We need to talk wli into doing a hashed wakeup for the buffer layer. Then buffer_head will be:

struct buffer_head {

        sector_t b_blocknr;             /* block number */
        unsigned short b_size;          /* block size */
        kdev_t b_dev;                   /* device (B_FREE = free) */

struct block_device *b_bdev;
atomic_t b_count; /* users using this block */
unsigned long b_state; /* buffer state bitmap (see above) */
struct buffer_head *b_this_page;/* circular list of buffers in one page */
char * b_data; /* pointer to data block */
struct page *b_page; /* the page this bh is mapped to */
void (*b_end_io)(struct buffer_head *bh, int uptodate); /* I/O completion */
void *b_private; /* reserved for b_end_io */
struct list_head b_inode_buffers; /* doubly linked list of inode dirty buffers */
};

I suspect we can also remove b_dev, maybe b_size, conceivably b_data.