activate pages in batch

Instead of re-acquiring the highly contented LRU lock on every single page activation, deploy an extra pagevec to do page activation in batch.

The first patch is just grouping all pagevecs we use into one array which makes further refactoring easier.

The second patch simplifies the interface for flushing a pagevec to the proper LRU list.

And finally, the last patch changes page activation to batch-mode.

Hannes

 include/linux/pagevec.h |   21 +++-
 mm/swap.c               |  216 ++++++++++++++++++++++++------------------------
 2 files changed, 127 insertions(+), 110 deletions(-)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Use an array for all pagevecs, not just for those we cache new LRU pages on. This will ease further refactoring.

---
 include/linux/pagevec.h |    7 +++++++
 mm/swap.c               |   15 +++++++--------
 2 files changed, 14 insertions(+), 8 deletions(-)

@@ -36,8 +36,7 @@
 /* How many pages do we try to swap or page in/out together? */
 int page_cluster;
 
-static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
-static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
+static DEFINE_PER_CPU(struct pagevec[NR_LRU_PAGEVECS], lru_pvecs);
 
 /*
  * This path almost never happens for VM activity - pages are normally

@@ -144,7 +143,7 @@ void  rotate_reclaimable_page(struct pag
 
 		page_cache_get(page);
 		local_irq_save(flags);
-		pvec = &__get_cpu_var(lru_rotate_pvecs);
+		pvec = &__get_cpu_var(lru_pvecs)[PAGEVEC_ROTATE];
 		if (!pagevec_add(pvec, page))
 			pagevec_move_tail(pvec);
 		local_irq_restore(flags);

@@ -198,12 +197,12 @@ EXPORT_SYMBOL(mark_page_accessed);
 
 void __lru_cache_add(struct page *page, enum lru_list lru)
 {
-	struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
+	struct pagevec *pvec = &get_cpu_var(lru_pvecs)[PAGEVEC_ADD + lru];
 
 	page_cache_get(page);
 	if (!pagevec_add(pvec, page))
 		____pagevec_lru_add(pvec, lru);
-	put_cpu_var(lru_add_pvecs);
+	put_cpu_var(lru_pvecs);
 }
 
 /**

@@ -272,17 +271,17 @@ void lru_cache_add_active_or_unevictable
  */
 static void drain_cpu_pagevecs(int cpu)
 {
-	struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
+	struct pagevec *pvecs = per_cpu(lru_pvecs, cpu);
 	struct pagevec *pvec;
 	int lru;
 
 	for_each_lru(lru) {
-		pvec = &pvecs[lru - LRU_BASE];
+		pvec = &pvecs[PAGEVEC_ADD + lru];
 		if (pagevec_count(pvec))
 			____pagevec_lru_add(pvec, lru);
 	}
 
-	pvec = &per_cpu(lru_rotate_pvecs, cpu);
+	pvec = &pvecs[PAGEVEC_ROTATE];
 	if (pagevec_count(pvec)) {
 		unsigned long flags;
 

@@ -20,6 +20,13 @@ struct pagevec {
 	struct page *pages[PAGEVEC_SIZE];
 };
 
+enum lru_pagevec {
+	PAGEVEC_BASE,
+	PAGEVEC_ADD = PAGEVEC_BASE,
+	PAGEVEC_ROTATE = NR_LRU_LISTS,
+	NR_LRU_PAGEVECS
+};
+
 void __pagevec_release(struct pagevec *pvec);
 void __pagevec_release_nonlru(struct pagevec *pvec);
 void __pagevec_free(struct pagevec *pvec);

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Having all pagevecs in one array allows for easier flushing. Use a single flush function that decides what to do based on the target LRU.

---
 include/linux/pagevec.h |   13 +++--
 mm/swap.c               |  121 +++++++++++++++++++++++-------------------------
 2 files changed, 66 insertions(+), 68 deletions(-)

@@ -27,10 +27,13 @@ enum lru_pagevec {
 	NR_LRU_PAGEVECS
 };
 
+#define for_each_lru_pagevec(pv)		\
+	for (pv = 0; pv < NR_LRU_PAGEVECS; pv++)
+
 void __pagevec_release(struct pagevec *pvec);
 void __pagevec_release_nonlru(struct pagevec *pvec);
 void __pagevec_free(struct pagevec *pvec);
-void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru);
+void __pagevec_flush(struct pagevec *pvec, enum lru_pagevec target);
 void pagevec_strip(struct pagevec *pvec);
 void pagevec_swap_free(struct pagevec *pvec);
 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,

@@ -90,22 +93,22 @@ static inline void pagevec_free(struct p
 
 static inline void __pagevec_lru_add_anon(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_INACTIVE_ANON);
+	__pagevec_flush(pvec, LRU_INACTIVE_ANON);
 }
 
 static inline void __pagevec_lru_add_active_anon(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_ACTIVE_ANON);
+	__pagevec_flush(pvec, LRU_ACTIVE_ANON);
 }
 
 static inline void __pagevec_lru_add_file(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_INACTIVE_FILE);
+	__pagevec_flush(pvec, LRU_INACTIVE_FILE);
 }
 
 static inline void __pagevec_lru_add_active_file(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_ACTIVE_FILE);
+	__pagevec_flush(pvec, LRU_ACTIVE_FILE);
 }
 
 static inline void pagevec_lru_add_file(struct pagevec *pvec)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

I don't see any other use-case for it now. So, yes, let's drop it.

Hmm, wondering which valid lvalue construction could break it...? Probably something involving stars...

Okay, get doubly rid of it. Replacement patch coming soon.

Hannes

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

---
Having all pagevecs in one array allows for easier flushing. Use a single flush function that decides what to do based on the target LRU.

---
v2: Drop for_each_lru_pagevec()

 include/linux/pagevec.h |   10 +--
 mm/swap.c               |  121 +++++++++++++++++++++++-------------------------
 2 files changed, 63 insertions(+), 68 deletions(-)

@@ -30,7 +30,7 @@ enum lru_pagevec {
 void __pagevec_release(struct pagevec *pvec);
 void __pagevec_release_nonlru(struct pagevec *pvec);
 void __pagevec_free(struct pagevec *pvec);
-void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru);
+void __pagevec_flush(struct pagevec *pvec, enum lru_pagevec target);
 void pagevec_strip(struct pagevec *pvec);
 void pagevec_swap_free(struct pagevec *pvec);
 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,

@@ -90,22 +90,22 @@ static inline void pagevec_free(struct p
 
 static inline void __pagevec_lru_add_anon(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_INACTIVE_ANON);
+	__pagevec_flush(pvec, LRU_INACTIVE_ANON);
 }
 
 static inline void __pagevec_lru_add_active_anon(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_ACTIVE_ANON);
+	__pagevec_flush(pvec, LRU_ACTIVE_ANON);
 }
 
 static inline void __pagevec_lru_add_file(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_INACTIVE_FILE);
+	__pagevec_flush(pvec, LRU_INACTIVE_FILE);
 }
 
 static inline void __pagevec_lru_add_active_file(struct pagevec *pvec)
 {
-	____pagevec_lru_add(pvec, LRU_ACTIVE_FILE);
+	__pagevec_flush(pvec, LRU_ACTIVE_FILE);
 }
 
 static inline void pagevec_lru_add_file(struct pagevec *pvec)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Instead of acquiring the highly contented LRU lock on each page activation, use a pagevec and activate pages batch-wise.

Also factor out the add-to-cache-maybe-flush mechanism that is shared between page rotation and activation code.

---
 include/linux/pagevec.h |    1 
 mm/swap.c               |   80 +++++++++++++++++++++++++++---------------------
 2 files changed, 47 insertions(+), 34 deletions(-)

@@ -24,6 +24,7 @@ enum lru_pagevec {
 	PAGEVEC_BASE,
 	PAGEVEC_ADD = PAGEVEC_BASE,
 	PAGEVEC_ROTATE = NR_LRU_LISTS,
+	PAGEVEC_ACTIVATE,
 	NR_LRU_PAGEVECS
 };
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Hi, Hannes

Do you have any mesurement result?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Not yet, sorry.

Spinlocks are no-ops on my architecture, though, so the best I can come up with is results from emulating an SMP machine, would that be okay?

Hannes

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

it's not ok..

if you can explain best mesurement way, I can mesure on 8 way machine :) (but, of cource, I should mesure your madv_sequence patch earlier)

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>

Ok.

Hmm, the `best way' is probably something else, but I played with the attached program. It causes around as much activations as I read in pages and a lot of scanning, too, so perhaps this could work. On your box, you most likely need to turn up the knobs a bit, though ;)

Thanks a lot for this, btw!

Hannes

---
Sample output from the program:

$ egrep '(pgactivate|pgscan_direct_normal)' /proc/vmstat; \
  /usr/bin/time ./activate-reclaim-smp; \
  egrep '(pgactivate|pgscan_direct_normal)' /proc/vmstat

pgactivate 9587603
pgscan_direct_normal 8150176
2: warning, can not migrate to cpu
1: warning, can not migrate to cpu
<snipped warnings, you shouldn't get those, of course!>
/loader
/children
1.93user 16.36system 0:58.17elapsed 31%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+0outputs (31major+526765minor)pagefaults 0swaps pgactivate 9856316
pgscan_direct_normal 8603232

---
#define _GNU_SOURCE
#include <stdio.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdlib.h>
#include <sched.h>
#include <sys/mman.h>

#define NR_CPUS 8
#define PROCS_PER_CPU 4

#define NR_PROCS (NR_CPUS * PROCS_PER_CPU)

#define FILE_SIZE (1<<30)
#define ANON_SIZE (1<<30)

static void move_self_to(int cpu)

{
	cpu_set_t set;

	CPU_ZERO(&set);
	CPU_SET(cpu, &set);
	if (sched_setaffinity(0, sizeof(set), &set))
		printf("%d: warning, can not migrate to cpu\n", cpu);
	sched_yield();
}

/* generate file pages */
static void reader(int cpu)

{
	int fd;
	char buf;
	unsigned long off;

	fd = open("zeroes", O_RDONLY);
	if (fd < 0) {
		printf("%d: open() failed\n", cpu);
		return;
	}

	for (off = 0; off < FILE_SIZE; off += sysconf(_SC_PAGESIZE)) {
		if (!read(fd, &buf, 1))
			puts("huh?");
		lseek(fd, off, SEEK_SET);
	}
	close(fd);
}

/* generate anon pages to trigger reclaims */
static void loader(void)

{
	char *map;
	unsigned long offset;

	map = mmap(NULL, ANON_SIZE, PROT_READ, MAP_PRIVATE|MAP_ANON, -1, 0);
	if (!map) {
		printf("failed to anon-map\n");
		return;
	}

	for (offset = 0; offset < ANON_SIZE; offset += sysconf(_SC_PAGESIZE))
		if (map[offset])
			puts("huh?");

	munmap(map, ANON_SIZE);
}

static pid_t spawn_on(int cpu)

{
	pid_t child = fork();

	switch (child) {
	case -1:
		printf("%d: fork() failed\n", cpu);
		exit(1);
	case 0:
		move_self_to(cpu);
		reader(cpu);
		exit(0);
	default:
		return child;
	}
}

int main(void)

{
	int cpu = -1, proc;
	pid_t children[NR_PROCS];

	while (++cpu < NR_CPUS)
		for (proc = 0; proc < PROCS_PER_CPU; proc++)
			children[cpu+proc] = spawn_on(cpu);

	loader();
	loader();
	puts("/loader");

	for (proc = 0; proc < NR_PROCS; proc++)
		waitpid(children[proc], &cpu, 0);
	puts("/children");

	return 0;
}

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>