per device dirty throttling -v3

Index: linux-2.6-mm/fs/nfs/write.c
===================================================================
@@ -235,10 +235,8 @@ static void nfs_end_page_writeback(struc
 	struct nfs_server *nfss = NFS_SERVER(inode);
 
 	end_page_writeback(page);
-	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH) {
+	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
 		clear_bdi_congested(&nfss->backing_dev_info, WRITE);
-		congestion_end(WRITE);
-	}
 }
 
 /*

Index: linux-2.6-mm/include/linux/backing-dev.h
===================================================================
@@ -96,7 +96,6 @@ void clear_bdi_congested(struct backing_
 void set_bdi_congested(struct backing_dev_info *bdi, int rw);
 long congestion_wait(int rw, long timeout);
 long congestion_wait_interruptible(int rw, long timeout);
-void congestion_end(int rw);
 
 #define bdi_cap_writeback_dirty(bdi) \
 	(!((bdi)->capabilities & BDI_CAP_NO_WRITEBACK))

Index: linux-2.6-mm/mm/backing-dev.c
===================================================================
@@ -70,16 +70,3 @@ long congestion_wait_interruptible(int r
 	return ret;
 }
 EXPORT_SYMBOL(congestion_wait_interruptible);
-
-/**
- * congestion_end - wake up sleepers on a congested backing_dev_info
- * @rw: READ or WRITE
- */
-void congestion_end(int rw)
-{
-	wait_queue_head_t *wqh = &congestion_wqh[rw];
-
-	if (waitqueue_active(wqh))
-		wake_up(wqh);
-}
-EXPORT_SYMBOL(congestion_end);

Index: linux-2.6-mm/block/ll_rw_blk.c
===================================================================
@@ -208,6 +208,7 @@ void blk_queue_make_request(request_queu
 	blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS);
 	blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS);
 	q->make_request_fn = mfn;
+	bdi_init(&q->backing_dev_info);
 	blk_queue_max_sectors(q, SAFE_MAX_SECTORS);
 	blk_queue_hardsect_size(q, 512);
 	blk_queue_dma_alignment(q, 511);

118 lines not on this page · raw message

106 lines not on this page · raw message

Index: linux-2.6-mm/drivers/block/rd.c
===================================================================
@@ -421,6 +421,8 @@ static int __init rd_init(void)
 	int i;
 	int err = -ENOMEM;
 
+	bdi_init(&rd_file_backing_dev_info);
+
 	if (rd_blocksize > PAGE_SIZE || rd_blocksize < 512 ||
 			(rd_blocksize & (rd_blocksize-1))) {
 		printk("RAMDISK: wrong blocksize %d, reverting to defaults\n",

Index: linux-2.6-mm/drivers/char/mem.c
===================================================================
@@ -987,6 +987,8 @@ static int __init chr_dev_init(void)
 			      MKDEV(MEM_MAJOR, devlist[i].minor),
 			      devlist[i].name);
 
+	bdi_init(&zero_bdi);
+
 	return 0;
 }
 

Index: linux-2.6-mm/fs/char_dev.c
===================================================================
@@ -548,6 +548,7 @@ static struct kobject *base_probe(dev_t 
 void __init chrdev_init(void)
 {
 	cdev_map = kobj_map_init(base_probe, &chrdevs_lock);
+	bdi_init(&directly_mappable_cdev_bdi);
 }
 
 

Index: linux-2.6-mm/fs/fuse/inode.c
===================================================================
@@ -413,6 +413,7 @@ static struct fuse_conn *new_conn(void)
 		atomic_set(&fc->num_waiting, 0);
 		fc->bdi.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
 		fc->bdi.unplug_io_fn = default_unplug_io_fn;
+		bdi_init(&fc->bdi);
 		fc->reqctr = 0;
 		fc->blocked = 1;
 		get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));

Index: linux-2.6-mm/fs/nfs/client.c
===================================================================
@@ -661,6 +661,7 @@ static void nfs_server_set_fsinfo(struct
 	server->backing_dev_info.ra_pages0 = min_t(unsigned, server->rpages,
 				VM_MIN_READAHEAD >> (PAGE_CACHE_SHIFT - 10));
 	server->backing_dev_info.ra_thrash_bytes = server->rsize * NFS_MAX_READAHEAD;
+	bdi_init(&server->backing_dev_info);
 
 	if (server->wsize > max_rpc_payload)
 		server->wsize = max_rpc_payload;

Index: linux-2.6/fs/buffer.c
===================================================================
@@ -740,6 +740,7 @@ int __set_page_dirty_buffers(struct page
 	if (page->mapping) {	/* Race with truncate? */
 		if (mapping_cap_account_dirty(mapping)) {
 			__inc_zone_page_state(page, NR_FILE_DIRTY);
+			__inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 			task_io_account_write(PAGE_CACHE_SIZE);
 		}
 		radix_tree_tag_set(&mapping->page_tree,

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -828,6 +828,7 @@ int __set_page_dirty_nobuffers(struct pa
 			BUG_ON(mapping2 != mapping);
 			if (mapping_cap_account_dirty(mapping)) {
 				__inc_zone_page_state(page, NR_FILE_DIRTY);
+				__inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 				task_io_account_write(PAGE_CACHE_SIZE);
 			}
 			radix_tree_tag_set(&mapping->page_tree,

@@ -961,6 +962,7 @@ int clear_page_dirty_for_io(struct page 
 		 */
 		if (TestClearPageDirty(page)) {
 			dec_zone_page_state(page, NR_FILE_DIRTY);
+			dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 			return 1;
 		}
 		return 0;

Index: linux-2.6/mm/truncate.c
===================================================================
@@ -71,6 +71,7 @@ void cancel_dirty_page(struct page *page
 		struct address_space *mapping = page->mapping;
 		if (mapping && mapping_cap_account_dirty(mapping)) {
 			dec_zone_page_state(page, NR_FILE_DIRTY);
+			dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 			if (account_size)
 				task_io_account_cancelled_write(account_size);
 		}

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -24,6 +24,7 @@ enum bdi_state {
 };
 
 enum bdi_stat_item {
+	BDI_DIRTY,
 	NR_BDI_STAT_ITEMS
 };
 

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -981,10 +981,12 @@ int test_clear_page_writeback(struct pag
 
 		write_lock_irqsave(&mapping->tree_lock, flags);
 		ret = TestClearPageWriteback(page);
-		if (ret)
+		if (ret) {
 			radix_tree_tag_clear(&mapping->page_tree,
 						page_index(page),
 						PAGECACHE_TAG_WRITEBACK);
+			__dec_bdi_stat(mapping->backing_dev_info, BDI_WRITEBACK);
+		}
 		write_unlock_irqrestore(&mapping->tree_lock, flags);
 	} else {
 		ret = TestClearPageWriteback(page);

@@ -1004,10 +1006,12 @@ int test_set_page_writeback(struct page 
 
 		write_lock_irqsave(&mapping->tree_lock, flags);
 		ret = TestSetPageWriteback(page);
-		if (!ret)
+		if (!ret) {
 			radix_tree_tag_set(&mapping->page_tree,
 						page_index(page),
 						PAGECACHE_TAG_WRITEBACK);
+			__inc_bdi_stat(mapping->backing_dev_info, BDI_WRITEBACK);
+		}
 		if (!PageDirty(page))
 			radix_tree_tag_clear(&mapping->page_tree,
 						page_index(page),

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -25,6 +25,7 @@ enum bdi_state {
 
 enum bdi_stat_item {
 	BDI_DIRTY,
+	BDI_WRITEBACK,
 	NR_BDI_STAT_ITEMS
 };
 

Index: linux-2.6/fs/nfs/write.c
===================================================================
@@ -451,6 +451,7 @@ nfs_mark_request_commit(struct nfs_page 
 	nfsi->ncommit++;
 	spin_unlock(&nfsi->req_lock);
 	inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+	inc_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
 }
 #endif

@@ -511,6 +512,7 @@ static void nfs_cancel_commit_list(struc
 	while(!list_empty(head)) {
 		req = nfs_list_entry(head->next);
 		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+		dec_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 		nfs_list_remove_request(req);
 		nfs_inode_remove_request(req);
 		nfs_unlock_request(req);

@@ -1236,6 +1238,7 @@ nfs_commit_list(struct inode *inode, str
 		nfs_list_remove_request(req);
 		nfs_mark_request_commit(req);
 		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+		dec_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 		nfs_clear_page_writeback(req);
 	}
 	return -ENOMEM;

@@ -1260,6 +1263,7 @@ static void nfs_commit_done(struct rpc_t
 		req = nfs_list_entry(data->pages.next);
 		nfs_list_remove_request(req);
 		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+		dec_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 
 		dprintk("NFS: commit (%s/%Ld %d@%Ld)",
 			req->wb_context->dentry->d_inode->i_sb->s_id,

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -26,6 +26,7 @@ enum bdi_state {
 enum bdi_stat_item {
 	BDI_DIRTY,
 	BDI_WRITEBACK,
+	BDI_UNSTABLE,
 	NR_BDI_STAT_ITEMS
 };
 

Index: linux-2.6/block/ll_rw_blk.c
===================================================================
@@ -3975,6 +3975,20 @@ static ssize_t queue_max_hw_sectors_show
 	return queue_var_show(max_hw_sectors_kb, (page));
 }
 
+static ssize_t queue_nr_dirty_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%lu\n", bdi_stat(&q->backing_dev_info, BDI_DIRTY));
+}
+
+static ssize_t queue_nr_writeback_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%lu\n", bdi_stat(&q->backing_dev_info, BDI_WRITEBACK));
+}
+
+static ssize_t queue_nr_unstable_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%lu\n", bdi_stat(&q->backing_dev_info, BDI_UNSTABLE));
+}
 
 static struct queue_sysfs_entry queue_requests_entry = {
 	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },

@@ -4005,6 +4019,21 @@ static struct queue_sysfs_entry queue_ma
 	.show = queue_max_hw_sectors_show,
 };
 
+static struct queue_sysfs_entry queue_dirty_entry = {
+	.attr = {.name = "dirty_pages", .mode = S_IRUGO },
+	.show = queue_nr_dirty_show,
+};
+
+static struct queue_sysfs_entry queue_writeback_entry = {
+	.attr = {.name = "writeback_pages", .mode = S_IRUGO },
+	.show = queue_nr_writeback_show,
+};
+
+static struct queue_sysfs_entry queue_unstable_entry = {
+	.attr = {.name = "unstable_pages", .mode = S_IRUGO },
+	.show = queue_nr_unstable_show,
+};
+
 static struct queue_sysfs_entry queue_iosched_entry = {
 	.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
 	.show = elv_iosched_show,

@@ -4017,6 +4046,9 @@ static struct attribute *default_attrs[]
 	&queue_initial_ra_entry.attr,
 	&queue_max_hw_sectors_entry.attr,
 	&queue_max_sectors_entry.attr,
+	&queue_dirty_entry.attr,
+	&queue_writeback_entry.attr,
+	&queue_unstable_entry.attr,
 	&queue_iosched_entry.attr,
 	NULL,
 };

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -27,6 +27,8 @@ enum bdi_stat_item {
 	BDI_DIRTY,
 	BDI_WRITEBACK,
 	BDI_UNSTABLE,
+	BDI_WRITEOUT,
+	BDI_WRITEOUT_TOTAL,
 	NR_BDI_STAT_ITEMS
 };
 

@@ -50,6 +52,12 @@ struct backing_dev_info {
 	void (*unplug_io_fn)(struct backing_dev_info *, struct page *);
 	void *unplug_io_data;
 
+	/*
+	 * data used for scaling the writeback cache
+	 */
+	spinlock_t lock;	/* protect the cycle count */
+	unsigned long cycles;	/* writeout cycles */
+
 	atomic_long_t bdi_stats[NR_BDI_STAT_ITEMS];
 #ifdef CONFIG_SMP
 	struct bdi_per_cpu_data pcd[NR_CPUS];

255 lines not on this page · raw message

Index: linux-2.6/mm/backing-dev.c
===================================================================
@@ -91,6 +91,9 @@ void bdi_init(struct backing_dev_in
 {
 	int i;
 
+	spin_lock_init(&bdi->lock);
+	bdi->cycles = 0;
+
 	for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
 		atomic_long_set(&bdi->bdi_stats[i], 0);
 

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -8,6 +8,7 @@
 #ifndef _LINUX_BACKING_DEV_H
 #define _LINUX_BACKING_DEV_H
 
+#include <linux/cpumask.h>
 #include <linux/spinlock.h>
 #include <asm/atomic.h>
 

@@ -34,7 +35,6 @@ enum bdi_stat_item {
 
 #ifdef CONFIG_SMP
 struct bdi_per_cpu_data {
-	s8 stat_threshold;
 	s8 bdi_stat_diff[NR_BDI_STAT_ITEMS];
 } ____cacheline_aligned_in_smp;
 #endif

@@ -60,6 +60,7 @@ struct backing_dev_info {
 
 	atomic_long_t bdi_stats[NR_BDI_STAT_ITEMS];
 #ifdef CONFIG_SMP
+	int stat_threshold;
 	struct bdi_per_cpu_data pcd[NR_CPUS];
 #endif
 };

@@ -109,6 +110,8 @@ static inline unsigned long bdi_stat(str
 }
 
 #ifdef CONFIG_SMP
+unsigned long bdi_stat_accurate(struct backing_dev_info *bdi, enum bdi_stat_item item);
+
 void __mod_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item, int delta);
 void __inc_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item);
 void __dec_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item);

@@ -117,8 +120,14 @@ void mod_bdi_stat(struct backing_dev_inf
 void inc_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item);
 void dec_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item);
 
+static inline unsigned long bdi_stat_delta(struct backing_dev_info *bdi)
+{
+	return num_online_cpus() * bdi->stat_threshold;
+}
 #else /* CONFIG_SMP */
 
+#define bdi_stat_accurate bdi_stat
+
 static inline void __mod_bdi_stat(struct backing_dev_info *bdi,
 		enum bdi_stat_item item, int delta)
 {

@@ -142,6 +151,8 @@ static inline void __dec_bdi_stat(struct
 #define mod_bdi_stat __mod_bdi_stat
 #define inc_bdi_stat __inc_bdi_stat
 #define dec_bdi_stat __dec_bdi_stat
+
+#define bdi_stat_delta(bdi) 1UL
 #endif
 
 void bdi_init(struct backing_dev_info *bdi);

Index: linux-2.6/mm/backing-dev.c
===================================================================
@@ -98,17 +98,36 @@ void bdi_init(struct backing_dev_in
 		atomic_long_set(&bdi->bdi_stats[i], 0);
 
 #ifdef CONFIG_SMP
+	bdi->stat_threshold = 8 * ilog2(num_online_cpus());
 	for (i = 0; i < NR_CPUS; i++) {
 		int j;
 		for (j = 0; j < NR_BDI_STAT_ITEMS; j++)
 			bdi->pcd[i].bdi_stat_diff[j] = 0;
-		bdi->pcd[i].stat_threshold = 8 * ilog2(num_online_cpus());
 	}
 #endif
 }
 EXPORT_SYMBOL(bdi_init);
 
 #ifdef CONFIG_SMP
+unsigned long bdi_stat_accurate(struct backing_dev_info *bdi,
+		enum bdi_stat_item item)
+{
+	long x = atomic_long_read(&bdi_stats[item]);
+	int cpu;
+
+	for_each_possible_cpu(cpu) {
+		struct bdi_per_cpu_data *pcd = &bdi->pcd[cpu];
+		s8 *p = pcd->bdi_stat_diff + item;
+
+		x += *p;
+	}
+
+	if (x < 0)
+		x = 0;
+
+	return x;
+}
+
 void __mod_bdi_stat(struct backing_dev_info *bdi,
 		enum bdi_stat_item item, int delta)
 {

@@ -118,7 +137,7 @@ void __mod_bdi_stat(struct backing_dev_i
 
 	x = delta + *p;
 
-	if (unlikely(x > pcd->stat_threshold || x < -pcd->stat_threshold)) {
+	if (unlikely(x > bdi->stat_threshold || x < -bdi->stat_threshold)) {
 		bdi_stat_add(x, bdi, item);
 		x = 0;
 	}

@@ -144,8 +163,8 @@ void __inc_bdi_stat(struct backing_dev_i
 
 	(*p)++;
 
-	if (unlikely(*p > pcd->stat_threshold)) {
-		int overstep = pcd->stat_threshold / 2;
+	if (unlikely(*p > bdi->stat_threshold)) {
+		int overstep = bdi->stat_threshold / 2;
 
 		bdi_stat_add(*p + overstep, bdi, item);
 		*p = -overstep;

@@ -170,8 +189,8 @@ void __dec_bdi_stat(struct backing_dev_i
 
 	(*p)--;
 
-	if (unlikely(*p < -pcd->stat_threshold)) {
-		int overstep = pcd->stat_threshold / 2;
+	if (unlikely(*p < -bdi->stat_threshold)) {
+		int overstep = bdi->stat_threshold / 2;
 
 		bdi_stat_add(*p - overstep, bdi, item);
 		*p = overstep;

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -341,14 +341,25 @@ static void balance_dirty_pages(struct a
 		 * been flushed to permanent storage.
 		 */
 		if (bdi_nr_reclaimable) {
+			unsigned long bdi_nr_writeback;
 			writeback_inodes(&wbc);
 
 			get_dirty_limits(&background_thresh, &dirty_thresh,
 				       &bdi_thresh, bdi);
-			bdi_nr_reclaimable = bdi_stat(bdi, BDI_DIRTY) +
-						bdi_stat(bdi, BDI_UNSTABLE);
-			if (bdi_nr_reclaimable + bdi_stat(bdi, BDI_WRITEBACK) <=
-			     	bdi_thresh)
+
+			if (bdi_thresh < bdi_stat_delta(bdi)) {
+				bdi_nr_reclaimable =
+					bdi_stat_accurate(bdi, BDI_DIRTY) +
+					bdi_stat_accurate(bdi, BDI_UNSTABLE);
+				bdi_nr_writeback =
+					bdi_stat_accurate(bdi, NR_WRITEBACK);
+			} else {
+				bdi_nr_reclaimable = bdi_stat(bdi, BDI_DIRTY) +
+					bdi_stat(bdi, BDI_UNSTABLE);
+				bdi_nr_writeback = bdi_stat(bdi, NR_WRITEBACK);
+			}
+
+			if (bdi_nr_reclaimable + bdi_nr_writeback <= bdi_thresh)
 				break;
 
 			pages_written += write_chunk - wbc.nr_to_write;

Index: linux-2.6-mm/include/linux/writeback.h
===================================================================
@@ -84,7 +84,6 @@ static inline void wait_on_inode(struct 
 int wakeup_pdflush(long nr_pages);
 void laptop_io_completion(void);
 void laptop_sync_completion(void);
-void throttle_vm_writeout(gfp_t gfp_mask);
 
 extern struct timer_list laptop_mode_wb_timer;
 static inline int laptop_spinned_down(void)

Index: linux-2.6-mm/mm/page-writeback.c
===================================================================
@@ -437,37 +437,6 @@ void balance_dirty_pages_ratelimited_nr(
 }
 EXPORT_SYMBOL(balance_dirty_pages_ratelimited_nr);
 
-void throttle_vm_writeout(gfp_t gfp_mask)
-{
-	long background_thresh;
-	long dirty_thresh;
-
-	if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) {
-		/*
-		 * The caller might hold locks which can prevent IO completion
-		 * or progress in the filesystem.  So we cannot just sit here
-		 * waiting for IO to complete.
-		 */
-		congestion_wait(WRITE, HZ/10);
-		return;
-	}
-
-        for ( ; ; ) {
-		get_dirty_limits(&background_thresh, &dirty_thresh, NULL, NULL);
-
-                /*
-                 * Boost the allowable dirty threshold a bit for page
-                 * allocators so they don't get DoS'ed by heavy writers
-                 */
-                dirty_thresh += dirty_thresh / 10;      /* wheeee... */
-
-                if (global_page_state(NR_UNSTABLE_NFS) +
-			global_page_state(NR_WRITEBACK) <= dirty_thresh)
-                        	break;
-                congestion_wait(WRITE, HZ/10);
-        }
-}
-
 /*
  * writeback at least _min_pages, and keep writing until the amount of dirty
  * memory is less than the background threshold, or until we're all clean.

Index: linux-2.6-mm/mm/vmscan.c
===================================================================
@@ -1047,8 +1047,6 @@ static unsigned long shrink_zone(int pri
 		}
 	}
 
-	throttle_vm_writeout(sc->gfp_mask);
-
 	atomic_dec(&zone->reclaim_in_progress);
 	return nr_reclaimed;
 }

Index: linux-2.6-mm/arch/i386/lib/usercopy.c
===================================================================
@@ -751,7 +751,7 @@ survive:
 
 			if (retval == -ENOMEM && is_init(current)) {
 				up_read(&current->mm->mmap_sem);
-				congestion_wait(WRITE, HZ/50);
+				page_alloc_wait(HZ/50);
 				goto survive;
 			}
 

Index: linux-2.6-mm/fs/xfs/linux-2.6/kmem.c
===================================================================
@@ -53,7 +53,7 @@ kmem_alloc(size_t size, unsigned int __n
 			printk(KERN_ERR "XFS: possible memory allocation "
 					"deadlock in %s (mode:0x%x)\n",
 					__FUNCTION__, lflags);
-		congestion_wait(WRITE, HZ/50);
+		page_alloc_wait(HZ/50);
 	} while (1);
 }
 

@@ -131,7 +131,7 @@ kmem_zone_alloc(kmem_zone_t *zone, unsig
 			printk(KERN_ERR "XFS: possible memory allocation "
 					"deadlock in %s (mode:0x%x)\n",
 					__FUNCTION__, lflags);
-		congestion_wait(WRITE, HZ/50);
+		page_alloc_wait(HZ/50);
 	} while (1);
 }
 

Index: linux-2.6-mm/include/linux/mm.h
===================================================================
@@ -1028,6 +1028,9 @@ extern void setup_per_cpu_pageset(void);
 static inline void setup_per_cpu_pageset(void) {}
 #endif
 
+void page_alloc_ok(void);
+long page_alloc_wait(long timeout);
+
 /* prio_tree.c */
 void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old);
 void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *);

Index: linux-2.6-mm/mm/page_alloc.c
===================================================================
@@ -107,6 +107,9 @@ unsigned long __meminitdata nr_kernel_pa
 unsigned long __meminitdata nr_all_pages;
 static unsigned long __initdata dma_reserve;
 
+static wait_queue_head_t page_alloc_wqh =
+	__WAIT_QUEUE_HEAD_INITIALIZER(page_alloc_wqh);
+
 #ifdef CONFIG_ARCH_POPULATES_NODE_MAP
   /*
    * MAX_ACTIVE_REGIONS determines the maxmimum number of distinct

@@ -1698,7 +1701,7 @@ nofail_alloc:
 			if (page)
 				goto got_pg;
 			if (gfp_mask & __GFP_NOFAIL) {
-				congestion_wait(WRITE, HZ/50);
+				page_alloc_wait(HZ/50);
 				goto nofail_alloc;
 			}
 		}

@@ -1763,7 +1766,7 @@ nofail_alloc:
 			do_retry = 1;
 	}
 	if (do_retry) {
-		congestion_wait(WRITE, HZ/50);
+		page_alloc_wait(HZ/50);
 		goto rebalance;
 	}
 

@@ -4217,3 +4220,21 @@ void set_pageblock_flags_group(struct pa
 		else
 			__clear_bit(bitidx + start_bitidx, bitmap);
 }
+
+void page_alloc_ok(void)
+{
+	if (waitqueue_active(&page_alloc_wqh))
+		wake_up(&page_alloc_wqh);
+}
+
+long page_alloc_wait(long timeout)
+{
+	long ret;
+	DEFINE_WAIT(wait);
+
+	prepare_to_wait(&page_alloc_wqh, &wait, TASK_UNINTERRUPTIBLE);
+	ret = schedule_timeout(timeout);
+	finish_wait(&page_alloc_wqh, &wait);
+	return ret;
+}
+EXPORT_SYMBOL(page_alloc_wait);

Index: linux-2.6-mm/mm/shmem.c
===================================================================
@@ -1216,7 +1216,7 @@ repeat:
 			page_cache_release(swappage);
 			if (error == -ENOMEM) {
 				/* let kswapd refresh zone for GFP_ATOMICs */
-				congestion_wait(WRITE, HZ/50);
+				page_alloc_wait(HZ/50);
 			}
 			goto repeat;
 		}

Index: linux-2.6-mm/mm/vmscan.c
===================================================================
@@ -1436,6 +1436,7 @@ static int kswapd(void *p)
 		finish_wait(&pgdat->kswapd_wait, &wait);
 
 		balance_pgdat(pgdat, order);
+		page_alloc_ok();
 	}
 	return 0;
 }

Index: linux-2.6-mm/mm/vmscan.c
===================================================================
@@ -70,6 +70,8 @@ struct scan_control {
 	int all_unreclaimable;
 
 	int order;
+
+	int encountered_congestion;
 };
 
 /*

@@ -315,7 +317,8 @@ typedef enum {
  * pageout is called by shrink_page_list() for each dirty page.
  * Calls ->writepage().
  */
-static pageout_t pageout(struct page *page, struct address_space *mapping)
+static pageout_t pageout(struct page *page, struct address_space *mapping,
+		struct scan_control *sc)
 {
 	/*
 	 * If the page is dirty, only perform writeback if that write

@@ -357,6 +360,7 @@ static pageout_t pageout(struct page *pa
 
 	if (clear_page_dirty_for_io(page)) {
 		int res;
+		struct backing_dev_info *bdi;
 		struct writeback_control wbc = {
 			.sync_mode = WB_SYNC_NONE,
 			.nr_to_write = SWAP_CLUSTER_MAX,

@@ -366,6 +370,14 @@ static pageout_t pageout(struct page *pa
 			.for_reclaim = 1,
 		};
 
+		if (mapping == &swapper_space)
+			bdi = swap_bdi(page);
+		else
+			bdi = mapping->backing_dev_info;
+
+		if (bdi_congested(bdi, WRITE))
+			sc->encountered_congestion = 1;
+
 		SetPageReclaim(page);
 		res = mapping->a_ops->writepage(page, &wbc);
 		if (res < 0)

@@ -533,7 +545,7 @@ static unsigned long shrink_page_list(st
 				goto keep_locked;
 
 			/* Page is dirty, try to write it out here */
-			switch(pageout(page, mapping)) {
+			switch(pageout(page, mapping, sc)) {
 			case PAGE_KEEP:
 				goto keep_locked;
 			case PAGE_ACTIVATE:

@@ -1141,6 +1153,7 @@ unsigned long try_to_free_pages(struct z
 
 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
 		sc.nr_scanned = 0;
+		sc.encountered_congestion = 0;
 		if (!priority)
 			disable_swap_token();
 		nr_reclaimed += shrink_zones(priority, zones, &sc);

@@ -1169,7 +1182,7 @@ unsigned long try_to_free_pages(struct z
 		}
 
 		/* Take a nap, wait for some writeback to complete */
-		if (sc.nr_scanned && priority < DEF_PRIORITY - 2)
+		if (sc.encountered_congestion)
 			congestion_wait(WRITE, HZ/10);
 	}
 	/* top priority shrink_caches still had more to do? don't OOM, then */

@@ -1250,6 +1263,7 @@ loop_again:
 		int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
 		unsigned long lru_pages = 0;
 
+		sc.encountered_congestion = 0;
 		/* The swap token gets in the way of swapout... */
 		if (!priority)
 			disable_swap_token();

@@ -1337,7 +1351,7 @@ loop_again:
 		 * OK, kswapd is getting into trouble.  Take a nap, then take
 		 * another pass across the zones.
 		 */
-		if (total_scanned && priority < DEF_PRIORITY - 2)
+		if (sc.encountered_congestion)
 			congestion_wait(WRITE, HZ/10);
 
 		/*

@@ -1580,6 +1594,7 @@ unsigned long shrink_all_memory(unsigned
 			unsigned long nr_to_scan = nr_pages - ret;
 
 			sc.nr_scanned = 0;
+			sc.encountered_congestion = 0;
 			ret += shrink_all_zones(nr_to_scan, prio, pass, &sc);
 			if (ret >= nr_pages)
 				goto out;

@@ -1591,7 +1606,7 @@ unsigned long shrink_all_memory(unsigned
 			if (ret >= nr_pages)
 				goto out;
 
-			if (sc.nr_scanned && prio < DEF_PRIORITY - 2)
+			if (sc.encountered_congestion)
 				congestion_wait(WRITE, HZ / 10);
 		}
 	}

Index: linux-2.6-mm/include/linux/swap.h
===================================================================
@@ -220,6 +220,7 @@ extern void swap_unplug_io_fn(struct bac
 
 #ifdef CONFIG_SWAP
 /* linux/mm/page_io.c */
+extern struct backing_dev_info *swap_bdi(struct page *);
 extern int swap_readpage(struct file *, struct page *);
 extern int swap_writepage(struct page *page, struct writeback_control *wbc);
 extern int end_swap_bio_read(struct bio *bio, unsigned int bytes_done, int err);

Index: linux-2.6-mm/mm/page_io.c
===================================================================
@@ -19,6 +19,15 @@
 #include <linux/writeback.h>
 #include <asm/pgtable.h>
 
+struct backing_dev_info *swap_bdi(struct page *page)
+{
+	struct swap_info_struct *sis;
+	swp_entry_t entry = { .val = page_private(page), };
+
+	sis = get_swap_info_struct(swp_type(entry));
+	return blk_get_backing_dev_info(sis->bdev);
+}
+
 static struct bio *get_swap_bio(gfp_t gfp_flags, pgoff_t index,
 				struct page *page, bio_end_io_t end_io)
 {

Index: linux-2.6-mm/include/linux/backing-dev.h
===================================================================
@@ -10,6 +10,7 @@
 
 #include <linux/cpumask.h>
 #include <linux/spinlock.h>
+#include <linux/wait.h>
 #include <asm/atomic.h>
 
 struct page;

@@ -52,6 +53,8 @@ struct backing_dev_info {
 	void (*unplug_io_fn)(struct backing_dev_info *, struct page *);
 	void *unplug_io_data;
 
+	wait_queue_head_t congestion_wqh[2];
+
 	/*
 	 * data used for scaling the writeback cache
 	 */

@@ -214,8 +217,8 @@ static inline int bdi_rw_congested(struc
 
 void clear_bdi_congested(struct backing_dev_info *bdi, int rw);
 void set_bdi_congested(struct backing_dev_info *bdi, int rw);
-long congestion_wait(int rw, long timeout);
-long congestion_wait_interruptible(int rw, long timeout);
+long congestion_wait(struct backing_dev_info *bdi, int rw, long timeout);
+long congestion_wait_interruptible(struct backing_dev_info *bdi, int rw, long timeout);
 
 #define bdi_cap_writeback_dirty(bdi) \
 	(!((bdi)->capabilities & BDI_CAP_NO_WRITEBACK))

95 lines not on this page · raw message

Index: linux-2.6-mm/mm/page-writeback.c
===================================================================
@@ -366,7 +366,7 @@ static void balance_dirty_pages(struct a
 			if (pages_written >= write_chunk)
 				break;		/* We've done our duty */
 		}
-		congestion_wait(WRITE, HZ/10);
+		congestion_wait(bdi, WRITE, HZ/10);
 	}
 
 	if (writeback_in_progress(bdi))

@@ -462,15 +462,17 @@ static void background_writeout(unsigned
 			global_page_state(NR_UNSTABLE_NFS) < background_thresh
 				&& min_pages <= 0)
 			break;
-		wbc.encountered_congestion = 0;
+		wbc.encountered_congestion = NULL;
 		wbc.nr_to_write = MAX_WRITEBACK_PAGES;
 		wbc.pages_skipped = 0;
 		writeback_inodes(&wbc);
 		min_pages -= MAX_WRITEBACK_PAGES - wbc.nr_to_write;
 		if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
 			/* Wrote less than expected */
-			congestion_wait(WRITE, HZ/10);
-			if (!wbc.encountered_congestion)
+			if (wbc.encountered_congestion)
+				congestion_wait(wbc.encountered_congestion,
+						WRITE, HZ/10);
+			else
 				break;
 		}
 	}

@@ -535,12 +537,13 @@ static void wb_kupdate(unsigned long arg
 			global_page_state(NR_UNSTABLE_NFS) +
 			(inodes_stat.nr_inodes - inodes_stat.nr_unused);
 	while (nr_to_write > 0) {
-		wbc.encountered_congestion = 0;
+		wbc.encountered_congestion = NULL;
 		wbc.nr_to_write = MAX_WRITEBACK_PAGES;
 		writeback_inodes(&wbc);
 		if (wbc.nr_to_write > 0) {
 			if (wbc.encountered_congestion)
-				congestion_wait(WRITE, HZ/10);
+				congestion_wait(wbc.encountered_congestion,
+					       	WRITE, HZ/10);
 			else
 				break;	/* All the old data is written */
 		}

@@ -698,7 +701,7 @@ int write_cache_pages(struct address_spa
 	int range_whole = 0;
 
 	if (wbc->nonblocking && bdi_write_congested(bdi)) {
-		wbc->encountered_congestion = 1;
+		wbc->encountered_congestion = bdi;
 		return 0;
 	}
 

@@ -760,7 +763,7 @@ retry:
 			if (ret || (--(wbc->nr_to_write) <= 0))
 				done = 1;
 			if (wbc->nonblocking && bdi_write_congested(bdi)) {
-				wbc->encountered_congestion = 1;
+				wbc->encountered_congestion = bdi;
 				done = 1;
 			}
 		}

Index: linux-2.6-mm/drivers/block/pktcdvd.c
===================================================================
@@ -2589,7 +2589,7 @@ static int pkt_make_request(request_queu
 		set_bdi_congested(&q->backing_dev_info, WRITE);
 		do {
 			spin_unlock(&pd->lock);
-			congestion_wait(WRITE, HZ);
+			congestion_wait(&q->backing_dev_info, WRITE, HZ);
 			spin_lock(&pd->lock);
 		} while(pd->bio_queue_size > pd->write_congestion_off);
 	}

Index: linux-2.6-mm/drivers/md/dm-crypt.c
===================================================================
@@ -640,8 +640,11 @@ static void process_write(struct crypt_i
 		 * may be gone already. */
 
 		/* out of memory -> run queues */
-		if (remaining)
-			congestion_wait(WRITE, HZ/100);
+		if (remaining) {
+			struct backing_dev_info *bdi =
+				&io->target->table->md->queue->backing_dev_info;
+			congestion_wait(bdi, WRITE, HZ/100);
+		}
 	}
 }
 

Index: linux-2.6-mm/fs/fat/file.c
===================================================================
@@ -118,8 +118,10 @@ static int fat_file_release(struct inode
 {
 	if ((filp->f_mode & FMODE_WRITE) &&
 	     MSDOS_SB(inode->i_sb)->options.flush) {
+		struct backing_dev_info *bdi =
+			inode->i_mapping->backing_dev_info;
 		fat_flush_inodes(inode->i_sb, inode, NULL);
-		congestion_wait(WRITE, HZ/10);
+		congestion_wait(bdi, WRITE, HZ/10);
 	}
 	return 0;
 }

Index: linux-2.6-mm/fs/reiserfs/journal.c
===================================================================
@@ -970,8 +970,11 @@ int reiserfs_async_progress_wait(struct 
 {
 	DEFINE_WAIT(wait);
 	struct reiserfs_journal *j = SB_JOURNAL(s);
-	if (atomic_read(&j->j_async_throttle))
-		congestion_wait(WRITE, HZ / 10);
+	if (atomic_read(&j->j_async_throttle)) {
+		struct backing_dev_info *bdi =
+			blk_get_backing_dev_info(j->j_dev_bd);
+		congestion_wait(bdi, WRITE, HZ / 10);
+	}
 	return 0;
 }
 

Index: linux-2.6-mm/include/linux/writeback.h
===================================================================
@@ -54,11 +54,12 @@ struct writeback_control {
 	loff_t range_end;
 
 	unsigned nonblocking:1;		/* Don't get stuck on request queues */
-	unsigned encountered_congestion:1; /* An output: a queue is full */
 	unsigned for_kupdate:1;		/* A kupdate writeback */
 	unsigned for_reclaim:1;		/* Invoked from the page allocator */
 	unsigned for_writepages:1;	/* This is a writepages() call */
 	unsigned range_cyclic:1;	/* range_start is cyclic */
+
+	struct backing_dev_info *encountered_congestion; /* An output: a queue is full */
 };
 
 /*

Index: linux-2.6-mm/fs/cifs/file.c
===================================================================
@@ -1143,7 +1143,7 @@ static int cifs_writepages(struct addres
 	 * If it is, we should test it again after we do I/O
 	 */
 	if (wbc->nonblocking && bdi_write_congested(bdi)) {
-		wbc->encountered_congestion = 1;
+		wbc->encountered_congestion = bdi;
 		kfree(iov);
 		return 0;
 	}

Index: linux-2.6-mm/fs/ext4/writeback.c
===================================================================
@@ -782,7 +782,7 @@ int ext4_wb_writepages(struct address_sp
 #ifdef EXT4_WB_STATS
 				atomic_inc(&EXT4_SB(inode->i_sb)->s_wb_congested);
 #endif
-				wbc->encountered_congestion = 1;
+				wbc->encountered_congestion = bdi;
 				done = 1;
 			}
 		}

Index: linux-2.6-mm/fs/fs-writeback.c
===================================================================
@@ -349,7 +349,7 @@ int generic_sync_sb_inodes(struct super_
 		}
 
 		if (wbc->nonblocking && bdi_write_congested(bdi)) {
-			wbc->encountered_congestion = 1;
+			wbc->encountered_congestion = bdi;
 			if (!sb_is_blkdev_sb(sb))
 				break;		/* Skip a congested fs */
 			list_move(&inode->i_list, &sb->s_dirty);

Index: linux-2.6-mm/fs/reiser4/vfs_ops.c
===================================================================
@@ -169,7 +169,7 @@ void reiser4_writeout(struct super_block
 		if (wbc->nonblocking &&
 		    bdi_write_congested(mapping->backing_dev_info)) {
 			blk_run_address_space(mapping);
-			wbc->encountered_congestion = 1;
+			wbc->encountered_congestion = mapping->backing_dev_info;
 			break;
 		}
 		repeats++;

Index: linux-2.6-mm/fs/xfs/linux-2.6/xfs_aops.c
===================================================================
@@ -777,7 +777,7 @@ xfs_convert_page(
 			bdi = inode->i_mapping->backing_dev_info;
 			wbc->nr_to_write--;
 			if (bdi_write_congested(bdi)) {
-				wbc->encountered_congestion = 1;
+				wbc->encountered_congestion = bdi;
 				done = 1;
 			} else if (wbc->nr_to_write <= 0) {
 				done = 1;

Index: linux-2.6-mm/mm/vmscan.c
===================================================================
@@ -71,7 +71,7 @@ struct scan_control {
 
 	int order;
 
-	int encountered_congestion;
+	struct backing_dev_info *encountered_congestion;
 };
 
 /*

@@ -376,7 +376,7 @@ static pageout_t pageout(struct page *pa
 			bdi = mapping->backing_dev_info;
 
 		if (bdi_congested(bdi, WRITE))
-			sc->encountered_congestion = 1;
+			sc->encountered_congestion = bdi;
 
 		SetPageReclaim(page);
 		res = mapping->a_ops->writepage(page, &wbc);

@@ -1153,7 +1153,7 @@ unsigned long try_to_free_pages(struct z
 
 	for (priority = DEF_PRIORITY; priority >= 0; priority--) {
 		sc.nr_scanned = 0;
-		sc.encountered_congestion = 0;
+		sc.encountered_congestion = NULL;
 		if (!priority)
 			disable_swap_token();
 		nr_reclaimed += shrink_zones(priority, zones, &sc);

@@ -1183,7 +1183,8 @@ unsigned long try_to_free_pages(struct z
 
 		/* Take a nap, wait for some writeback to complete */
 		if (sc.encountered_congestion)
-			congestion_wait(WRITE, HZ/10);
+			congestion_wait(sc.encountered_congestion,
+					WRITE, HZ/10);
 	}
 	/* top priority shrink_caches still had more to do? don't OOM, then */
 	if (!sc.all_unreclaimable)

@@ -1263,7 +1264,7 @@ loop_again:
 		int end_zone = 0;	/* Inclusive.  0 = ZONE_DMA */
 		unsigned long lru_pages = 0;
 
-		sc.encountered_congestion = 0;
+		sc.encountered_congestion = NULL;
 		/* The swap token gets in the way of swapout... */
 		if (!priority)
 			disable_swap_token();

@@ -1352,7 +1353,8 @@ loop_again:
 		 * another pass across the zones.
 		 */
 		if (sc.encountered_congestion)
-			congestion_wait(WRITE, HZ/10);
+			congestion_wait(sc.encountered_congestion,
+					WRITE, HZ/10);
 
 		/*
 		 * We do this so kswapd doesn't build up large priorities for

@@ -1594,7 +1596,7 @@ unsigned long shrink_all_memory(unsigned
 			unsigned long nr_to_scan = nr_pages - ret;
 
 			sc.nr_scanned = 0;
-			sc.encountered_congestion = 0;
+			sc.encountered_congestion = NULL;
 			ret += shrink_all_zones(nr_to_scan, prio, pass, &sc);
 			if (ret >= nr_pages)
 				goto out;

@@ -1607,7 +1609,8 @@ unsigned long shrink_all_memory(unsigned
 				goto out;
 
 			if (sc.encountered_congestion)
-				congestion_wait(WRITE, HZ / 10);
+				congestion_wait(sc.encountered_congestion,
+						WRITE, HZ / 10);
 		}
 	}
 

Index: linux-2.6-mm/fs/nfs/write.c
===================================================================
@@ -567,7 +567,7 @@ static int nfs_wait_on_write_congestion(
 		sigset_t oldset;
 
 		rpc_clnt_sigmask(clnt, &oldset);
-		ret = congestion_wait_interruptible(WRITE, HZ/10);
+		ret = congestion_wait_interruptible(bdi, WRITE, HZ/10);
 		rpc_clnt_sigunmask(clnt, &oldset);
 		if (ret == -ERESTARTSYS)
 			break;