per device dirty throttling

Index: linux-2.6/block/ll_rw_blk.c
===================================================================
@@ -211,6 +211,7 @@ void blk_queue_make_request(request_queu
 	q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
 	q->backing_dev_info.state = 0;
 	q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
+	bdi_stat_init(&q->backing_dev_info);
 	blk_queue_max_sectors(q, SAFE_MAX_SECTORS);
 	blk_queue_hardsect_size(q, 512);
 	blk_queue_dma_alignment(q, 511);

100 lines not on this page · raw message

106 lines not on this page · raw message

Index: linux-2.6/drivers/block/rd.c
===================================================================
@@ -421,6 +421,8 @@ static int __init rd_init(void)
 	int i;
 	int err = -ENOMEM;
 
+	bdi_stat_init(&rd_file_backing_dev_info);
+
 	if (rd_blocksize > PAGE_SIZE || rd_blocksize < 512 ||
 			(rd_blocksize & (rd_blocksize-1))) {
 		printk("RAMDISK: wrong blocksize %d, reverting to defaults\n",

Index: linux-2.6/drivers/char/mem.c
===================================================================
@@ -988,6 +988,8 @@ static int __init chr_dev_init(void)
 			      MKDEV(MEM_MAJOR, devlist[i].minor),
 			      devlist[i].name);
 
+	bdi_stat_init(&zero_bdi);
+
 	return 0;
 }
 

Index: linux-2.6/fs/char_dev.c
===================================================================
@@ -545,6 +545,7 @@ static struct kobject *base_probe(dev_t 
 void __init chrdev_init(void)
 {
 	cdev_map = kobj_map_init(base_probe, &chrdevs_lock);
+	bdi_stat_init(&directly_mappable_cdev_bdi);
 }
 
 

Index: linux-2.6/fs/buffer.c
===================================================================
@@ -732,6 +732,7 @@ int __set_page_dirty_buffers(struct page
 	if (page->mapping) {	/* Race with truncate? */
 		if (mapping_cap_account_dirty(mapping)) {
 			__inc_zone_page_state(page, NR_FILE_DIRTY);
+			__inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 			task_io_account_write(PAGE_CACHE_SIZE);
 		}
 		radix_tree_tag_set(&mapping->page_tree,

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -766,6 +766,7 @@ int __set_page_dirty_nobuffers(struct pa
 			BUG_ON(mapping2 != mapping);
 			if (mapping_cap_account_dirty(mapping)) {
 				__inc_zone_page_state(page, NR_FILE_DIRTY);
+				__inc_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 				task_io_account_write(PAGE_CACHE_SIZE);
 			}
 			radix_tree_tag_set(&mapping->page_tree,

@@ -892,6 +893,7 @@ int clear_page_dirty_for_io(struct page 
 			set_page_dirty(page);
 		if (TestClearPageDirty(page)) {
 			dec_zone_page_state(page, NR_FILE_DIRTY);
+			dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 			return 1;
 		}
 		return 0;

Index: linux-2.6/mm/truncate.c
===================================================================
@@ -71,6 +71,7 @@ void cancel_dirty_page(struct page *page
 		struct address_space *mapping = page->mapping;
 		if (mapping && mapping_cap_account_dirty(mapping)) {
 			dec_zone_page_state(page, NR_FILE_DIRTY);
+			dec_bdi_stat(mapping->backing_dev_info, BDI_DIRTY);
 			if (account_size)
 				task_io_account_cancelled_write(account_size);
 		}

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -23,6 +23,7 @@ enum bdi_state {
 };
 
 enum bdi_stat_item {
+	BDI_DIRTY,
 	NR_BDI_STAT_ITEMS
 };
 

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -912,10 +912,12 @@ int test_clear_page_writeback(struct pag
 
 		write_lock_irqsave(&mapping->tree_lock, flags);
 		ret = TestClearPageWriteback(page);
-		if (ret)
+		if (ret) {
 			radix_tree_tag_clear(&mapping->page_tree,
 						page_index(page),
 						PAGECACHE_TAG_WRITEBACK);
+			__dec_bdi_stat(mapping->backing_dev_info, BDI_WRITEBACK);
+		}
 		write_unlock_irqrestore(&mapping->tree_lock, flags);
 	} else {
 		ret = TestClearPageWriteback(page);

@@ -933,10 +935,12 @@ int test_set_page_writeback(struct page 
 
 		write_lock_irqsave(&mapping->tree_lock, flags);
 		ret = TestSetPageWriteback(page);
-		if (!ret)
+		if (!ret) {
 			radix_tree_tag_set(&mapping->page_tree,
 						page_index(page),
 						PAGECACHE_TAG_WRITEBACK);
+			__inc_bdi_stat(mapping->backing_dev_info, BDI_WRITEBACK);
+		}
 		if (!PageDirty(page))
 			radix_tree_tag_clear(&mapping->page_tree,
 						page_index(page),

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -24,6 +24,7 @@ enum bdi_state {
 
 enum bdi_stat_item {
 	BDI_DIRTY,
+	BDI_WRITEBACK,
 	NR_BDI_STAT_ITEMS
 };
 

Index: linux-2.6/fs/nfs/write.c
===================================================================
@@ -474,6 +474,7 @@ nfs_mark_request_commit(struct nfs_page 
 	nfsi->ncommit++;
 	spin_unlock(&nfsi->req_lock);
 	inc_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+	inc_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
 }
 #endif

@@ -545,6 +546,7 @@ static void nfs_cancel_commit_list(struc
 	while(!list_empty(head)) {
 		req = nfs_list_entry(head->next);
 		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+		dec_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 		nfs_list_remove_request(req);
 		nfs_inode_remove_request(req);
 		nfs_unlock_request(req);

@@ -1278,6 +1280,7 @@ nfs_commit_list(struct inode *inode, str
 		nfs_list_remove_request(req);
 		nfs_mark_request_commit(req);
 		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+		dec_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 		nfs_clear_page_writeback(req);
 	}
 	return -ENOMEM;

@@ -1302,6 +1305,7 @@ static void nfs_commit_done(struct rpc_t
 		req = nfs_list_entry(data->pages.next);
 		nfs_list_remove_request(req);
 		dec_zone_page_state(req->wb_page, NR_UNSTABLE_NFS);
+		dec_bdi_stat(req->wb_page->mapping->backing_dev_info, BDI_UNSTABLE);
 
 		dprintk("NFS: commit (%s/%Ld %d@%Ld)",
 			req->wb_context->dentry->d_inode->i_sb->s_id,

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -25,6 +25,7 @@ enum bdi_state {
 enum bdi_stat_item {
 	BDI_DIRTY,
 	BDI_WRITEBACK,
+	BDI_UNSTABLE,
 	NR_BDI_STAT_ITEMS
 };
 

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -26,6 +26,8 @@ enum bdi_stat_item {
 	BDI_DIRTY,
 	BDI_WRITEBACK,
 	BDI_UNSTABLE,
+	BDI_WRITEOUT,
+	BDI_WRITEOUT_TOTAL,
 	NR_BDI_STAT_ITEMS
 };
 

@@ -47,6 +49,12 @@ struct backing_dev_info {
 	void (*unplug_io_fn)(struct backing_dev_info *, struct page *);
 	void *unplug_io_data;
 
+	/*
+	 * data used for scaling the writeback cache
+	 */
+	spinlock_t lock;	/* protect the cycle count */
+	unsigned long cycles;	/* writeout cycles */
+
 	atomic_long_t bdi_stats[NR_BDI_STAT_ITEMS];
 #ifdef CONFIG_SMP
 	struct bdi_per_cpu_data pcd[NR_CPUS];

234 lines not on this page · raw message

Index: linux-2.6/mm/backing-dev.c
===================================================================
@@ -75,6 +75,9 @@ void bdi_stat_init(struct backing_dev_in
 {
 	int i;
 
+	spin_lock_init(&bdi->lock);
+	bdi->cycles = 0;
+
 	for (i = 0; i < NR_BDI_STAT_ITEMS; i++)
 		atomic_long_set(&bdi->bdi_stats[i], 0);
 

Index: linux-2.6/block/ll_rw_blk.c
===================================================================
@@ -3923,6 +3923,33 @@ static ssize_t queue_max_hw_sectors_show
 	return queue_var_show(max_hw_sectors_kb, (page));
 }
 
+static ssize_t queue_nr_dirty_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%lu\n", bdi_stat(&q->backing_dev_info, BDI_DIRTY));
+}
+
+static ssize_t queue_nr_writeback_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%lu\n", bdi_stat(&q->backing_dev_info, BDI_WRITEBACK));
+}
+
+static ssize_t queue_nr_unstable_show(struct request_queue *q, char *page)
+{
+	return sprintf(page, "%lu\n", bdi_stat(&q->backing_dev_info, BDI_UNSTABLE));
+}
+
+extern void get_writeout_scale(struct backing_dev_info *, int *, int *);
+
+static ssize_t queue_nr_cache_show(struct request_queue *q, char *page)
+{
+	int scale, div;
+
+	get_writeout_scale(&q->backing_dev_info, &scale, &div);
+	scale *= 1024;
+	scale /= div;
+
+	return sprintf(page, "%d\n", scale);
+}
 
 static struct queue_sysfs_entry queue_requests_entry = {
 	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },

@@ -3947,6 +3974,26 @@ static struct queue_sysfs_entry queue_ma
 	.show = queue_max_hw_sectors_show,
 };
 
+static struct queue_sysfs_entry queue_dirty_entry = {
+	.attr = {.name = "dirty_pages", .mode = S_IRUGO },
+	.show = queue_nr_dirty_show,
+};
+
+static struct queue_sysfs_entry queue_writeback_entry = {
+	.attr = {.name = "writeback_pages", .mode = S_IRUGO },
+	.show = queue_nr_writeback_show,
+};
+
+static struct queue_sysfs_entry queue_unstable_entry = {
+	.attr = {.name = "unstable_pages", .mode = S_IRUGO },
+	.show = queue_nr_unstable_show,
+};
+
+static struct queue_sysfs_entry queue_cache_entry = {
+	.attr = {.name = "cache_ratio", .mode = S_IRUGO },
+	.show = queue_nr_cache_show,
+};
+
 static struct queue_sysfs_entry queue_iosched_entry = {
 	.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
 	.show = elv_iosched_show,

@@ -3958,6 +4005,10 @@ static struct attribute *default_attrs[]
 	&queue_ra_entry.attr,
 	&queue_max_hw_sectors_entry.attr,
 	&queue_max_sectors_entry.attr,
+	&queue_dirty_entry.attr,
+	&queue_writeback_entry.attr,
+	&queue_unstable_entry.attr,
+	&queue_cache_entry.attr,
 	&queue_iosched_entry.attr,
 	NULL,
 };

Index: linux-2.6/include/linux/backing-dev.h
===================================================================
@@ -70,6 +70,12 @@ static inline void bdi_stat_add(long x, 
 	atomic_long_add(x, &bdi_stats[item]);
 }
 
+
+static inline unsigned long __global_bdi_stat(enum bdi_stat_item item)
+{
+	return atomic_long_read(&bdi_stats[item]);
+}
+
 /*
  * cannot be unsigned long and clip on 0.
  */

@@ -83,6 +89,12 @@ static inline unsigned long global_bdi_s
 	return x;
 }
 
+static inline unsigned long __bdi_stat(struct backing_dev_info *bdi,
+		enum bdi_stat_item item)
+{
+	return atomic_long_read(&bdi->bdi_stats[item]);
+}
+
 static inline unsigned long bdi_stat(struct backing_dev_info *bdi,
 		enum bdi_stat_item item)
 {

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -119,16 +119,13 @@ static int vm_cycle_shift __read_mostly;
 
 /*
  * Sync up the per BDI average to the global cycle.
- *
- * NOTE: we mask out the MSB of the cycle count because bdi_stats really are
- * not unsigned long. (see comment in backing_dev.h)
  */
 static void bdi_writeout_norm(struct backing_dev_info *bdi)
 {
 	int bits = vm_cycle_shift;
 	unsigned long cycle = 1UL << bits;
-	unsigned long mask = ~(cycle - 1) | (1UL << BITS_PER_LONG-1);
-	unsigned long total = global_bdi_stat(BDI_WRITEOUT_TOTAL) << 1;
+	unsigned long mask = ~(cycle - 1);
+	unsigned long total = __global_bdi_stat(BDI_WRITEOUT_TOTAL) << 1;
 	unsigned long flags;
 
 	if ((bdi->cycles & mask) == (total & mask))

@@ -136,7 +133,7 @@ static void bdi_writeout_norm(struct bac
 
 	spin_lock_irqsave(&bdi->lock, flags);
 	while ((bdi->cycles & mask) != (total & mask)) {
-		unsigned long half = bdi_stat(bdi, BDI_WRITEOUT) / 2;
+		unsigned long half = __bdi_stat(bdi, BDI_WRITEOUT) / 2;
 
 		mod_bdi_stat(bdi, BDI_WRITEOUT, -half);
 		bdi->cycles += cycle;

@@ -158,13 +155,13 @@ static void bdi_writeout_inc(struct back
 void get_writeout_scale(struct backing_dev_info *bdi, int *scale, int *div)
 {
 	int bits = vm_cycle_shift - 1;
-	unsigned long total = global_bdi_stat(BDI_WRITEOUT_TOTAL);
+	unsigned long total = __global_bdi_stat(BDI_WRITEOUT_TOTAL);
 	unsigned long cycle = 1UL << bits;
 	unsigned long mask = cycle - 1;
 
 	if (bdi_cap_writeback_dirty(bdi)) {
 		bdi_writeout_norm(bdi);
-		*scale = bdi_stat(bdi, BDI_WRITEOUT);
+		*scale = __bdi_stat(bdi, BDI_WRITEOUT);
 	} else
 		*scale = 0;
 

@@ -234,19 +231,29 @@ get_dirty_limits(long *pbackground, long
 	*pdirty = dirty;
 
 	if (mapping) {
+		struct backing_dev_info *bdi = mapping->backing_dev_info;
+		long reserve;
 		long long tmp = dirty;
 		int scale, div;
 
-		get_writeout_scale(mapping->backing_dev_info, &scale, &div);
-
-		if (scale > div)
-			scale = div;
+		get_writeout_scale(bdi, &scale, &div);
 
-		tmp = (tmp * 122) >> 7; /* take ~95% of total dirty value */
 		tmp *= scale;
 		do_div(tmp, div);
 
-		*pbdi_dirty = (long)tmp;
+		reserve = dirty -
+			(global_bdi_stat(BDI_DIRTY) +
+			 global_bdi_stat(BDI_WRITEBACK) +
+			 global_bdi_stat(BDI_UNSTABLE));
+
+		if (reserve < 0)
+			reserve = 0;
+
+		reserve += bdi_stat(bdi, BDI_DIRTY) +
+			bdi_stat(bdi, BDI_WRITEBACK) +
+			bdi_stat(bdi, BDI_UNSTABLE);
+
+		*pbdi_dirty = min((long)tmp, reserve);
 	}
 }
 

@@ -627,7 +634,7 @@ void __init page_writeback_init(void)
 	mod_timer(&wb_timer, jiffies + dirty_writeback_interval);
 	writeback_set_ratelimit();
 	register_cpu_notifier(&ratelimit_nb);
-	vm_cycle_shift = 3 + ilog2(int_sqrt(vm_total_pages));
+	vm_cycle_shift = ilog2(vm_total_pages);
 }
 
 /**

Index: linux-2.6/fs/nfs/client.c
===================================================================
@@ -657,6 +657,7 @@ static void nfs_server_set_fsinfo(struct
 		server->rsize = NFS_MAX_FILE_IO_SIZE;
 	server->rpages = (server->rsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
 	server->backing_dev_info.ra_pages = server->rpages * NFS_MAX_READAHEAD;
+	bdi_stat_init(&server->backing_dev_info);
 
 	if (server->wsize > max_rpc_payload)
 		server->wsize = max_rpc_payload;

Index: linux-2.6/mm/page-writeback.c
===================================================================
@@ -114,6 +114,13 @@ static void background_writeout(unsigned
  *
  * Furthermore, when we choose thresh to be 2^n it can be written in terms of
  * binary operations and wraparound artifacts disappear.
+ *
+ * Also note that this yields a natural counter of the elapsed periods:
+ *
+ *   i / thresh
+ *
+ * Its monotonous increasing property can be applied to mitigate the wrap-
+ * around issue.
  */
 static int vm_cycle_shift __read_mostly;
 

@@ -125,19 +132,25 @@ static void bdi_writeout_norm(struct bac
 	int bits = vm_cycle_shift;
 	unsigned long cycle = 1UL << bits;
 	unsigned long mask = ~(cycle - 1);
-	unsigned long total = __global_bdi_stat(BDI_WRITEOUT_TOTAL) << 1;
+	unsigned long global_cycle =
+		(__global_bdi_stat(BDI_WRITEOUT_TOTAL) << 1) & mask;
 	unsigned long flags;
 
-	if ((bdi->cycles & mask) == (total & mask))
+	if ((bdi->cycles & mask) == global_cycle)
 		return;
 
 	spin_lock_irqsave(&bdi->lock, flags);
-	while ((bdi->cycles & mask) != (total & mask)) {
-		unsigned long half = __bdi_stat(bdi, BDI_WRITEOUT) / 2;
+	while ((bdi->cycles & mask) != global_cycle) {
+		unsigned long val = __bdi_stat(bdi, BDI_WRITEOUT);
+		unsigned long half = (val + 1) / 2;
+
+		if (!val)
+			break;
 
 		mod_bdi_stat(bdi, BDI_WRITEOUT, -half);
 		bdi->cycles += cycle;
 	}
+	bdi->cycles = global_cycle;
 	spin_unlock_irqrestore(&bdi->lock, flags);
 }
 

@@ -146,10 +159,10 @@ static void bdi_writeout_inc(struct back
 	if (!bdi_cap_writeback_dirty(bdi))
 		return;
 
+	bdi_writeout_norm(bdi);
+
 	__inc_bdi_stat(bdi, BDI_WRITEOUT);
 	__inc_bdi_stat(bdi, BDI_WRITEOUT_TOTAL);
-
-	bdi_writeout_norm(bdi);
 }
 
 void get_writeout_scale(struct backing_dev_info *bdi, int *scale, int *div)

Index: linux-2.6/fs/fuse/inode.c
===================================================================
@@ -413,6 +413,7 @@ static struct fuse_conn *new_conn(void)
 		atomic_set(&fc->num_waiting, 0);
 		fc->bdi.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
 		fc->bdi.unplug_io_fn = default_unplug_io_fn;
+		bdi_stat_init(&fc->bdi);
 		fc->reqctr = 0;
 		fc->blocked = 1;
 		get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));