Skip to contentSkip to conversation

bcache: ver 3

RFC PATCH In review Kent Overstreet 3 participants 8 messages Apr 23, 2010 – Apr 25, 2010

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

RFC PATCH 1/1 bcache: ver 3

The core code is getting close to stable and feature complete. Garbage collection is done, and locking is mostly done (I'm not refcounting data buckets yet, and it does need to be stared at a whole lot more but it passes lockdep and seems to be working).

Basic bucket priority calculation is done (right now it acts as an LRU); you can fill the cache up and everything works like it ought to. And, I've yet to see it return bad data; I think it's not all that far off from being useful.

The one thing that has me at a loss right now is the performance isn't where it should be, and for no obvious reason. The problem is most succinctly demonstrated thusly:

With cache, no cache misses:

root@utumno:~# mount -o ro /dev/sdb /mnt/
root@utumno:~# cd /mnt/
root@utumno:/mnt# time find > /dev/null 

real 0m29.618s
user 0m0.000s
sys 0m0.840s

No cache device:

root@utumno:~# mount -o ro /dev/sdb /mnt/
root@utumno:~# cd /mnt/
root@utumno:/mnt# time find > /dev/null 

real 0m1.597s
user 0m0.050s
sys 0m1.240s

This is a vm, so the wall clock time in the latter case doesn't mean much, it's all in the hosts's cache. But the former case is bewildering; the cache device lives on a tmpfs so it shouldn't be waiting on IO, and cpu time is less than the uncached case. Clearly it's waiting on something, but I haven't been able to figure out what the heck to trace; there's no lock contention, blktrace shows nothing unexpected, I keep running into dead ends. Any pointers here would be much appreciated.

Besides that, there shouldn't be anything complex left before I add write support, and that should be pretty straightforward too.

This patch does have some changes to the generic bio code for completion of split bios, I'm going to split it out and submit it seperately after I've gone over it more and it's actually been tested.

Any comments/review much appreciated.

 block/Kconfig       |    5 +
 block/Makefile      |    2 +
 block/bcache.c      | 2337 +++++++++++++++++++++++++++++++++++++++++++++++++++
 block/blk-core.c    |    7 +-
 fs/bio.c            |   32 +-
 include/linux/bio.h |    2 +
 include/linux/fs.h  |    5 +
 7 files changed, 2386 insertions(+), 4 deletions(-)
block/Kconfig +5 -0
index f9e89f4..85adf8d 100644
@@ -100,6 +100,11 @@ config DEBUG_BLK_CGROUP
 	in the blk group which can be used by cfq for tracing various
 	group related activity.
 
+config BLK_CACHE
+	tristate "Block device as cache"
+	select SLOW_WORK
+	default m
+
 endif # BLOCK
 
 config BLOCK_COMPAT
block/Makefile +2 -0
index cb2d515..e9b5fc0 100644
@@ -15,3 +15,5 @@ obj-$(CONFIG_IOSCHED_CFQ)	+= cfq-iosched.o
 
 obj-$(CONFIG_BLOCK_COMPAT)	+= compat_ioctl.o
 obj-$(CONFIG_BLK_DEV_INTEGRITY)	+= blk-integrity.o
+
+obj-$(CONFIG_BLK_CACHE)		+= bcache.o
block/bcache.c new file +2337 -0

2,337 lines not on this page · raw message

block/blk-core.c +5 -2
index 9fe174d..41b4d21 100644
@@ -1405,7 +1405,7 @@ static inline void __generic_make_request(struct bio *bio)
 {
 	struct request_queue *q;
 	sector_t old_sector;
-	int ret, nr_sectors = bio_sectors(bio);
+	int ret = 1, nr_sectors = bio_sectors(bio);
 	dev_t old_dev;
 	int err = -EIO;
 
@@ -1478,7 +1478,10 @@ static inline void __generic_make_request(struct bio *bio)
 
 		trace_block_bio_queue(q, bio);
 
-		ret = q->make_request_fn(q, bio);
+		if (bio->bi_bdev->bd_cache_fn)
+			ret = bio->bi_bdev->bd_cache_fn(q, bio);
+		if (ret)
+			ret = q->make_request_fn(q, bio);
 	} while (ret);
 
 	return;
fs/bio.c +30 -2
index e7bf6ca..5ad2403 100644
@@ -257,6 +257,7 @@ void bio_init(struct bio *bio)
 	bio->bi_flags = 1 << BIO_UPTODATE;
 	bio->bi_comp_cpu = -1;
 	atomic_set(&bio->bi_cnt, 1);
+	atomic_set(&bio->bi_remaining, 1);
 }
 EXPORT_SYMBOL(bio_init);
 
@@ -1422,13 +1423,40 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
  **/
 void bio_endio(struct bio *bio, int error)
 {
+	bool put = true;
+	struct bio *p;
+	bio_end_io_t *f = bio->bi_end_io;
+
 	if (error)
 		clear_bit(BIO_UPTODATE, &bio->bi_flags);
 	else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
 		error = -EIO;
 
-	if (bio->bi_end_io)
-		bio->bi_end_io(bio, error);
+	if (atomic_dec_and_test(&bio->bi_remaining))
+		goto end_io;
+
+	if (atomic_read(&bio->bi_remaining) < 0) {
+		WARN(1, "bio incorrectly initialized");
+		goto end_io;
+	}
+
+	if (error) {
+		do
+			f = bio->bi_end_io;
+		while (cmpxchg(&bio->bi_end_io, f, NULL) != f);
+		put = false;
+		goto end_io;
+	}
+
+	return;
+end_io:
+	if (f)
+		f(bio, error);
+	else if (test_bit(BIO_CLONED, &bio->bi_flags)) {
+		p = bio->bi_private;
+		bio_put(bio);
+		bio_endio(p, error);
+	}
 }
 EXPORT_SYMBOL(bio_endio);
 
include/linux/bio.h +2 -0
index 7fc5606..c3ee4c3 100644
@@ -94,6 +94,8 @@ struct bio {
 
 	struct bio_vec		*bi_io_vec;	/* the actual vec list */
 
+	atomic_t		bi_remaining;	/* split count */
+
 	bio_end_io_t		*bi_end_io;
 
 	void			*bi_private;
include/linux/fs.h +5 -0
index 39d57bc..8aba38c 100644
@@ -514,6 +514,8 @@ enum positive_aop_returns {
 struct page;
 struct address_space;
 struct writeback_control;
+struct bio;
+struct request_queue;
 
 struct iov_iter {
 	const struct iovec *iov;
@@ -664,6 +666,9 @@ struct block_device {
 	int			bd_invalidated;
 	struct gendisk *	bd_disk;
 	struct list_head	bd_list;
+
+	int (*bd_cache_fn)(struct request_queue *q, struct bio *bio);
+	char			bd_cache_identifier;
 	/*
 	 * Private data.  You must have bd_claim'ed the block_device
 	 * to use this.  NOTE:  bd_claim allows an owner to claim
#1 lore raw

block/Kconfig config DEBUG_BLK_CGROUP · line 106

Quoted 12 lines · On Fri, 23 Apr 2010 11:41:07 -0800 Kent Overstreet wrote:
block/Kconfig +4 -0
index f9e89f4..85adf8d 100644
@@ -100,6 +100,11 @@ config DEBUG_BLK_CGROUP
 	in the blk group which can be used by cfq for tracing various
 	group related activity.
 
+config BLK_CACHE
+	tristate "Block device as cache"
+	select SLOW_WORK
+	default m

help
<some explanatory help text>
please. The tristate prompt didn't do anything for me.

block/Kconfig config DEBUG_BLK_CGROUP · line 108

Quoted 2 lines
block/Kconfig +1 -0
+
 endif # BLOCK

---
~Randy
*** Remember to use Documentation/SubmitChecklist when testing your code ***

#2 lore raw
Quoted 3 lines · Randy Dunlap

help
<some explanatory help text>
please. The tristate prompt didn't do anything for me.

Did the prompt not show up at all? That's odd, I wouldn't have thought help text would have anything to do with that - I just added help to the kconfig, though. I'll have it up in a git repository soon, I don't suppose I should be posting a 2.4k line patch multiple times per day...

#3 lore raw
Quoted 9 lines · Kent Overstreet
Quoted 3 lines · Randy Dunlap

help
<some explanatory help text>
please. The tristate prompt didn't do anything for me.

Did the prompt not show up at all? That's odd, I wouldn't have thought help text would have anything to do with that - I just added help to the kconfig, though. I'll have it up in a git repository soon, I don't suppose I should be posting a 2.4k line patch multiple times per day...

I'm just saying that it needs help text -- the prompt doesn't say enough.

Signature · 2 lines
~Randy
*** Remember to use Documentation/SubmitChecklist when testing your code ***
#4 lore raw
Quoted 6 lines · Randy Dunlap
Quoted 5 lines · Kent Overstreet
Quoted 3 lines · Randy Dunlap

help
<some explanatory help text>
please. The tristate prompt didn't do anything for me.

Quoted 7 lines
Quoted 4 lines

Did the prompt not show up at all? That's odd, I wouldn't have thought help text would have anything to do with that - I just added help to the kconfig, though. I'll have it up in a git repository soon, I don't suppose I should be posting a 2.4k line patch multiple times per day...

I'm just saying that it needs help text -- the prompt doesn't say enough.

Ah, right. It still needs real documentation in general, that should probably be a higher priority.

(Somewhat messy, will be rewritten) repo is up at
http://modzer0.cs.uaf.edu/~kent/linux-bcache.git, kconfig help is in. If there's something in particular you think ought to be documented for understanding the code, I'll do that now. End user documentation I was planning on waiting a bit on as I'm sure stuff is going to have to change.

#5 lore raw
In reply to Kent Overstreet
Quoted 15 lines · On Fri, 23 Apr 2010 11:41:07 -0800 Kent Overstreet wrote:
 block/Kconfig       |    5 +
 block/Makefile      |    2 +
 block/bcache.c      | 2337 +++++++++++++++++++++++++++++++++++++++++++++++++++
 block/blk-core.c    |    7 +-
 fs/bio.c            |   32 +-
 include/linux/bio.h |    2 +
 include/linux/fs.h  |    5 +
 7 files changed, 2386 insertions(+), 4 deletions(-)
block/bcache.c new file +0 -0
new file mode 100644
index 0000000..3bf3fff

block/bcache.c line 2

Quoted 2 lines
block/bcache.c new file +2 -0
@@ -0,0 +1,2337 @@
+
+#define label(l, foo)	if (0) { l: foo; }

I'd prefer that macro to go away.

block/bcache.c line 234

Quoted 10 lines
block/bcache.c new file +10 -0
+
+/*
+ * key: 8 bit device, 56 bit offset
+ * value: 8 bit generation, 16 bit length, 40 bit offset
+ * All units are in sectors
+ */
+
+static inline struct btree_key *node(struct btree_key *d[], int i)
+{
+	return d[i / keys_per_page] + (i % keys_per_page);

That builds OK on i386? or does it need udivdi3() and/or umoddi3()?

block/bcache.c line 267

Quoted 6 lines
block/bcache.c new file +5 -0
+}
 
+static int lookup_dev(struct cache_device *c, struct bio *bio)
+{
+	int dev;
+	for (dev = 0; dev < 256; dev++)

Use a macro for 256.. in lots of places.

Quoted 19 lines
block/bcache.c new file +17 -0
+		if (c->devices[dev] == bio->bi_bdev->bd_cache_identifier)
+			break;
+
+	if (dev == 256)
+		printk(KERN_DEBUG "bcache: unknown device %i",
+		       bio->bi_bdev->bd_cache_identifier);
+
+	return dev;
+}
 
 
+static void register_dev(const char *buffer, size_t size)
+{
+	int i;
+	char *path;
+	unsigned char uuid[16];
+	struct block_device *bdev;
+	struct list_head *l;
+

block/bcache.c

Quoted 10 lines
block/bcache.c new file +10 -0
+	i = parse_uuid(buffer, &uuid[0]);
+
+	if (i < 4) {
+		printk(KERN_DEBUG "bcache: Bad uuid");
+		return;
+	}
+
+	path = kmalloc(size + 1 - i, GFP_KERNEL);
+	if (!path) {
+		printk(KERN_DEBUG "bcache: kmalloc error");

bcache: cannot allocate memory
"kmalloc error" sounds like kmalloc() had an internal error.

Quoted 22 lines
block/bcache.c new file +22 -0
+		return;
+	}
+	strcpy(path, skip_spaces(buffer + i));
+	bdev = lookup_bdev(strim(path));
+
+	if (IS_ERR(bdev)) {
+		printk(KERN_DEBUG "bcache: Failed to open %s", path);
+		goto out;
+	}
+
+	for (i = 0; i < 256; i++) {
+		if (is_zero(&uuids[i*16], 16))
+			break;
+
+		if (!memcmp(&uuids[i*16], uuid, 16)) {
+			printk(KERN_DEBUG "bcache: %s already registered", path);
+			bdput(bdev);
+			goto out;
+		}
+	}
+	memcpy(&uuids[i*16], uuid, 16);
+	bdev->bd_cache_identifier = i;

block/bcache.c line 1989

Quoted 10 lines
block/bcache.c new file +10 -0
+	/*devices[i] = bdev->bd_disk;*/
+
+	list_for_each(l, &cache_devices)
+		register_dev_on_cache(list_entry(l, struct cache_device, list), i);
+
+	bdev->bd_cache_fn = request_hook;
+	printk(KERN_DEBUG "bcache: Caching %s index %i", path, i);
+out:
+	kfree(path);
+}

Need to document the sysfs interfaces:

Quoted 70 lines
block/bcache.c new file +68 -0
+static ssize_t show_cache(struct kobject *kobj, struct attribute *attr,
+			  char *buffer)
+{
+	struct cache_device *c = container_of(kobj, struct cache_device, kobj);
+
+	sysfs_print(bucket_size, "%i\n", c->sb.bucket_size << 9);
+	sysfs_print(buckets_used, "%lli\n", c->sb.first_free_bucket);
+	sysfs_print(buckets_free, "%lli\n", c->sb.nbuckets -
+		    c->sb.first_free_bucket);
+	sysfs_print(nbuckets,	"%lli\n", c->sb.nbuckets);
+	sysfs_print(cache_hits, "%lu\n", c->cache_hits);
+	sysfs_print(tree_depth, "%u\n", c->sb.btree_level);
+	sysfs_print(min_priority, "%u\n", c->heap[0] ? c->heap[0]->priority : 0);
+	return 0;
+}
 
 
+static void register_cache(const char *buffer, size_t size)
+{
+	char *err = NULL, *path, b[BDEVNAME_SIZE];
+	int i;
+	struct cache_device *c;
+	struct search_context s, *sp = &s;
+
+	static struct attribute *files[] = {
+		&sysfs_unregister,
+		&sysfs_bucket_size,
+		&sysfs_buckets_used,
+		&sysfs_buckets_free,
+		&sysfs_nbuckets,
+		&sysfs_cache_hits,
+		&sysfs_tree_depth,
+		&sysfs_min_priority,
+		NULL
+	};
+	static const struct sysfs_ops ops = {
+		.show = show_cache,
+		.store = store_cache
+	};
+	static struct kobj_type cache_obj = {
+		.release = unregister_cache,
+		.sysfs_ops = &ops,
+		.default_attrs = files
+	};
+
+	if (!try_module_get(THIS_MODULE))
+		return;
+
+	path = kmalloc(size + 1, GFP_KERNEL);
+	strcpy(path, skip_spaces(buffer));
+
+	err = "Insufficient memory";
+	if (!(c = kzalloc(sizeof(*c), GFP_KERNEL)))
+		goto err;
+
+	err = "Failed to open cache device";
+	c->bdev = open_bdev_exclusive(strim(path), FMODE_READ|FMODE_WRITE, c);
+	if (IS_ERR(c->bdev)) {
+		if (c->bdev == ERR_PTR(EBUSY))
+			err = "Device busy";
+		goto err;
+	}
+
+	set_blocksize(c->bdev, 4096);
+
+	err = "IO error";
+	if (!(c->sb_bh = __bread(c->bdev, 1, PAGE_SIZE)))
+		goto err;
+
+	if (!(c->uuids = __bread(c->bdev, 2, PAGE_SIZE)))

block/bcache.c

Quoted 10 lines
block/bcache.c new file +10 -0
+		goto err;
+
+	if ((err = read_super(c)))
+		goto err;
+
+	c->free_size = 1;
+	while (c->free_size << 7 < c->sb.nbuckets)
+		c->free_size <<= 1;
+
+	err = "vmalloc error";
		"cannot vmalloc memory";
Quoted 62 lines
block/bcache.c new file +62 -0
+	c->heap		= vmalloc(c->sb.nbuckets * sizeof(struct bucket *));
+	c->buckets	= vmalloc(c->sb.nbuckets * sizeof(struct bucket));
+	c->freelist	= vmalloc(c->free_size-- * sizeof(long));
+	if (!c->heap || !c->buckets || !c->freelist)
+		goto err;
+
+	memset(c->heap,	   0, c->sb.nbuckets * sizeof(struct bucket *));
+	memset(c->buckets, 0, c->sb.nbuckets * sizeof(struct bucket));
+
+	spin_lock_init(&c->bucket_lock);
+	spin_lock_init(&c->alloc_lock);
+	init_rwsem(&c->gc_lock);
+
+	INIT_LIST_HEAD(&c->lru);
+	c->btree_buckets_cached = 10;
+
+	load_priorities(c);
+
+	memset(&s, 0, sizeof(s));
+	c->root = get_bucket(c, c->sb.btree_root, c->sb.btree_level, false, &sp);
+	c->buckets[sector_to_bucket(c->root->offset)].priority = ~0;
+
+	list_del(&c->root->lru);
+	rw_unlock(false, &c->root->lock);
+
+	/*for (i = 0; i < 256 && devices[i]; i++)
+		register_dev_on_cache(c, i);*/
+
+	for (i = 0; i < 256; i++)
+		c->devices[i] = ~0;
+
+	for (i = 0; i < 256 && !is_zero(&uuids[i*16], 16); i++)
+		register_dev_on_cache(c, i);
+
+	err = "kobject create error";
+	bdevname(c->bdev, b);
+	if (!kobject_get(bcache_kobj))
+		goto err;
+
+	if (kobject_init_and_add(&c->kobj, &cache_obj,
+				 bcache_kobj,
+				 "%s", b))
+		goto err;
+
+	list_add(&c->list, &cache_devices);
+
+	printk(KERN_DEBUG "bcache: Loaded cache device %s, pages_per_btree %i, keys_per_page %li",
+	       path, pages_per_btree, keys_per_page);
+	kfree(path);
+	return;
+err:
+	if (c) {
+		if (c->sb_bh)
+			put_bh(c->sb_bh);
+		if (c->uuids)
+			put_bh(c->uuids);
+		if (c->kobj.state_initialized)
+			kobject_put(&c->kobj);
+		if (c->freelist)
+			vfree(c->freelist);
+		if (c->buckets)
+			vfree(c->buckets);

block/bcache.c line 2240

Quoted 10 lines
block/bcache.c new file +10 -0
+		if (c->heap)
+			vfree(c->heap);
+		if (!IS_ERR_OR_NULL(c->bdev))
+			close_bdev_exclusive(c->bdev, FMODE_READ|FMODE_WRITE);
+		kzfree(c);
+	}
+	printk(KERN_DEBUG "bcache: error opening %s: %s", path, err);
+	kfree(path);
+	return;
+}

---
~Randy
*** Remember to use Documentation/SubmitChecklist when testing your code ***

#6 lore raw
Quoted 3 lines · Randy Dunlap
Quoted 1 line

+#define label(l, foo) if (0) { l: foo; }

I'd prefer that macro to go away.

I kind of like it, the way I use it it's shorthand for "return with x status"; it just makes return codes and exiting cleaner. But if you still hate it after you've read the functions where it's used, I can take it out.

Quoted 3 lines

block/bcache.c line 234

Quoted 1 line
block/bcache.c new file +1 -0
+	return d[i / keys_per_page] + (i % keys_per_page);

That builds OK on i386? or does it need udivdi3() and/or umoddi3()?

If my understanding is correct, those are only needed for 64 bits, i indexes into a single node so 32 bits is plenty.

Quoted 9 lines

block/bcache.c

Quoted 7 lines
block/bcache.c new file +5 -0
+}
 
+static int lookup_dev(struct cache_device *c, struct bio *bio)
+{
+	int dev;
+	for (dev = 0; dev<  256; dev++)

Use a macro for 256.. in lots of places.

Yes, definitely.

Quoted 2 lines

bcache: cannot allocate memory
"kmalloc error" sounds like kmalloc() had an internal error.

Agreed.

Quoted 1 line

Need to document the sysfs interfaces:

Ok, I'll start on that.

Quoted 3 lines

block/bcache.c line 2168

Quoted 1 line
block/bcache.c new file +1 -0
+	err = "vmalloc error";
		"cannot vmalloc memory";

Done.

#7 lore raw
In reply to Kent Overstreet

Hi!

block/bcache.c line 4

Quoted 7 lines
block/bcache.c new file +4 -0
@@ -0,0 +1,2337 @@
+#define pr_fmt(fmt) "bcache: %s() " fmt, __func__
+
+#include <linux/blkdev.h>
+#include <linux/buffer_head.h>

Some copyright, gpl and short note what it is would be nice.
Pavel

Signature · 2 lines
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html