Skip to contentSkip to conversation

Block i/o patches.

PATCH New Nigel Cunningham 1 participant 33 messages Jun 26, 2006

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover Block i/o patches.

This set of patches contains routines in kernel/power/block_io.c.

The routines use struct block_device * pointers and lists of blocks for each block_device. The blocks, in turn, are stored as chains of extents.

Multiple swap devices can thus be supported trivially, and the position at which each part of the image starts can be described in terms of which device/chain is used, what actual value we're sitting on and (for optimisation) which extent in the chain is current.

On top of this, we implement support for asynchronous I/O and readahead when synchronous I/O is required (ie when reading compressed data at resume time).

The swapwriter and filewriter can then use a really simple interface where they don't have to worry at all about what device or block is the target.

Oh, and we handle variable blocksizes too.

#1 lore raw

PATCH 1/32 Block I/O Header File

kernel/power/block_io.h is the header file for the lowlevel block i/o code. It defines the operations that the swapwriter and filewriter use to actually perform the I/O.

 kernel/power/block_io.h |   55 +++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 55 insertions(+), 0 deletions(-)
kernel/power/block_io.h new file +55 -0
new file mode 100644
index 0000000..2e89ca2
@@ -0,0 +1,55 @@
+/*
+ * kernel/power/block_io.h
+ *
+ * Copyright 2004-2006 Nigel Cunningham <[email protected]>
+ *
+ * Distributed under GPLv2.
+ *
+ * This file contains declarations for functions exported from
+ * block_io.c, which contains low level io functions.
+ */
+
+#include <linux/buffer_head.h>
+#include "extent.h"
+
+struct suspend_bdev_info {
+	struct block_device *bdev;
+	dev_t dev_t;
+	int bmap_shift;
+	int blocks_per_page;
+};
+
+/* 
+ * Our exported interface so the swapwriter and filewriter don't
+ * need these functions duplicated.
+ */
+struct suspend_bio_ops {
+	int (*bdev_page_io) (int rw, struct block_device *bdev, long pos,
+			struct page *page);
+	void (*check_io_stats) (void);
+	void (*reset_io_stats) (void);
+	void (*finish_all_io) (void);
+	int (*prepare_readahead) (int index);
+	void (*cleanup_readahead) (int index);
+	struct page ** readahead_pages;
+	int (*readahead_ready) (int readahead_index);
+	int (*forward_one_page) (void);
+	void (*set_extra_page_forward) (void);
+	void (*set_devinfo) (struct suspend_bdev_info *info);
+	int (*read_chunk) (struct page *buffer_page, int sync);
+	int (*write_chunk) (struct page *buffer_page);
+	int (*rw_header_chunk) (int rw, struct suspend_module_ops *owner,
+			char *buffer, int buffer_size);
+	int (*write_header_chunk_finish) (void);
+	int (*rw_init) (int rw, int stream_number);
+	int (*rw_cleanup) (int rw);
+};
+
+extern struct suspend_bio_ops suspend_bio_ops;
+
+extern char *suspend_writer_buffer;
+extern int suspend_writer_buffer_posn;
+extern int suspend_read_fd;
+extern struct extent_iterate_saved_state suspend_writer_posn_save[3];
+extern struct extent_iterate_state suspend_writer_posn;
+extern int suspend_header_bytes_used;

--
Nigel Cunningham nigel at suspend2 dot net

#2 lore raw

PATCH 2/32 Block io c file header.

In reply to Nigel Cunningham

kernel/power/suspend_block_io.c does all the hard work of getting the data written to storage via bio routines. The swapwriter and filewriter are really only allocators - this file implements completely asynchronous submission of the I/O and cleanup on completion. All the writers need to do is feed a list of bdevs and blocks on each bdev, and then the pages to be written or read.

The io_info struct stores the data on each page in flight. As pages progress through processing, their io_info structs move from the free list to the submit_batch list if being batched, to the busy list when submitted to the block layer, to the ready_for_cleanup list with the completion function is called, to the free list after cleanup. Each list has its own spinlock to reduce locking issues.

Readahead pages are used like a ring buffer.

The posn_save struct is used to record where the three parts of the image begin, thereby allowing quick 'seeking' to the start of a pageset or the start of the header.

 kernel/power/suspend_block_io.c |  130 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 130 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c new file +130 -0
new file mode 100644
index 0000000..94f1946
@@ -0,0 +1,130 @@
+/*
+ * kernel/power/suspend_block_io.c
+ *
+ * Copyright 2004-2006 Nigel Cunningham <[email protected]>
+ *
+ * Distributed under GPLv2.
+ * 
+ * This file contains block io functions for suspend2. These are
+ * used by the swapwriter and it is planned that they will also
+ * be used by the NFSwriter.
+ *
+ */
+
+#include <linux/blkdev.h>
+#include <linux/syscalls.h>
+#include <linux/suspend.h>
+
+#include "suspend2.h"
+#include "proc.h"
+#include "modules.h"
+#include "prepare_image.h"
+#include "block_io.h"
+#include "ui.h"
+
+/* Bits in struct io_info->flags */
+enum {
+	IO_AWAITING_READ,
+	IO_AWAITING_SUBMIT,
+	IO_AWAITING_CLEANUP,
+};
+
+#define MAX_OUTSTANDING_IO 2048
+
+/*
+ *
+ *     IO in progress information storage and helpers
+ *
+ */
+
+struct io_info {
+	struct bio *sys_struct;
+	sector_t block[MAX_BUF_PER_PAGE];
+	struct page *buffer_page;
+	struct page *data_page;
+	unsigned long flags;
+	struct block_device *dev;
+	struct list_head list;
+	int readahead_index;
+};
+
+/*
+ * submit_params
+ */
+struct submit_params {
+	swp_entry_t swap_address;
+	struct page *page;
+	struct block_device *dev;
+	sector_t block[MAX_BUF_PER_PAGE];
+	int readahead_index;
+	struct submit_params *next;
+	int printme;
+};
+
+/* Locks separated to allow better SMP support.
+ * An io_struct moves through the lists as follows.
+ * free -> submit_batch -> busy -> ready_for_cleanup -> free
+ */
+static LIST_HEAD(ioinfo_free);
+static DEFINE_SPINLOCK(ioinfo_free_lock);
+
+static LIST_HEAD(ioinfo_ready_for_cleanup);
+static DEFINE_SPINLOCK(ioinfo_ready_lock);
+
+static LIST_HEAD(ioinfo_submit_batch);
+static DEFINE_SPINLOCK(ioinfo_submit_lock);
+
+static LIST_HEAD(ioinfo_busy);
+static DEFINE_SPINLOCK(ioinfo_busy_lock);
+
+static atomic_t submit_batch;
+static int submit_batch_size = 64;
+static int submit_batched(void);
+
+/* [Max] number of I/O operations pending */
+static atomic_t outstanding_io;
+static int max_outstanding_io = 0;
+static atomic_t buffer_allocs, buffer_frees;
+
+/* [Max] number of pages used for above struct */
+static int infopages = 0;
+static int maxinfopages = 0;
+
+static int extra_page_forward = 0;
+
+static volatile unsigned long suspend_readahead_flags[
+	(MAX_OUTSTANDING_IO + BITS_PER_LONG - 1) / BITS_PER_LONG];
+static spinlock_t suspend_readahead_flags_lock = SPIN_LOCK_UNLOCKED;
+static struct page *suspend_readahead_pages[MAX_OUTSTANDING_IO];
+static int readahead_index, readahead_submit_index;
+
+static int current_stream;
+struct extent_iterate_saved_state suspend_writer_posn_save[3];
+
+/* Pointer to current entry being loaded/saved. */
+struct extent_iterate_state suspend_writer_posn;
+
+/* Not static, so that the allocators can setup and complete
+ * writing the header */
+char *suspend_writer_buffer;
+int suspend_writer_buffer_posn;
+
+int suspend_read_fd;
+
+static unsigned long nr_schedule_calls[8];
+
+static char *sch_caller[] = {
+	"get_io_info_struct #1    ",
+	"get_io_info_struct #2    ",
+	"get_io_info_struct #3    ",
+	"suspend_finish_all_io    ",
+	"wait_on_one_page         ",
+	"submit                   ",
+	"start_one                ",
+	"suspend_wait_on_readahead",
+};
+
+static struct suspend_bdev_info *suspend_devinfo;
+
+int suspend_header_bytes_used = 0;
+

--
Nigel Cunningham nigel at suspend2 dot net

#3 lore raw

PATCH 3/32 Reset block io stats.

In reply to Nigel Cunningham

Simple helper which resets the statistics to zero at the start of a cycle.

 kernel/power/suspend_block_io.c |   16 ++++++++++++++++
 1 files changed, 16 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +16 -0
index 94f1946..51e02ef 100644
@@ -128,3 +128,19 @@ static struct suspend_bdev_info *suspend
 
 int suspend_header_bytes_used = 0;
 
+/*
+ * suspend_reset_io_stats
+ *
+ * Description:	Reset all our sanity-checking statistics.
+ */
+static void suspend_reset_io_stats(void)
+{
+	int i;
+	
+	max_outstanding_io = 0;
+	maxinfopages = 0;
+	
+	for (i = 0; i < 8; i++)
+		nr_schedule_calls[i] = 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#4 lore raw

PATCH 4/32 Check the io stats.

In reply to Nigel Cunningham

Perform sanity checking at the conclusion of performing some I/O. By the time this is called, there should be no outstanding I/O, and the io_info structure lists should all be empty. We check these conditions, and display the vital statistics if the appropriate debugging level is enabled.

 kernel/power/suspend_block_io.c |   35 +++++++++++++++++++++++++++++++++++
 1 files changed, 35 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +35 -0
index 51e02ef..09682c4 100644
@@ -144,3 +144,38 @@ static void suspend_reset_io_stats(void)
 		nr_schedule_calls[i] = 0;
 }
 
+/*
+ * suspend_check_io_stats
+ *
+ * Description:	Check that our statistics look right and print
+ * 		any debugging info wanted.
+ */
+static void suspend_check_io_stats(void)
+{
+	int i;
+
+	BUG_ON(atomic_read(&outstanding_io));
+	BUG_ON(infopages);
+	BUG_ON(!list_empty(&ioinfo_submit_batch));
+	BUG_ON(!list_empty(&ioinfo_busy));
+	BUG_ON(!list_empty(&ioinfo_ready_for_cleanup));
+	BUG_ON(!list_empty(&ioinfo_free));
+	BUG_ON(atomic_read(&buffer_allocs) != atomic_read(&buffer_frees));
+
+	suspend_message(SUSPEND_WRITER, SUSPEND_LOW, 0,
+			"Maximum outstanding_io was %d.\n",
+			max_outstanding_io);
+	suspend_message(SUSPEND_WRITER, SUSPEND_LOW, 0,
+			"Max info pages was %d.\n",
+			maxinfopages);
+	if (atomic_read(&buffer_allocs) != atomic_read(&buffer_frees))
+		suspend_message(SUSPEND_WRITER, SUSPEND_MEDIUM, 0,
+			"Buffer allocs (%d) != buffer frees (%d)",
+				atomic_read(&buffer_allocs),
+				atomic_read(&buffer_frees));
+	for(i = 0; i < 8; i++)
+		suspend_message(SUSPEND_WRITER, SUSPEND_MEDIUM, 0,
+			"Nr schedule calls %s: %lu.\n", sch_caller[i],
+			nr_schedule_calls[i]);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#5 lore raw

PATCH 5/32 Cleanup one page.

In reply to Nigel Cunningham

Clean up upon completion of the I/O for a page.

 kernel/power/suspend_block_io.c |   96 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 96 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +96 -0
index 09682c4..037535b 100644
@@ -179,3 +179,99 @@ static void suspend_check_io_stats(void)
 			nr_schedule_calls[i]);
 }
 
+/*
+ * __suspend_bio_cleanup_one
+ * 
+ * Description: Clean up after completing I/O on a page.
+ * Arguments:	struct io_info:	Data for I/O to be completed.
+ */
+static void __suspend_bio_cleanup_one(struct io_info *io_info)
+{
+	struct page *buffer_page;
+	struct page *data_page;
+	char *buffer_address, *data_address;
+	int reading;
+
+	buffer_page = io_info->buffer_page;
+	data_page = io_info->data_page;
+
+	reading = test_bit(IO_AWAITING_READ, &io_info->flags);
+	suspend_message(SUSPEND_WRITER, SUSPEND_HIGH, 0,
+		"Cleanup IO: [%p]\n", 
+		io_info);
+
+	if (reading && io_info->readahead_index == -1) {
+		/*
+		 * Copy the page we read into the buffer our caller provided.
+		 */
+		data_address = (char *) kmap(data_page);
+		buffer_address = (char *) kmap(buffer_page);
+		memcpy(data_address, buffer_address, PAGE_SIZE);
+		kunmap(data_page);
+		kunmap(buffer_page);
+	
+	}
+
+	if (!reading || io_info->readahead_index == -1) {
+		/* Sanity check */
+		if (page_count(buffer_page) != 2)
+			printk(KERN_EMERG "Cleanup IO: Page count on page %p"
+					" is %d. Not good!\n",
+					buffer_page, page_count(buffer_page));
+		put_page(buffer_page);
+		__free_page(buffer_page);
+		atomic_inc(&buffer_frees);
+	} else
+		put_page(buffer_page);
+	
+	bio_put(io_info->sys_struct);
+	io_info->sys_struct = NULL;
+	io_info->flags = 0;
+}
+
+/* suspend_bio_cleanup_one
+ */
+
+static int suspend_bio_cleanup_one(void *data)
+{
+	struct io_info *io_info = (struct io_info *) data;
+	int readahead_index;
+	unsigned long flags;
+
+	/*
+	 * If this I/O was a readahead, remember its index.
+	 */
+	readahead_index = io_info->readahead_index;
+
+	/*
+	 * Add it to the free list.
+	 */
+	list_del_init(&io_info->list);
+	
+	/*
+	 * Do the cleanup.
+	 */
+	__suspend_bio_cleanup_one(io_info);
+
+	/*
+	 * Record the readahead as done.
+	 */
+	if (readahead_index > -1) {
+		int index = readahead_index/BITS_PER_LONG;
+		int bit = readahead_index - (index * BITS_PER_LONG);
+		spin_lock_irqsave(&suspend_readahead_flags_lock, flags);
+		set_bit(bit, &suspend_readahead_flags[index]);
+		spin_unlock_irqrestore(&suspend_readahead_flags_lock, flags);
+	}
+
+	spin_lock_irqsave(&ioinfo_free_lock, flags);
+	list_add_tail(&io_info->list, &ioinfo_free);
+	spin_unlock_irqrestore(&ioinfo_free_lock, flags);
+	
+	/* Important: Must be last thing we do to avoid a race with
+	 * finish_all_io when using keventd to do the cleanup */
+	atomic_dec(&outstanding_io);
+
+	return 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#6 lore raw

PATCH 6/32 Cleanup some completed I/O

In reply to Nigel Cunningham

Iterate through the read_for_cleanup list, cleaning up a maximum of submit_batch_size completed pages.

 kernel/power/suspend_block_io.c |   37 ++++++++++++++++++++++++++++++++++++-
 1 files changed, 36 insertions(+), 1 deletions(-)
kernel/power/suspend_block_io.c +36 -1
index 037535b..cfff687 100644
@@ -229,7 +229,7 @@ static void __suspend_bio_cleanup_one(st
 	io_info->flags = 0;
 }
 
-/* suspend_bio_cleanup_one
+/* __suspend_io_cleanup
  */
 
 static int suspend_bio_cleanup_one(void *data)
@@ -275,3 +275,38 @@ static int suspend_bio_cleanup_one(void 
 	return 0;
 }
 
+/* suspend_cleanup_some_completed_io
+ *
+ * NB: This is designed so that multiple callers can be in here simultaneously.
+ */
+
+static void suspend_cleanup_some_completed_io(void)
+{
+	int num_cleaned = 0;
+	struct io_info *first;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ioinfo_ready_lock, flags);
+	while(!list_empty(&ioinfo_ready_for_cleanup)) {
+		int result;
+		first = list_entry(ioinfo_ready_for_cleanup.next,
+				struct io_info, list);
+
+		BUG_ON(!test_and_clear_bit(IO_AWAITING_CLEANUP, &first->flags));
+
+		list_del_init(&first->list);
+
+		spin_unlock_irqrestore(&ioinfo_ready_lock, flags);
+
+		result = suspend_bio_cleanup_one((void *) first);
+
+		spin_lock_irqsave(&ioinfo_ready_lock, flags);
+		if (result)
+			continue;
+		num_cleaned++;
+		if (num_cleaned == submit_batch_size)
+			break;
+	}
+	spin_unlock_irqrestore(&ioinfo_ready_lock, flags);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#7 lore raw

PATCH 7/32 Wait for bio completion.

In reply to Nigel Cunningham

Submit any outstanding batched I/O, kick kblockd, io_schedule() and see what we can cleanup.

 kernel/power/suspend_block_io.c |   23 +++++++++++++++++++++++
 1 files changed, 23 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +23 -0
index cfff687..6f8b29e 100644
@@ -310,3 +310,26 @@ static void suspend_cleanup_some_complet
 	spin_unlock_irqrestore(&ioinfo_ready_lock, flags);
 }
 
+/* do_bio_wait
+ *
+ * Actions taken when we want some I/O to get run.
+ * 
+ * Submit any I/O that's batched up (if we're not already doing
+ * that, unplug queues, schedule and clean up whatever we can.
+ */
+static void do_bio_wait(int caller)
+{
+	int num_submitted = 0;
+
+	nr_schedule_calls[caller]++;
+	
+	/* Don't want to wait on I/O we haven't submitted! */
+	num_submitted = submit_batched();
+
+	kblockd_flush();
+	
+	io_schedule();
+
+	suspend_cleanup_some_completed_io();
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#8 lore raw

PATCH 8/32 Finish and cleanup all outstanding I/O.

In reply to Nigel Cunningham

Wait for all outstanding I/O to complete, then clean it all up and free the io_info structs.

 kernel/power/suspend_block_io.c |   42 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 42 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +42 -0
index 6f8b29e..5b93ad2 100644
@@ -333,3 +333,45 @@ static void do_bio_wait(int caller)
 	suspend_cleanup_some_completed_io();
 }
 
+/*
+ * suspend_finish_all_io
+ *
+ * Description:	Finishes all IO and frees all IO info struct pages.
+ */
+static void suspend_finish_all_io(void)
+{
+	struct io_info *this, *next = NULL;
+	unsigned long flags;
+
+	/* Wait for all I/O to complete. */
+	while (atomic_read(&outstanding_io))
+		do_bio_wait(2);
+
+	spin_lock_irqsave(&ioinfo_free_lock, flags);
+	
+	/* 
+	 * Two stages, to avoid using freed pages.
+	 *
+	 * First free all io_info structs on a page except the first.
+	 */
+	list_for_each_entry_safe(this, next, &ioinfo_free, list) {
+		if (((unsigned long) this) & ~PAGE_MASK)
+			list_del(&this->list);
+	}
+
+	/* 
+	 * Now we have only one reference to each page, and can safely
+	 * free pages, knowing we're not going to be trying to access the
+	 * same page after freeing it.
+	 */
+	list_for_each_entry_safe(this, next, &ioinfo_free, list) {
+		list_del(&this->list);
+		free_page((unsigned long) this);
+		infopages--;
+		suspend_message(SUSPEND_MEMORY, SUSPEND_VERBOSE, 0,
+				"[FreedIOPage %lx]", this);
+	}
+	
+	spin_unlock_irqrestore(&ioinfo_free_lock, flags);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#9 lore raw

PATCH 9/32 Wait on a particular page.

In reply to Nigel Cunningham

Wait for i/o on a particular page to be complete.

 kernel/power/suspend_block_io.c |   10 ++++++++++
 1 files changed, 10 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +10 -0
index 5b93ad2..f9c883a 100644
@@ -375,3 +375,13 @@ static void suspend_finish_all_io(void)
 	spin_unlock_irqrestore(&ioinfo_free_lock, flags);
 }
 
+/*
+ * wait_on_one_page
+ *
+ * Description:	Wait for a particular I/O to complete.
+ */
+static void wait_on_one_page(struct io_info *io_info)
+{
+	do { do_bio_wait(3); } while (io_info->flags);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#10 lore raw

PATCH 10/32 Wait on readahead.

In reply to Nigel Cunningham

Wait for the next page of readahead to be complete.

 kernel/power/suspend_block_io.c |   15 +++++++++++++++
 1 files changed, 15 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +15 -0
index f9c883a..ae45fc3 100644
@@ -385,3 +385,18 @@ static void wait_on_one_page(struct io_i
 	do { do_bio_wait(3); } while (io_info->flags);
 }
 
+/*
+ * wait_on_readahead
+ *
+ * Wait until a particular readahead is ready.
+ */
+static void suspend_wait_on_readahead(int readahead_index)
+{
+	int index = readahead_index / BITS_PER_LONG;
+	int bit = readahead_index - index * BITS_PER_LONG;
+
+	/* read_ahead_index is the one we want to return */
+	while (!test_bit(bit, &suspend_readahead_flags[index]))
+		do_bio_wait(6);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#11 lore raw

PATCH 11/32 Test whether readahead is ready.

In reply to Nigel Cunningham

Determine whether a particular page's readahead has completed.

 kernel/power/suspend_block_io.c |   14 ++++++++++++++
 1 files changed, 14 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +14 -0
index ae45fc3..f14939e 100644
@@ -400,3 +400,17 @@ static void suspend_wait_on_readahead(in
 		do_bio_wait(6);
 }
 
+/*
+ * readahead_done
+ *
+ * Returns whether the readahead requested is ready.
+ */
+
+static int suspend_readahead_ready(int readahead_index)
+{
+	int index = readahead_index / BITS_PER_LONG;
+	int bit = readahead_index - (index * BITS_PER_LONG);
+
+	return test_bit(bit, &suspend_readahead_flags[index]);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#12 lore raw

PATCH 12/32 Prepare & cleanup readahead pages.

In reply to Nigel Cunningham

Allocate and cleanup pages used for readahead.

 kernel/power/suspend_block_io.c |   22 ++++++++++++++++++++++
 1 files changed, 22 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +22 -0
index f14939e..a1fcacb 100644
@@ -414,3 +414,25 @@ static int suspend_readahead_ready(int r
 	return test_bit(bit, &suspend_readahead_flags[index]);
 }
 
+/* suspend_readahead_prepare
+ * Set up for doing readahead on an image */
+static int suspend_prepare_readahead(int index)
+{
+	unsigned long new_page = get_zeroed_page(GFP_ATOMIC);
+
+	if(!new_page)
+		return -ENOMEM;
+
+	suspend_readahead_pages[index] = virt_to_page(new_page);
+	return 0;
+}
+
+/* suspend_readahead_cleanup
+ * Clean up structures used for readahead */
+static void suspend_cleanup_readahead(int page)
+{
+	__free_page(suspend_readahead_pages[page]);
+	suspend_readahead_pages[page] = 0;
+	return;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#13 lore raw

PATCH 13/32 Suspend bio completion.

In reply to Nigel Cunningham

Routine called in an interrupt context when block io completes. We move the io_info struct from the busy list to the ready-for-cleanup list.

 kernel/power/suspend_block_io.c |   27 +++++++++++++++++++++++++++
 1 files changed, 27 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +27 -0
index a1fcacb..288718c 100644
@@ -436,3 +436,30 @@ static void suspend_cleanup_readahead(in
 	return;
 }
 
+/*
+ * suspend_end_bio
+ *
+ * Description:	Function called by block driver from interrupt context when I/O
+ * 		is completed. This is the reason we use spinlocks in
+ * 		manipulating the io_info lists. 		
+ * 		Nearly the fs/buffer.c version, but we want to mark the page as 
+ * 		done in our own structures too.
+ */
+
+static int suspend_end_bio(struct bio *bio, unsigned int num, int err)
+{
+	struct io_info *io_info = bio->bi_private;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ioinfo_busy_lock, flags);
+	list_del_init(&io_info->list);
+	spin_unlock_irqrestore(&ioinfo_busy_lock, flags);
+
+	set_bit(IO_AWAITING_CLEANUP, &io_info->flags);
+		
+	spin_lock_irqsave(&ioinfo_ready_lock, flags);
+	list_add_tail(&io_info->list, &ioinfo_ready_for_cleanup);
+	spin_unlock_irqrestore(&ioinfo_ready_lock, flags);
+	return 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#14 lore raw

PATCH 14/32 Submit io on a page.

In reply to Nigel Cunningham

Given a struct io_info, allocate and populate a struct bio for the page, link it to the struct io_info and submit the i/o.

 kernel/power/suspend_block_io.c |   51 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 51 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +51 -0
index 288718c..4d489f4 100644
@@ -463,3 +463,54 @@ static int suspend_end_bio(struct bio *b
 	return 0;
 }
 
+/**
+ *	submit - submit BIO request.
+ *	@rw:	READ or WRITE.
+ *	@io_info: IO info structure.
+ *
+ * 	Based on Patrick's pmdisk code from long ago:
+ *	"Straight from the textbook - allocate and initialize the bio.
+ *	If we're writing, make sure the page is marked as dirty.
+ *	Then submit it and carry on."
+ *
+ *	With a twist, though - we handle block_size != PAGE_SIZE.
+ *	Caller has already checked that our page is not fragmented.
+ */
+
+static int submit(int rw, struct io_info *io_info)
+{
+	int error = 0;
+	struct bio *bio = NULL;
+	unsigned long flags;
+
+	while (!bio) {
+		bio = bio_alloc(GFP_ATOMIC,1);
+		if (!bio)
+			do_bio_wait(4);
+	}
+
+	bio->bi_bdev = io_info->dev;
+	bio->bi_sector = io_info->block[0];
+	bio->bi_private = io_info;
+	bio->bi_end_io = suspend_end_bio;
+	io_info->sys_struct = bio;
+
+	if (bio_add_page(bio, io_info->buffer_page, PAGE_SIZE, 0) < PAGE_SIZE) {
+		printk("ERROR: adding page to bio at %lld\n",
+				(unsigned long long) io_info->block[0]);
+		bio_put(bio);
+		return -EFAULT;
+	}
+
+	if (rw == WRITE)
+		bio_set_pages_dirty(bio);
+
+	spin_lock_irqsave(&ioinfo_busy_lock, flags);
+	list_add_tail(&io_info->list, &ioinfo_busy);
+	spin_unlock_irqrestore(&ioinfo_busy_lock, flags);
+	
+	submit_bio(rw,bio);
+
+	return error;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#15 lore raw

PATCH 15/32 Submit batched i/o.

In reply to Nigel Cunningham

Submit a group of pages that have been batched up.

 kernel/power/suspend_block_io.c |   42 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 42 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +42 -0
index 4d489f4..07db518 100644
@@ -514,3 +514,45 @@ static int submit(int rw, struct io_info
 	return error;
 }
 
+/* 
+ * submit a batch. The submit function can wait on I/O, so we have
+ * simple locking to avoid infinite recursion.
+ */
+static int submit_batched(void)
+{
+	static int running_already = 0;
+	struct io_info *first;
+	unsigned long flags;
+	int num_submitted = 0;
+
+	running_already = 1;
+	spin_lock_irqsave(&ioinfo_submit_lock, flags);
+	while(!list_empty(&ioinfo_submit_batch)) {
+		first = list_entry(ioinfo_submit_batch.next, struct io_info,
+									list);
+
+		BUG_ON(!test_and_clear_bit(IO_AWAITING_SUBMIT, &first->flags));
+
+		list_del_init(&first->list);
+
+		atomic_dec(&submit_batch);
+
+		spin_unlock_irqrestore(&ioinfo_submit_lock, flags);
+
+		if (test_bit(IO_AWAITING_READ, &first->flags))
+			submit(READ, first);
+		else
+			submit(WRITE, first);
+
+		spin_lock_irqsave(&ioinfo_submit_lock, flags);
+		
+		num_submitted++;
+		if (num_submitted == submit_batch_size)
+			break;
+	}
+	spin_unlock_irqrestore(&ioinfo_submit_lock, flags);
+	running_already = 0;
+
+	return num_submitted;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#16 lore raw

PATCH 16/32 Add page io to a batch.

In reply to Nigel Cunningham

Add a page to the current batch. If the batch becomes full as a result, submit it.

 kernel/power/suspend_block_io.c |   17 +++++++++++++++++
 1 files changed, 17 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +17 -0
index 07db518..728df04 100644
@@ -556,3 +556,20 @@ static int submit_batched(void)
 	return num_submitted;
 }
 
+static void add_to_batch(struct io_info *io_info)
+{
+	unsigned long flags;
+
+	set_bit(IO_AWAITING_SUBMIT, &io_info->flags);
+
+	/* Put our prepared I/O struct on the batch list. */
+	spin_lock_irqsave(&ioinfo_submit_lock, flags);
+	list_add_tail(&io_info->list, &ioinfo_submit_batch);
+	spin_unlock_irqrestore(&ioinfo_submit_lock, flags);
+
+	atomic_inc(&submit_batch);
+
+	if (atomic_read(&submit_batch) >= submit_batch_size)
+		submit_batched();
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#17 lore raw

PATCH 17/32 Get an io_info struct.

In reply to Nigel Cunningham

Allocate the struct used to store the details of an i/o in flight.

 kernel/power/suspend_block_io.c |   59 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 59 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +59 -0
index 728df04..48a2d09 100644
@@ -573,3 +573,62 @@ static void add_to_batch(struct io_info 
 		submit_batched();
 }
 
+/*
+ * get_io_info_struct
+ *
+ * Description:	Get an I/O struct.
+ * Returns:	Pointer to the struct prepared for use.
+ */
+static struct io_info *get_io_info_struct(void)
+{
+	unsigned long newpage = 0, flags;
+	struct io_info *this = NULL;
+	int remaining = 0;
+
+	do {
+		while (atomic_read(&outstanding_io) >= MAX_OUTSTANDING_IO)
+			do_bio_wait(0);
+
+		/* Can start a new I/O. Is there a free one? */
+		if (!list_empty(&ioinfo_free)) {
+			/* Yes. Grab it. */
+			spin_lock_irqsave(&ioinfo_free_lock, flags);
+			break;
+		}
+
+		/* No. Need to allocate a new page for I/O info structs. */
+		newpage = get_zeroed_page(GFP_ATOMIC);
+		if (!newpage) {
+			do_bio_wait(1);
+			continue;
+		}
+
+		suspend_message(SUSPEND_MEMORY, SUSPEND_VERBOSE, 0,
+				"[NewIOPage %lx]", newpage);
+		infopages++;
+		if (infopages > maxinfopages)
+			maxinfopages++;
+
+		/* Prepare the new page for use. */
+		this = (struct io_info *) newpage;
+		remaining = PAGE_SIZE;
+		spin_lock_irqsave(&ioinfo_free_lock, flags);
+		while (remaining >= (sizeof(struct io_info))) {
+			list_add_tail(&this->list, &ioinfo_free);
+			this = (struct io_info *) (((char *) this) + 
+					sizeof(struct io_info));
+			remaining -= sizeof(struct io_info);
+		}
+		break;
+	} while (1);
+
+	/*
+	 * We have an I/O info struct. Remove it from the free list.
+	 * It will be added to the submit or busy list later.
+	 */
+	this = list_entry(ioinfo_free.next, struct io_info, list);
+	list_del_init(&this->list);
+	spin_unlock_irqrestore(&ioinfo_free_lock, flags);
+	return this;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#18 lore raw

PATCH 18/32 Prepare to do i/o on a page.

In reply to Nigel Cunningham

Prepare to do i/o on a page, either directly submitting the resulting io_info struct, or adding it to a batch.

 kernel/power/suspend_block_io.c |   90 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 90 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +90 -0
index 48a2d09..0d01483 100644
@@ -632,3 +632,93 @@ static struct io_info *get_io_info_struc
 	return this;
 }
 
+/*
+ * start_one
+ *
+ * Description:	Prepare and start a read or write operation.
+ * 		Note that we use our own buffer for reading or writing.
+ * 		This simplifies doing readahead and asynchronous writing.
+ * 		We can begin a read without knowing the location into which
+ * 		the data will eventually be placed, and the buffer passed
+ * 		for a write can be reused immediately (essential for the
+ * 		modules system).
+ * 		Failure? What's that?
+ * Returns:	The io_info struct created.
+ */
+static struct io_info *start_one(int rw, struct submit_params *submit_info)
+{
+	struct io_info *io_info = get_io_info_struct();
+	unsigned long buffer_virt = 0;
+	char *to, *from;
+	struct page *buffer_page;
+
+	if (!io_info)
+		return NULL;
+
+	/* Get our local buffer */
+	suspend_message(SUSPEND_WRITER, SUSPEND_HIGH, 1,
+			"Start_IO: [%p]", io_info);
+	
+	/* Copy settings to the io_info struct */
+	io_info->data_page = submit_info->page;
+	io_info->readahead_index = submit_info->readahead_index;
+
+	if (io_info->readahead_index == -1) {
+		while (!(buffer_virt = get_zeroed_page(GFP_ATOMIC)))
+			do_bio_wait(5);
+
+		atomic_inc(&buffer_allocs);
+		suspend_message(SUSPEND_WRITER, SUSPEND_HIGH, 0,
+				"[ALLOC BUFFER]->%d",
+				real_nr_free_pages());
+		buffer_page = virt_to_page(buffer_virt);
+	
+		io_info->buffer_page = buffer_page;
+	} else {
+		unsigned long flags;
+		int index = io_info->readahead_index / BITS_PER_LONG;
+		int bit = io_info->readahead_index - index * BITS_PER_LONG;
+
+		spin_lock_irqsave(&suspend_readahead_flags_lock, flags);
+		clear_bit(bit, &suspend_readahead_flags[index]);
+		spin_unlock_irqrestore(&suspend_readahead_flags_lock, flags);
+
+		io_info->buffer_page = buffer_page = submit_info->page;
+	}
+
+	/* If writing, copy our data. The data is probably in
+	 * lowmem, but we cannot be certain. If there is no
+	 * compression/encryption, we might be passed the
+	 * actual source page's address. */
+	if (rw == WRITE) {
+		to = (char *) buffer_virt;
+		from = kmap_atomic(io_info->data_page, KM_USER1);
+		memcpy(to, from, PAGE_SIZE);
+		kunmap_atomic(from, KM_USER1);
+	}
+
+	/* Submit the page */
+	get_page(buffer_page);
+	
+	io_info->dev = submit_info->dev;
+	io_info->block[0] = submit_info->block[0];
+
+	if (rw == READ)
+		set_bit(IO_AWAITING_READ, &io_info->flags);
+
+	suspend_message(SUSPEND_WRITER, SUSPEND_HIGH, 1,
+			"-> (PRE BRW) %d\n",
+			real_nr_free_pages());
+
+	if (submit_batch_size > 1)
+		add_to_batch(io_info);
+	else
+	 	submit(rw, io_info);
+	
+	atomic_inc(&outstanding_io);
+	if (atomic_read(&outstanding_io) > max_outstanding_io)
+		max_outstanding_io++;
+	
+	return io_info;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#19 lore raw

PATCH 19/32 Do i/o on a page.

In reply to Nigel Cunningham

Perform i/o on a page, possibly waiting for completion.

 kernel/power/suspend_block_io.c |   44 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 44 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +44 -0
index 0d01483..555d4fa 100644
@@ -722,3 +722,47 @@ static struct io_info *start_one(int rw,
 	return io_info;
 }
 
+static int suspend_do_io(int rw, 
+		struct submit_params *submit_info, int syncio)
+{
+	struct io_info *io_info;
+
+	if(!submit_info->dev) {
+		printk("Suspend_do_io: submit_info->dev is NULL!\n");
+		return 1;
+	}
+	
+	io_info = start_one(rw, submit_info);
+
+	if (!io_info) {
+		printk("Unable to allocate an io_info struct.\n");
+		return 1;
+	} else if (syncio)
+		wait_on_one_page(io_info);
+
+	/* If we were the only one, clean everything up */
+	if (!atomic_read(&outstanding_io))
+		suspend_finish_all_io();
+	return 0;
+} 
+
+/* We used to use bread here, but it doesn't correctly handle
+ * blocksize != PAGE_SIZE. Now we create a submit_info to get the data we
+ * want and use our normal routines (synchronously).
+ */
+
+static int suspend_bdev_page_io(int rw, struct block_device *bdev, long pos,
+		struct page *page)
+{
+	struct submit_params submit_info;
+
+	if (!bdev)
+		return 0;
+
+	submit_info.page = page;
+	submit_info.dev = bdev;
+	submit_info.block[0] = pos;
+	submit_info.readahead_index = -1;
+	return suspend_do_io(rw, &submit_info, 1);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#20 lore raw

PATCH 20/32 Return memory needed for block io.

In reply to Nigel Cunningham

Return the maximum amount of memory needed for block io.

 kernel/power/suspend_block_io.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +9 -0
index 555d4fa..6e0f22c 100644
@@ -766,3 +766,12 @@ static int suspend_bdev_page_io(int rw, 
 	return suspend_do_io(rw, &submit_info, 1);
 }
 
+static unsigned long suspend_bio_memory_needed(void)
+{
+	/* We want to have at least enough memory so as to have
+	 * MAX_OUTSTANDING_IO transactions on the fly at once. If we 
+	 * can to more, fine. */
+	return (MAX_OUTSTANDING_IO * (PAGE_SIZE + sizeof(struct request) +
+				sizeof(struct bio) + sizeof(struct io_info)));
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#21 lore raw

PATCH 21/32 Set device info.

In reply to Nigel Cunningham

Set the list of devices and blocks to use for i/o.

 kernel/power/suspend_block_io.c |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +5 -0
index 6e0f22c..3902758 100644
@@ -775,3 +775,8 @@ static unsigned long suspend_bio_memory_
 				sizeof(struct bio) + sizeof(struct io_info)));
 }
 
+static void suspend_set_devinfo(struct suspend_bdev_info *info)
+{
+	suspend_devinfo = info;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#22 lore raw

PATCH 22/32 Move forward extra blocks in a page.

In reply to Nigel Cunningham

Shift the pointer to the current block forward (n-1) blocks, where n is the blocks per page used for this device.

 kernel/power/suspend_block_io.c |   16 ++++++++++++++++
 1 files changed, 16 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +16 -0
index 3902758..fbce186 100644
@@ -780,3 +780,19 @@ static void suspend_set_devinfo(struct s
 	suspend_devinfo = info;
 }
 
+static int forward_extra_blocks(void)
+{
+	int i;
+
+	for (i = 1; i < suspend_devinfo[suspend_writer_posn.current_chain].
+							blocks_per_page; i++)
+		suspend_extent_state_next(&suspend_writer_posn);
+
+	if (suspend_extent_state_eof(&suspend_writer_posn)) {
+		printk("Extent state eof.\n");
+		return -ENODATA;
+	}
+
+	return 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#23 lore raw

PATCH 23/32 Advance one page in the extent state.

In reply to Nigel Cunningham

Advance one page in the list of devices and blocks (extent state).

 kernel/power/suspend_block_io.c |   19 +++++++++++++++++++
 1 files changed, 19 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +19 -0
index fbce186..fdbbe4b 100644
@@ -796,3 +796,22 @@ static int forward_extra_blocks(void)
 	return 0;
 }
 
+static int forward_one_page(void)
+{
+	int at_start = (suspend_writer_posn.current_chain == -1);
+
+	/* Have to go forward one to ensure we're on the right chain,
+	 * before we can know how many more blocks to skip.*/
+	suspend_extent_state_next(&suspend_writer_posn);
+
+	if (!at_start)
+		if (forward_extra_blocks())
+			return -ENODATA;
+
+	if (extra_page_forward) {
+		extra_page_forward = 0;
+		return forward_one_page();
+	}
+
+	return 0;
+}

--
Nigel Cunningham nigel at suspend2 dot net

#24 lore raw

PATCH 24/32 Set extra page forward flag.

In reply to Nigel Cunningham

Set the flag that says the next forward_one_page should actually go forward to pages. This is used at resume time when we get the first page of the header from the page pointed to by resume2= (bootstrapping), and then actually want the second page at the point where we do need to get another page.

We can't just go forward one page right away because we haven't yet loaded the extent chain that contains the block data. But loading the extent chain that contains the block data might need the extra page (if badly fragmented). Catch 22, hence this flag.

 kernel/power/suspend_block_io.c |    6 ++++++
 1 files changed, 6 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +6 -0
index fdbbe4b..ffc2e16 100644
@@ -815,3 +815,9 @@ static int forward_one_page(void)
 
 	return 0;
 }
+
+static void set_extra_page_forward(void)
+{
+	extra_page_forward = 1;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#25 lore raw

PATCH 25/32 Do io on a page in the image proper.

In reply to Nigel Cunningham

Advance one page and perform the requested i/o.

 kernel/power/suspend_block_io.c |   35 +++++++++++++++++++++++++++++++++++
 1 files changed, 35 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +35 -0
index ffc2e16..c746f9f 100644
@@ -821,3 +821,38 @@ static void set_extra_page_forward(void)
 	extra_page_forward = 1;
 }
 
+static int suspend_rw_page(int rw, struct page *page,
+		int readahead_index, int sync, int debug)
+{
+	int i, current_chain;
+	struct submit_params submit_params;
+
+	if (test_action_state(SUSPEND_TEST_FILTER_SPEED))
+		return 0;
+		
+	submit_params.readahead_index = readahead_index;
+	submit_params.page = page;
+	
+	if (forward_one_page()) {
+		printk("Failed to advance a page in the extent data.\n");
+		return -ENODATA;
+	}
+
+	current_chain = suspend_writer_posn.current_chain;
+	submit_params.dev = suspend_devinfo[current_chain].bdev;
+	submit_params.block[0] = suspend_writer_posn.current_offset <<
+		suspend_devinfo[current_chain].bmap_shift;
+
+	if (debug)
+		printk("%s: %lx:%lx.\n", rw ? "Write" : "Read",
+				(long) submit_params.dev->bd_dev,
+				(long) submit_params.block[0]);
+
+	i = suspend_do_io(rw, &submit_params, sync);
+
+	if (i)
+		return -EIO;
+
+	return 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#26 lore raw

PATCH 26/32 Read a page of the image.

In reply to Nigel Cunningham

Read a page of the image, using readahead if synchronous i/o is requested.

 kernel/power/suspend_block_io.c |   63 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 63 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +63 -0
index c746f9f..c095eac 100644
@@ -856,3 +856,66 @@ static int suspend_rw_page(int rw, struc
 	return 0;
 }
 
+static int suspend_bio_read_chunk(struct page *buffer_page, int sync)
+{
+	static int last_result;
+	unsigned long *virt;
+
+	if (sync == SUSPEND_ASYNC)
+		return suspend_rw_page(READ, buffer_page, -1, sync, 0);
+
+	/* Start new readahead while we wait for our page */
+	if (readahead_index == -1) {
+		last_result = 0;
+		readahead_index = readahead_submit_index = 0;
+	}
+
+	/* Start a new readahead? */
+	if (last_result) {
+		/* We failed to submit a read, and have cleaned up
+		 * all the readahead previously submitted */
+		if (readahead_submit_index == readahead_index)
+			return -EPERM;
+		goto wait;
+	}
+	
+	do {
+		if (suspend_prepare_readahead(readahead_submit_index))
+			break;
+
+		last_result = suspend_rw_page(
+			READ,
+			suspend_readahead_pages[readahead_submit_index], 
+			readahead_submit_index, SUSPEND_ASYNC, 0);
+		if (last_result) {
+			printk("Begin read chunk for page %d returned %d.\n",
+				readahead_submit_index, last_result);
+			suspend_cleanup_readahead(readahead_submit_index);
+			break;
+		}
+
+		readahead_submit_index++;
+
+		if (readahead_submit_index == MAX_OUTSTANDING_IO)
+			readahead_submit_index = 0;
+
+	} while((!last_result) && (readahead_submit_index != readahead_index) &&
+			(!suspend_readahead_ready(readahead_index)));
+
+wait:
+	suspend_wait_on_readahead(readahead_index);
+
+	virt = kmap_atomic(buffer_page, KM_USER1);
+	memcpy(virt, page_address(suspend_readahead_pages[readahead_index]),
+			PAGE_SIZE);
+	kunmap_atomic(virt, KM_USER1);
+
+	suspend_cleanup_readahead(readahead_index);
+
+	readahead_index++;
+	if (readahead_index == MAX_OUTSTANDING_IO)
+		readahead_index = 0;
+
+	return 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#27 lore raw

PATCH 27/32 Init and cleanup routines for pageset i/o.

In reply to Nigel Cunningham

Preparation and cleanup routines for doing I/O on a pageset.

 kernel/power/suspend_block_io.c |   37 +++++++++++++++++++++++++++++++++++++
 1 files changed, 37 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +37 -0
index c095eac..374a861 100644
@@ -919,3 +919,40 @@ wait:
 	return 0;
 }
 
+static int suspend_rw_init(int rw, int stream_number)
+{
+	suspend_extent_state_restore(&suspend_writer_posn,
+			&suspend_writer_posn_save[stream_number]);
+	current_stream = stream_number;
+
+	BUG_ON(!suspend_writer_posn.current_extent);
+
+	suspend_reset_io_stats();
+
+	readahead_index = readahead_submit_index = -1;
+
+	return 0;
+}
+
+static int suspend_rw_cleanup(int rw)
+{
+	if (rw == WRITE && current_stream == 2)
+		suspend_extent_state_save(&suspend_writer_posn,
+				&suspend_writer_posn_save[1]);
+	
+	suspend_finish_all_io();
+	
+	if (rw == READ) {
+		while (readahead_index != readahead_submit_index) {
+			suspend_cleanup_readahead(readahead_index);
+			readahead_index++;
+			if (readahead_index == MAX_OUTSTANDING_IO)
+				readahead_index = 0;
+		}
+	}
+
+	suspend_check_io_stats();
+
+	return 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#28 lore raw

PATCH 28/32 Write a page in the image proper.

In reply to Nigel Cunningham

Write a page in the image proper to storage.

 kernel/power/suspend_block_io.c |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +5 -0
index 374a861..f881fc7 100644
@@ -956,3 +956,8 @@ static int suspend_rw_cleanup(int rw)
 	return 0;
 }
 
+static int suspend_write_chunk(struct page *buffer_page)
+{
+	return suspend_rw_page(WRITE, buffer_page, -1, 0, 0);
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#29 lore raw

PATCH 29/32 Read or write a chunk of the header.

In reply to Nigel Cunningham

Submit a buffer with data to write, or a buffer to be filled. In contrast to the functions for the main portion of the image, the buffer is normally less than a full page.

 kernel/power/suspend_block_io.c |   67 +++++++++++++++++++++++++++++++++++++++
 1 files changed, 67 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +67 -0
index f881fc7..e822b19 100644
@@ -961,3 +961,70 @@ static int suspend_write_chunk(struct pa
 	return suspend_rw_page(WRITE, buffer_page, -1, 0, 0);
 }
 
+static int suspend_rw_header_chunk(int rw, struct suspend_module_ops *owner,
+		char *buffer, int buffer_size)
+{
+	int bytes_left = buffer_size;
+	
+	if (owner) {
+		owner->header_used += buffer_size;
+		if (owner->header_used > owner->header_requested) {
+			printk(KERN_EMERG "Suspend2 module %s is using more"
+				"header space (%lu) than it requested (%lu).\n",
+				owner->name,
+				owner->header_used,
+				owner->header_requested);
+			BUG();
+		}
+	}
+
+	/* Read a chunk of the header */
+	while (bytes_left) {
+		char *source_start = buffer + buffer_size - bytes_left;
+		char *dest_start = suspend_writer_buffer + suspend_writer_buffer_posn;
+		int capacity = PAGE_SIZE - suspend_writer_buffer_posn;
+		char *to = rw ? dest_start : source_start;
+		char *from = rw ? source_start : dest_start;
+
+		if (bytes_left <= capacity) {
+			if (test_debug_state(SUSPEND_HEADER))
+				printk("Copy %d bytes %d-%d from %p to %p.\n",
+						bytes_left,
+						suspend_header_bytes_used,
+						suspend_header_bytes_used + bytes_left,
+						from, to);
+			memcpy(to, from, bytes_left);
+			suspend_writer_buffer_posn += bytes_left;
+			suspend_header_bytes_used += bytes_left;
+			return rw ? 0 : buffer_size;
+		}
+
+		/* Next to read the next page */
+		if (test_debug_state(SUSPEND_HEADER))
+			printk("Copy %d bytes (%d-%d) from %p to %p.\n",
+					capacity,
+					suspend_header_bytes_used,
+					suspend_header_bytes_used + capacity,
+					from, to);
+		memcpy(to, from, capacity);
+		bytes_left -= capacity;
+		suspend_header_bytes_used += capacity;
+
+		if (rw == READ && test_suspend_state(SUSPEND_TRY_RESUME_RD))
+			sys_read(suspend_read_fd,
+				suspend_writer_buffer, BLOCK_SIZE);
+		else {
+			if (suspend_rw_page(rw,
+					virt_to_page(suspend_writer_buffer),
+					-1, !rw,
+					test_debug_state(SUSPEND_HEADER)))
+				return -EIO;
+		}
+
+		suspend_writer_buffer_posn = 0;
+		suspend_cond_pause(0, NULL);
+	}
+
+	return rw ? 0 : buffer_size;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#30 lore raw

PATCH 30/32 Finish writing an image header.

In reply to Nigel Cunningham

Cleanup after writing an image header.

 kernel/power/suspend_block_io.c |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +7 -0
index e822b19..f93d985 100644
@@ -1028,3 +1028,10 @@ static int suspend_rw_header_chunk(int r
 	return rw ? 0 : buffer_size;
 }
 
+static int write_header_chunk_finish(void)
+{
+	return suspend_rw_page(WRITE,
+		virt_to_page(suspend_writer_buffer),
+		-1, 0, test_debug_state(SUSPEND_HEADER)) ? -EIO : 0;
+}
+

--
Nigel Cunningham nigel at suspend2 dot net

#31 lore raw

PATCH 31/32 Suspend block io ops.

In reply to Nigel Cunningham

Declare the structure that the writers use to access the functions in this file.

 kernel/power/suspend_block_io.c |   20 ++++++++++++++++++++
 1 files changed, 20 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +20 -0
index f93d985..9339826 100644
@@ -1035,3 +1035,23 @@ static int write_header_chunk_finish(voi
 		-1, 0, test_debug_state(SUSPEND_HEADER)) ? -EIO : 0;
 }
 
+struct suspend_bio_ops suspend_bio_ops = {
+	.bdev_page_io = suspend_bdev_page_io,
+	.check_io_stats = suspend_check_io_stats,
+	.reset_io_stats = suspend_reset_io_stats,
+	.finish_all_io = suspend_finish_all_io,
+	.prepare_readahead = suspend_prepare_readahead,
+	.cleanup_readahead = suspend_cleanup_readahead,
+	.readahead_pages = suspend_readahead_pages,
+	.readahead_ready = suspend_readahead_ready,
+	.forward_one_page = forward_one_page,
+	.set_extra_page_forward = set_extra_page_forward,
+	.set_devinfo = suspend_set_devinfo,
+	.read_chunk = suspend_bio_read_chunk,
+	.write_chunk = suspend_write_chunk,
+	.rw_init = suspend_rw_init,
+	.rw_cleanup = suspend_rw_cleanup,
+	.rw_header_chunk = suspend_rw_header_chunk,
+	.write_header_chunk_finish = write_header_chunk_finish,
+};
+

--
Nigel Cunningham nigel at suspend2 dot net

#32 lore raw

PATCH 32/32 Module ops for the block i/o functions.

In reply to Nigel Cunningham

Structure defining ops for the block io code, and the routines to register and deregister them.

 kernel/power/suspend_block_io.c |   27 +++++++++++++++++++++++++++
 1 files changed, 27 insertions(+), 0 deletions(-)
kernel/power/suspend_block_io.c +27 -0
index 9339826..2d9a2f0 100644
@@ -1055,3 +1055,30 @@ struct suspend_bio_ops suspend_bio_ops =
 	.write_header_chunk_finish = write_header_chunk_finish,
 };
 
+static struct suspend_module_ops suspend_blockwriter_ops = 
+{
+	.name					= "Block I/O",
+	.type					= MISC_MODULE,
+	.module					= THIS_MODULE,
+	.memory_needed				= suspend_bio_memory_needed,
+};
+
+static __init int suspend_block_io_load(void)
+{
+	return suspend_register_module(&suspend_blockwriter_ops);
+}
+
+#ifdef MODULE
+static __exit void suspend_block_io_unload(void)
+{
+	suspend_unregister_module(&suspend_blockwriter_ops);
+}
+
+module_init(suspend_block_io_load);
+module_exit(suspend_block_io_unload);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Nigel Cunningham");
+MODULE_DESCRIPTION("Suspend2 block io functions");
+#else
+late_initcall(suspend_block_io_load);
+#endif

--
Nigel Cunningham nigel at suspend2 dot net