IDE TCQ for 2.4.19-pre8

diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/drivers/block/ll_rw_blk.c linux/drivers/block/ll_rw_blk.c
@@ -178,6 +178,9 @@
 	if (count)
 		printk("blk_cleanup_queue: leaked requests (%d)\n", count);
 
+	if (blk_queue_tagged(q))
+		blk_queue_free_tags(q);
+
 	memset(q, 0, sizeof(*q));
 }
 

@@ -238,6 +241,205 @@
 	q->make_request_fn = mfn;
 }
 
+/**
+ * blk_queue_free_tags - release tag maintenance info
+ * @q:  the request queue for the device
+ *
+ *  Notes:
+ *    blk_cleanup_queue() will take care of calling this function, if tagging
+ *    has been used. So there's usually no need to call this directly, unless
+ *    tagging is just being disabled but the queue remains in function.
+ **/
+void blk_queue_free_tags(request_queue_t *q)
+{
+	struct blk_queue_tag *bqt = q->queue_tags;
+
+	if (!bqt)
+		return;
+
+	BUG_ON(bqt->busy);
+	BUG_ON(!list_empty(&bqt->busy_list));
+
+	kfree(bqt->tag_index);
+	bqt->tag_index = NULL;
+
+	kfree(bqt->tag_map);
+	bqt->tag_map = NULL;
+
+	kfree(bqt);
+	q->queue_tags = NULL;
+	q->tagged_queueing = 0;
+}
+
+/**
+ * blk_queue_init_tags - initialize the queue tag info
+ * @q:  the request queue for the device
+ * @depth:  the maximum queue depth supported
+ **/
+int blk_queue_init_tags(request_queue_t *q, int depth)
+{
+	struct blk_queue_tag *tags;
+	int bits, i;
+
+	if (depth > q->nr_requests) {
+		depth = q->nr_requests;
+		printk("blk_queue_init_tags: adjusted depth to %d\n", depth);
+	}
+
+	tags = kmalloc(sizeof(struct blk_queue_tag), GFP_ATOMIC);
+	if (!tags)
+		goto fail;
+
+	tags->tag_index = kmalloc(depth * sizeof(struct request *), GFP_ATOMIC);
+	if (!tags->tag_index)
+		goto fail_index;
+
+	bits = (depth / BLK_TAGS_PER_LONG) + 1;
+	tags->tag_map = kmalloc(bits * sizeof(unsigned long), GFP_ATOMIC);
+	if (!tags->tag_map)
+		goto fail_map;
+
+	memset(tags->tag_index, 0, depth * sizeof(struct request *));
+	memset(tags->tag_map, 0, bits * sizeof(unsigned long));
+	INIT_LIST_HEAD(&tags->busy_list);
+	tags->busy = 0;
+	tags->max_depth = depth;
+
+	/*
+	 * set the upper bits if the depth isn't a multiple of the word size
+	 */
+	for (i = depth; i < bits * BLK_TAGS_PER_LONG; i++)
+		__set_bit(i, tags->tag_map);
+
+	/*
+	 * assign it, all done
+	 */
+	q->queue_tags = tags;
+	q->tagged_queueing = 1;
+	return 0;
+
+fail_map:
+	kfree(tags->tag_index);
+fail_index:
+	kfree(tags);
+fail:
+	return -ENOMEM;
+}
+
+/**
+ * blk_queue_end_tag - end tag operations for a request
+ * @q:  the request queue for the device
+ * @tag:  the tag that has completed
+ *
+ *  Description:
+ *    Typically called when end_that_request_first() returns 0, meaning
+ *    all transfers have been done for a request. It's important to call
+ *    this function before end_that_request_last(), as that will put the
+ *    request back on the free list thus corrupting the internal tag list.
+ *
+ *  Notes:
+ *   io_request_lock must be held.
+ **/
+void blk_queue_end_tag(request_queue_t *q, struct request *rq)
+{
+	struct blk_queue_tag *bqt = q->queue_tags;
+	int tag = rq->tag;
+
+	BUG_ON(tag == -1);
+
+	if (unlikely(tag >= bqt->max_depth))
+		return;
+
+	if (unlikely(!__test_and_clear_bit(tag, bqt->tag_map))) {
+		printk("attempt to clear non-busy tag (%d)\n", tag);
+		return;
+	}
+
+	list_del(&rq->queue);
+	rq->tag = -1;
+	rq->tagged = 0;
+
+	if (unlikely(bqt->tag_index[tag] == NULL))
+		printk("tag %d is missing\n", tag);
+
+	bqt->tag_index[tag] = NULL;
+	bqt->busy--;
+}
+
+/**
+ * blk_queue_start_tag - find a free tag and assign it
+ * @q:  the request queue for the device
+ * @rq:  the block request that needs tagging
+ *
+ *  Description:
+ *    Note that this function assumes that only regular read/write requests
+ *    can be queued! The request will also be removed from the request queue,
+ *    so it's the drivers responsibility to readd it if it should need to
+ *    be restarted for some reason.
+ *
+ *  Notes:
+ *   io_request_lock must be held
+ **/
+int blk_queue_start_tag(request_queue_t *q, struct request *rq)
+{
+	struct blk_queue_tag *bqt = q->queue_tags;
+	unsigned long *map = bqt->tag_map;
+	int tag = 0;
+
+	if (rq->cmd != READ && rq->cmd != WRITE)
+		return 1;
+
+	for (map = bqt->tag_map; *map == -1UL; map++) {
+		tag += BLK_TAGS_PER_LONG;
+
+		if (tag >= bqt->max_depth)
+			return 1;
+	}
+
+	tag += ffz(*map);
+	__set_bit(tag, bqt->tag_map);
+
+	rq->tag = tag;
+	rq->tagged = 1;
+	bqt->tag_index[tag] = rq;
+	blkdev_dequeue_request(rq);
+	list_add(&rq->queue, &bqt->busy_list);
+	bqt->busy++;
+	return 0;
+}
+
+/**
+ * blk_queue_invalidate_tags - invalidate all pending tags
+ * @q:  the request queue for the device
+ *
+ *  Description:
+ *   Hardware conditions may dictate a need to stop all pending requests.
+ *   In this case, we will safely clear the block side of the tag queue and
+ *   readd all requests to the request queue in the right order.
+ *
+ *  Notes:
+ *   io_request_lock must be held.
+ **/
+void blk_queue_invalidate_tags(request_queue_t *q)
+{
+	struct blk_queue_tag *bqt = q->queue_tags;
+	struct list_head *tmp;
+	struct request *rq;
+
+	list_for_each(tmp, &bqt->busy_list) {
+		rq = list_entry(tmp, struct request, queue);
+
+		if (rq->tag == -1) {
+			printk("bad tag found on list\n");
+			list_del(&rq->queue);
+			rq->tagged = 0;
+		} else
+			blk_queue_end_tag(q, rq);
+
+		list_add(&rq->queue, &q->queue_head);
+	}
+}
+
 static inline int ll_new_segment(request_queue_t *q, struct request *req, int max_segments)
 {
 	if (req->nr_segments < max_segments) {

@@ -667,6 +869,7 @@
 		hd->wr_sectors += sectors;
 		break;
 	default:
+		;
 	}
 	if (!merge)
 		up_ios(hd);

@@ -685,6 +888,7 @@
 		hd->wr_ios++;
 		break;
 	default:
+		;
 	}
 	down_ios(hd);
 }

@@ -1441,3 +1645,9 @@
 EXPORT_SYMBOL(blkdev_release_request);
 EXPORT_SYMBOL(req_finished_io);
 EXPORT_SYMBOL(generic_unplug_device);
+
+EXPORT_SYMBOL(blk_queue_init_tags);
+EXPORT_SYMBOL(blk_queue_free_tags);
+EXPORT_SYMBOL(blk_queue_start_tag);
+EXPORT_SYMBOL(blk_queue_end_tag);
+EXPORT_SYMBOL(blk_queue_invalidate_tags);

diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/include/linux/blkdev.h linux/include/linux/blkdev.h
@@ -37,12 +37,14 @@
 	unsigned int nr_segments;
 	unsigned int nr_hw_segments;
 	unsigned long current_nr_sectors;
+	int tag;
 	void * special;
 	char * buffer;
 	struct completion * waiting;
 	struct buffer_head * bh;
 	struct buffer_head * bhtail;
 	request_queue_t *q;
+	char tagged;
 };
 
 #include <linux/elevator.h>

@@ -72,6 +74,18 @@
 	struct list_head free;
 };
 
+#define BLK_TAGS_PER_LONG	(sizeof(unsigned long) * 8)
+#define BLK_TAGS_MASK		(BLK_TAGS_PER_LONG - 1)
+
+struct blk_queue_tag {
+	struct request **tag_index;	/* map of busy tags */
+	unsigned long *tag_map;		/* bit map of free/busy tags */
+	struct list_head busy_list;	/* fifo list of busy tags */
+	int busy;			/* current depth */
+	int max_depth;
+};
+
+
 struct request_queue
 {
 	/*

@@ -123,6 +137,8 @@
 	 */
 	char			head_active;
 
+	char			tagged_queueing;
+
 	/*
 	 * Is meant to protect the queue in the future instead of
 	 * io_request_lock

@@ -133,8 +149,12 @@
 	 * Tasks wait here for free read and write requests
 	 */
 	wait_queue_head_t	wait_for_requests[2];
+
+	struct blk_queue_tag	*queue_tags;
 };
 
+#define blk_queue_tagged(q)	(q)->tagged_queueing
+
 struct blk_dev_struct {
 	/*
 	 * queue_proc has to be atomic

@@ -175,6 +195,19 @@
 extern void blk_queue_make_request(request_queue_t *, make_request_fn *);
 extern void generic_unplug_device(void *);
 
+/*
+ * tag stuff
+ */
+#define blk_queue_tag_request(q, tag)	((q)->queue_tags->tag_index[(tag)])
+#define blk_queue_tag_depth(q)		((q)->queue_tags->busy)
+#define blk_queue_tag_queue(q)		((q)->queue_tags->busy < (q)->queue_tags->max_depth)
+#define blk_rq_tagged(rq)		((rq)->tagged)
+extern int blk_queue_start_tag(request_queue_t *, struct request *);
+extern void blk_queue_end_tag(request_queue_t *, struct request *);
+extern int blk_queue_init_tags(request_queue_t *, int);
+extern void blk_queue_free_tags(request_queue_t *);
+extern void blk_queue_invalidate_tags(request_queue_t *);
+
 extern int * blk_size[MAX_BLKDEV];
 
 extern int * blksize_size[MAX_BLKDEV];

diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/drivers/ide/ide.c linux/drivers/ide/ide.c
@@ -577,7 +577,10 @@
 
 	if (!end_that_request_first(rq, uptodate, hwgroup->drive->name)) {
 		add_blkdev_randomness(MAJOR(rq->rq_dev));
-		blkdev_dequeue_request(rq);
+		if (!blk_rq_tagged(rq))
+			blkdev_dequeue_request(rq);
+		else
+			blk_queue_end_tag(&drive->queue, rq);
         	hwgroup->rq = NULL;
 		end_that_request_last(rq);
 	}

@@ -1577,13 +1580,31 @@
 		}
 		hwgroup->hwif = hwif;
 		hwgroup->drive = drive;
+queue_next:
+		if (!ata_can_queue(drive)) {
+			if (!ata_pending_commands(drive))
+				hwgroup->busy = 0;
+			break;
+		}
+
 		drive->sleep = 0;
 		drive->service_start = jiffies;
 
-		if ( drive->queue.plugged )	/* paranoia */
+		if (drive->queue.plugged) {
+			if (drive->using_tcq)
+				break;
 			printk("%s: Huh? nuking plugged queue\n", drive->name);
+		}
+
+		if (list_empty(&drive->queue.queue_head))
+			break;
+
+		rq = blkdev_entry_next_request(&drive->queue.queue_head);
+		if (!rq->bh && ata_pending_commands(drive))
+			break;
+
+		hwgroup->rq = rq;
 
-		rq = hwgroup->rq = blkdev_entry_next_request(&drive->queue.queue_head);
 		/*
 		 * Some systems have trouble with IDE IRQs arriving while
 		 * the driver is still setting things up.  So, here we disable

@@ -1602,6 +1623,8 @@
 			enable_irq(hwif->irq);
 		if (startstop == ide_stopped)
 			hwgroup->busy = 0;
+		else if (startstop == ide_released)
+			goto queue_next;
 	}
 }
 

348 lines not on this page · raw message

90 lines not on this page · raw message

diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/drivers/ide/ide-probe.c linux/drivers/ide/ide-probe.c
@@ -176,6 +176,17 @@
 	drive->media = ide_disk;
 	printk("ATA DISK drive\n");
 	QUIRK_LIST(HWIF(drive),drive);
+
+	/* Initialize queue depth settings */
+	drive->queue_depth = 1;
+#ifdef CONFIG_BLK_DEV_IDE_TCQ_DEPTH
+	drive->queue_depth = CONFIG_BLK_DEV_IDE_TCQ_DEPTH;
+#else
+	drive->queue_depth = drive->id->queue_depth + 1;
+#endif
+	if (drive->queue_depth < 1 || drive->queue_depth > IDE_MAX_TAG)
+		drive->queue_depth = IDE_MAX_TAG;
+
 	return;
 
 err_misc:

diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/drivers/ide/ide-taskfile.c linux/drivers/ide/ide-taskfile.c
@@ -243,8 +243,38 @@
 		}
 #endif
 	} else {
-		/* for dma commands we down set the handler */
-		if (drive->using_dma && !(HWIF(drive)->dmaproc(((taskfile->command == WIN_WRITEDMA) || (taskfile->command == WIN_WRITEDMA_EXT)) ? ide_dma_write : ide_dma_read, drive)));
+		ide_dma_action_t dma_act;
+		int tcq = 0;
+
+		if (!drive->using_dma)
+			return ide_stopped;
+
+		/* for dma commands we don't set the handler */
+		if (taskfile->command == WIN_WRITEDMA || taskfile->command == WIN_WRITEDMA_EXT)
+			dma_act = ide_dma_write;
+		else if (taskfile->command == WIN_READDMA || taskfile->command == WIN_READDMA_EXT)
+			dma_act = ide_dma_read;
+		else if (taskfile->command == WIN_WRITEDMA_QUEUED || taskfile->command == WIN_WRITEDMA_QUEUED_EXT) {
+			tcq = 1;
+			dma_act = ide_dma_write_queued;
+		} else if (taskfile->command == WIN_READDMA_QUEUED || taskfile->command == WIN_READDMA_QUEUED_EXT) {
+			tcq = 1;
+			dma_act = ide_dma_read_queued;
+		} else {
+			printk("ata_taskfile: unknown command %x\n", taskfile->command);
+			return ide_stopped;
+		}
+
+		/*
+		 * FIXME: this is a gross hack, need to unify tcq dma proc and
+		 * regular dma proc -- basically split stuff that needs to act
+		 * on a request from things like ide_dma_check etc.
+		 */
+		if (tcq)
+			return HWIF(drive)->dmaproc(dma_act, drive);
+
+		if (HWIF(drive)->dmaproc(dma_act, drive))
+			return ide_stopped;
 	}
 
 	return ide_started;

672 lines not on this page · raw message

diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/include/linux/hdreg.h linux/include/linux/hdreg.h
@@ -34,6 +34,7 @@
 #define ECC_STAT		0x04	/* Corrected error */
 #define DRQ_STAT		0x08
 #define SEEK_STAT		0x10
+#define SERVICE_STAT		SEEK_STAT
 #define WRERR_STAT		0x20
 #define READY_STAT		0x40
 #define BUSY_STAT		0x80

@@ -50,6 +51,13 @@
 #define ICRC_ERR		0x80	/* new meaning:  CRC error during transfer */
 
 /*
+ * sector count bits
+ */
+#define NSEC_CD			0x01
+#define NSEC_IO			0x02
+#define NSEC_REL		0x04
+
+/*
  * Command Header sizes for IOCTL commands
  *	HDIO_DRIVE_CMD, HDIO_DRIVE_TASK, and HDIO_DRIVE_TASKFILE
  */

104 lines not on this page · raw message