Skip to contentSkip to conversation

remove scsi_req_map_sg

PATCH In review FUJITA Tomonori 4 participants 38 messages Dec 13, 2008 – Feb 12, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

cover remove scsi_req_map_sg

This patchset removes scsi_req_map_sg. IOW, this patchset removes 'SCSI-ml does something wrong' comment in the block layer :)

/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,

                            struct bio *bio);

This patchset also removes scsi_execute_async and struct scsi_io_context() and unexport bio_add_pc_page().

This is against scsi-misc and depends on two patchsets:

'removing scsi_execute_async in st' the second half patchset:

http://marc.info/?l=linux-scsi&m=122900494917385&w=2

'removing scsi_execute_async in osst' the patchset:

http://marc.info/?l=linux-scsi&m=122918478521903&w=2

#1 lore raw

PATCH 1/3 remove scsi_req_map_sg

No one uses scsi_execute_async with data transfer now. We can remove scsi_req_map_sg.

Only scsi_eh_lock_door uses scsi_execute_async. scsi_eh_lock_door doesn't handle sense and the callback. So we can remove scsi_io_context too.

---
 drivers/scsi/scsi_error.c  |   34 +++++--
 drivers/scsi/scsi_lib.c    |  203 +-------------------------------------------
 include/scsi/scsi_device.h |    6 --
 3 files changed, 25 insertions(+), 218 deletions(-)
drivers/scsi/scsi_error.c +24 -10
index 381838e..29fb11a 100644
@@ -1453,6 +1453,11 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd)
 	}
 }
 
+static void eh_lock_door_done(struct request *req, int uptodate)
+{
+	__blk_put_request(req->q, req);
+}
+
 /**
  * scsi_eh_lock_door - Prevent medium removal for the specified device
  * @sdev:	SCSI device to prevent medium removal
@@ -1475,19 +1480,28 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd)
  */
 static void scsi_eh_lock_door(struct scsi_device *sdev)
 {
-	unsigned char cmnd[MAX_COMMAND_SIZE];
+	struct request *req;
 
-	cmnd[0] = ALLOW_MEDIUM_REMOVAL;
-	cmnd[1] = 0;
-	cmnd[2] = 0;
-	cmnd[3] = 0;
-	cmnd[4] = SCSI_REMOVAL_PREVENT;
-	cmnd[5] = 0;
+	req = blk_get_request(sdev->request_queue, READ, GFP_KERNEL);
+	if (!req)
+		return;
 
-	scsi_execute_async(sdev, cmnd, 6, DMA_NONE, NULL, 0, 0, 10 * HZ,
-			   5, NULL, NULL, GFP_KERNEL);
-}
+	req->cmd[0] = ALLOW_MEDIUM_REMOVAL;
+	req->cmd[1] = 0;
+	req->cmd[2] = 0;
+	req->cmd[3] = 0;
+	req->cmd[4] = SCSI_REMOVAL_PREVENT;
+	req->cmd[5] = 0;
 
+	req->cmd_len = COMMAND_SIZE(req->cmd[0]);
+
+	req->cmd_type = REQ_TYPE_BLOCK_PC;
+	req->cmd_flags |= REQ_QUIET;
+	req->timeout = 10 * HZ;
+	req->retries = 5;
+
+	blk_execute_rq_nowait(req->q, NULL, req, 1, eh_lock_door_done);
+}
 
 /**
  * scsi_restart_operations - restart io operations to the specified host.
drivers/scsi/scsi_lib.c +1 -202

232 lines not on this page · raw message

include/scsi/scsi_device.h +0 -6
index 01a4c58..d9e5ef0 100644
@@ -370,12 +370,6 @@ extern int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd,
 			    int data_direction, void *buffer, unsigned bufflen,
 			    struct scsi_sense_hdr *, int timeout, int retries,
 			    int *resid);
-extern int scsi_execute_async(struct scsi_device *sdev,
-			      const unsigned char *cmd, int cmd_len, int data_direction,
-			      void *buffer, unsigned bufflen, int use_sg,
-			      int timeout, int retries, void *privdata,
-			      void (*done)(void *, char *, int, int),
-			      gfp_t gfp);
 
 static inline int __must_check scsi_device_reprobe(struct scsi_device *sdev)
 {
Signature · 1 line
1.5.5.GIT
#2 lore raw

PATCH 2/3 block: unexport blk_rq_append_bio

Now we can unexport blk_rq_append_bio and remove 'SCSI-ml does something wrong' comment in blkdev.h. :)

---
 block/blk-map.c        |    5 ++---
 include/linux/blkdev.h |    6 ------
 2 files changed, 2 insertions(+), 9 deletions(-)
block/blk-map.c +2 -3
index e1bb727..a70f7d8 100644
@@ -9,8 +9,8 @@
 
 #include "blk.h"
 
-int blk_rq_append_bio(struct request_queue *q, struct request *rq,
-		      struct bio *bio)
+static int blk_rq_append_bio(struct request_queue *q, struct request *rq,
+			     struct bio *bio)
 {
 	if (!rq->bio)
 		blk_rq_bio_prep(q, rq, bio);
@@ -24,7 +24,6 @@ int blk_rq_append_bio(struct request_queue *q, struct request *rq,
 	}
 	return 0;
 }
-EXPORT_SYMBOL(blk_rq_append_bio);
 
 static int __blk_rq_unmap_user(struct bio *bio)
 {
include/linux/blkdev.h +0 -6
index 8922c25..bbd481f 100644
@@ -725,12 +725,6 @@ extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
 			 struct scsi_ioctl_command __user *);
 
 /*
- * Temporary export, until SCSI gets fixed up.
- */
-extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,
-			     struct bio *bio);
-
-/*
  * A queue has just exitted congestion.  Note this in the global counter of
  * congested queues, and wake up anyone who was waiting for requests to be
  * put back.
Signature · 1 line
1.5.5.GIT
#3 lore raw

PATCH 3/3 block: unexport bio_add_pc_page

We can unexport bio_add_pc_page since scsi_req_map_sg has gone.

---
 fs/bio.c            |    6 +++---
 include/linux/bio.h |    2 --
 2 files changed, 3 insertions(+), 5 deletions(-)
fs/bio.c +3 -3
index a482e1f..206209c 100644
@@ -464,8 +464,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page
  *      smaller than PAGE_SIZE, so it is always possible to add a single
  *      page to an empty bio. This should only be used by REQ_PC bios.
  */
-int bio_add_pc_page(struct request_queue *q, struct bio *bio, struct page *page,
-		    unsigned int len, unsigned int offset)
+static int bio_add_pc_page(struct request_queue *q, struct bio *bio,
+			   struct page *page, unsigned int len,
+			   unsigned int offset)
 {
 	return __bio_add_page(q, bio, page, len, offset, q->max_hw_sectors);
 }
@@ -1454,7 +1455,6 @@ EXPORT_SYMBOL(__bio_clone);
 EXPORT_SYMBOL(bio_clone);
 EXPORT_SYMBOL(bio_phys_segments);
 EXPORT_SYMBOL(bio_add_page);
-EXPORT_SYMBOL(bio_add_pc_page);
 EXPORT_SYMBOL(bio_get_nr_vecs);
 EXPORT_SYMBOL(bio_map_user);
 EXPORT_SYMBOL(bio_unmap_user);
include/linux/bio.h +0 -2
index 6a64209..87e7f77 100644
@@ -351,8 +351,6 @@ extern struct bio *bio_clone(struct bio *, gfp_t);
 extern void bio_init(struct bio *);
 
 extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
-extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
-			   unsigned int, unsigned int);
 extern int bio_get_nr_vecs(struct block_device *);
 extern sector_t bio_sector_offset(struct bio *, unsigned short, unsigned int);
 extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
Signature · 1 line
1.5.5.GIT
maintainer
#4 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

In reply to FUJITA Tomonori
Quoted 14 lines · FUJITA Tomonori

Now we can unexport blk_rq_append_bio and remove 'SCSI-ml does something wrong' comment in blkdev.h. :)

---
 block/blk-map.c        |    5 ++---
 include/linux/blkdev.h |    6 ------
 2 files changed, 2 insertions(+), 9 deletions(-)
block/blk-map.c +0 -0
index e1bb727..a70f7d8 100644

block/blk-map.c line 13

Quoted 7 lines
block/blk-map.c +2 -2
@@ -9,8 +9,8 @@
 
 #include "blk.h"
 
-int blk_rq_append_bio(struct request_queue *q, struct request *rq,
-		      struct bio *bio)
+static int blk_rq_append_bio(struct request_queue *q, struct request *rq,
+			     struct bio *bio)

There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().

It seems to be emulating block internals, so I think the fix is twofold:

1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.

I can code up a patch to see if it works.

James

maintainer
#5 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 10 lines · James Bottomley

There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().

It seems to be emulating block internals, so I think the fix is twofold:

1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.

I can code up a patch to see if it works.

So this is the patch to allow blk_rq_map_kern to append to requests, which is what I think is needed.

Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.

James

---

block/blk-map.c +8 -1
index 25d15ff..1a1e26d 100644
@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	int reading = rq_data_dir(rq) == READ;
 	int do_copy = 0;
 	struct bio *bio;
+	int ret;
 
 	if (len > (q->max_hw_sectors << 9))
 		return -EINVAL;
@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	if (do_copy)
 		rq->cmd_flags |= REQ_COPY_USER;
 
-	blk_rq_bio_prep(q, rq, bio);
+	ret = blk_rq_append_bio(q, rq, bio);
+	if (unlikely(ret)) {
+		/* request is too big */
+		bio_put(bio);
+		return ret;
+	}
+		
 	blk_queue_bounce(q, &rq->bio);
 	rq->buffer = rq->data = NULL;
 	return 0;
#6 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 10 lines · On Tue, 10 Feb 2009 18:19:33 +0000 James Bottomley…
Quoted 9 lines · James Bottomley

There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().

It seems to be emulating block internals, so I think the fix is twofold:

1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.

Quoted 7 lines
Quoted 1 line

I can code up a patch to see if it works.

So this is the patch to allow blk_rq_map_kern to append to requests, which is what I think is needed.

Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.

Looks like the osd initiator code uses bio nicely; e.g. using blk_rq_append_bio and bio_map_kern is completely wrong. I'll see the code to fix it...

#7 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

In reply to James Bottomley
Quoted 41 lines · James Bottomley
Quoted 10 lines · James Bottomley

There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().

It seems to be emulating block internals, so I think the fix is twofold:

1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.

I can code up a patch to see if it works.

So this is the patch to allow blk_rq_map_kern to append to requests, which is what I think is needed.

Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.

James

---

block/blk-map.c +1 -1
index 25d15ff..1a1e26d 100644
@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	int reading = rq_data_dir(rq) == READ;
 	int do_copy = 0;
 	struct bio *bio;
+	int ret;
 
 	if (len > (q->max_hw_sectors << 9))
 		return -EINVAL;
@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	if (do_copy)
 		rq->cmd_flags |= REQ_COPY_USER;
 
-	blk_rq_bio_prep(q, rq, bio);

block/blk-map.c int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · line 323

Quoted 10 lines
block/blk-map.c +7 -0
+	ret = blk_rq_append_bio(q, rq, bio);
+	if (unlikely(ret)) {
+		/* request is too big */
+		bio_put(bio);
+		return ret;
+	}
+		
 	blk_queue_bounce(q, &rq->bio);
 	rq->buffer = rq->data = NULL;
 	return 0;

This works, with your patch and below code
I'm passing my tests. I would say it is pretty safe too, since blk_rq_bio_prep would leak the bio if it was called twice on same request, before.

Thanks, I'll send a proper patch after some more testing.

---
git diff --stat -p

 drivers/scsi/osd/osd_initiator.c |   12 +++---------
 1 files changed, 3 insertions(+), 9 deletions(-)
drivers/scsi/osd/osd_initiator.c +3 -9
index 1696130..9cfdce4 100644
@@ -824,18 +824,12 @@ EXPORT_SYMBOL(osd_req_add_set_attr_list);
 static int _append_map_kern(struct request *req,
 	void *buff, unsigned len, gfp_t flags)
 {
-	struct bio *bio;
 	int ret;
 
-	bio = bio_map_kern(req->q, buff, len, flags);
-	if (IS_ERR(bio)) {
-		OSD_ERR("Failed bio_map_kern(%p, %d) => %ld\n", buff, len,
-			PTR_ERR(bio));
-		return PTR_ERR(bio);
-	}
-	ret = blk_rq_append_bio(req->q, req, bio);
+	ret = blk_rq_map_kern(req->q, req, buff, len, flags);
 	if (ret) {
-		OSD_ERR("Failed blk_rq_append_bio(%p) => %d\n", bio, ret);
+		OSD_DEBUG("Failed blk_rq_map_kern(%p, %u) => %d\n",
+			buff, len, ret);
 		bio_put(bio);
 	}
 	return ret;
#8 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 78 lines · Boaz Harrosh
Quoted 40 lines · James Bottomley
Quoted 10 lines · James Bottomley

There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().

It seems to be emulating block internals, so I think the fix is twofold:

1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.

I can code up a patch to see if it works.

So this is the patch to allow blk_rq_map_kern to append to requests, which is what I think is needed.

Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.

James

---

block/blk-map.c +1 -1
index 25d15ff..1a1e26d 100644
@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	int reading = rq_data_dir(rq) == READ;
 	int do_copy = 0;
 	struct bio *bio;
+	int ret;
 
 	if (len > (q->max_hw_sectors << 9))
 		return -EINVAL;
@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	if (do_copy)
 		rq->cmd_flags |= REQ_COPY_USER;
 
-	blk_rq_bio_prep(q, rq, bio);

block/blk-map.c int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · line 323

Quoted 10 lines
block/blk-map.c +7 -0
+	ret = blk_rq_append_bio(q, rq, bio);
+	if (unlikely(ret)) {
+		/* request is too big */
+		bio_put(bio);
+		return ret;
+	}
+		
 	blk_queue_bounce(q, &rq->bio);
 	rq->buffer = rq->data = NULL;
 	return 0;

This works, with your patch and below code
I'm passing my tests. I would say it is pretty safe too, since blk_rq_bio_prep would leak the bio if it was called twice on same request, before.

Thanks, I'll send a proper patch after some more testing.

---
git diff --stat -p

 drivers/scsi/osd/osd_initiator.c |   12 +++---------
 1 files changed, 3 insertions(+), 9 deletions(-)
drivers/scsi/osd/osd_initiator.c +0 -4
index 1696130..9cfdce4 100644
@@ -824,18 +824,12 @@ EXPORT_SYMBOL(osd_req_add_set_attr_list);
 static int _append_map_kern(struct request *req,
 	void *buff, unsigned len, gfp_t flags)
 {
-	struct bio *bio;
 	int ret;
 
-	bio = bio_map_kern(req->q, buff, len, flags);
-	if (IS_ERR(bio)) {
-		OSD_ERR("Failed bio_map_kern(%p, %d) => %ld\n", buff, len,

drivers/scsi/osd/osd_initiator.c EXPORT_SYMBOL(osd_req_add_set_attr_list); · line 833

Quoted 10 lines
drivers/scsi/osd/osd_initiator.c +3 -5
-			PTR_ERR(bio));
-		return PTR_ERR(bio);
-	}
-	ret = blk_rq_append_bio(req->q, req, bio);
+	ret = blk_rq_map_kern(req->q, req, buff, len, flags);
 	if (ret) {
-		OSD_ERR("Failed blk_rq_append_bio(%p) => %d\n", bio, ret);
+		OSD_DEBUG("Failed blk_rq_map_kern(%p, %u) => %d\n",
+			buff, len, ret);
 		bio_put(bio);

- bio_put(bio);
Sorry

Quoted 3 lines
drivers/scsi/osd/osd_initiator.c +0 -0
 	}
 	return ret;
#9 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

In reply to Boaz Harrosh
Quoted 82 lines · Boaz Harrosh
Quoted 40 lines · James Bottomley
Quoted 10 lines · James Bottomley

There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().

It seems to be emulating block internals, so I think the fix is twofold:

1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.

I can code up a patch to see if it works.

So this is the patch to allow blk_rq_map_kern to append to requests, which is what I think is needed.

Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.

James

---

block/blk-map.c +1 -1
index 25d15ff..1a1e26d 100644
@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	int reading = rq_data_dir(rq) == READ;
 	int do_copy = 0;
 	struct bio *bio;
+	int ret;
 
 	if (len > (q->max_hw_sectors << 9))
 		return -EINVAL;
@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,
 	if (do_copy)
 		rq->cmd_flags |= REQ_COPY_USER;
 
-	blk_rq_bio_prep(q, rq, bio);

block/blk-map.c int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · line 323

Quoted 10 lines
block/blk-map.c +7 -0
+	ret = blk_rq_append_bio(q, rq, bio);
+	if (unlikely(ret)) {
+		/* request is too big */
+		bio_put(bio);
+		return ret;
+	}
+		
 	blk_queue_bounce(q, &rq->bio);
 	rq->buffer = rq->data = NULL;
 	return 0;

This works, with your patch and below code
I'm passing my tests. I would say it is pretty safe too, since blk_rq_bio_prep would leak the bio if it was called twice on same request, before.

Thanks, I'll send a proper patch after some more testing.

---
git diff --stat -p

 drivers/scsi/osd/osd_initiator.c |   12 +++---------
 1 files changed, 3 insertions(+), 9 deletions(-)
drivers/scsi/osd/osd_initiator.c +0 -8
index 1696130..9cfdce4 100644
@@ -824,18 +824,12 @@ EXPORT_SYMBOL(osd_req_add_set_attr_list);
 static int _append_map_kern(struct request *req,
 	void *buff, unsigned len, gfp_t flags)
 {
-	struct bio *bio;
 	int ret;
 
-	bio = bio_map_kern(req->q, buff, len, flags);
-	if (IS_ERR(bio)) {
-		OSD_ERR("Failed bio_map_kern(%p, %d) => %ld\n", buff, len,
-			PTR_ERR(bio));
-		return PTR_ERR(bio);
-	}
-	ret = blk_rq_append_bio(req->q, req, bio);

drivers/scsi/osd/osd_initiator.c EXPORT_SYMBOL(osd_req_add_set_attr_list);

Quoted 10 lines
drivers/scsi/osd/osd_initiator.c +3 -2
+	ret = blk_rq_map_kern(req->q, req, buff, len, flags);
 	if (ret) {
-		OSD_ERR("Failed blk_rq_append_bio(%p) => %d\n", bio, ret);
+		OSD_DEBUG("Failed blk_rq_map_kern(%p, %u) => %d\n",
+			buff, len, ret);
 		bio_put(bio);
 	}
 	return ret;
 
--

I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.

I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)

So the first thing we have to decide is what API we need at read/write today there is:
void osd_req_read(struct osd_request *or,

	const struct osd_obj_id *, struct bio *data_in, u64 offset);

in exofs I use these two:

int osd_req_read_kern(struct osd_request *or,

	const struct osd_obj_id *obj, u64 offset, void *buff, u64 len);
int osd_req_read_pages(struct osd_request *or,
	const struct osd_obj_id *, u64 offset, u64 length,
	struct page **pages, int page_count);

(Same for write)
pNFS layout driver is very similar.

Thanks
Boaz

#10 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:

I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice

The API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?

Quoted 8 lines

general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.

I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)

We need to remove the usage of blk_rq_append_bio() in scsi.

Quoted 2 lines

So the first thing we have to decide is what API we need at read/write today there is:

You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.

Quoted 11 lines

void osd_req_read(struct osd_request *or,

	const struct osd_obj_id *, struct bio *data_in, u64 offset);

in exofs I use these two:

int osd_req_read_kern(struct osd_request *or,

	const struct osd_obj_id *obj, u64 offset, void *buff, u64 len);
int osd_req_read_pages(struct osd_request *or,
	const struct osd_obj_id *, u64 offset, u64 length,
	struct page **pages, int page_count);

As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.

#11 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 45 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:

I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice

The API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?

Quoted 8 lines

general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.

I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)

We need to remove the usage of blk_rq_append_bio() in scsi.

Quoted 2 lines

So the first thing we have to decide is what API we need at read/write today there is:

You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.

Quoted 11 lines

void osd_req_read(struct osd_request *or,

	const struct osd_obj_id *, struct bio *data_in, u64 offset);

in exofs I use these two:

int osd_req_read_kern(struct osd_request *or,

	const struct osd_obj_id *obj, u64 offset, void *buff, u64 len);
int osd_req_read_pages(struct osd_request *or,
	const struct osd_obj_id *, u64 offset, u64 length,
	struct page **pages, int page_count);
Quoted 2 lines

As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.

Sorry you miss understand me, I did not explain well.

int osd_req_read_pages(,,,, struct page **pages, int page_count);

Is in exofs.ko. It builds a bio out of **pages then calls osd_req_read(,,bio,) which will then call blk_rq_append_bio()

So for exofs (filesystem) and pNFS-objects layout driver (Also part of filesystem) I currently need blk_rq_append_bio(), unless we want to do something new for these people.

Thanks
Boaz

#12 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 49 lines · On Wed, 11 Feb 2009 16:52:37 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 45 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:

I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice

The API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?

Quoted 8 lines

general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.

I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)

We need to remove the usage of blk_rq_append_bio() in scsi.

Quoted 2 lines

So the first thing we have to decide is what API we need at read/write today there is:

You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.

Quoted 11 lines

void osd_req_read(struct osd_request *or,

	const struct osd_obj_id *, struct bio *data_in, u64 offset);

in exofs I use these two:

int osd_req_read_kern(struct osd_request *or,

	const struct osd_obj_id *obj, u64 offset, void *buff, u64 len);
int osd_req_read_pages(struct osd_request *or,
	const struct osd_obj_id *, u64 offset, u64 length,
	struct page **pages, int page_count);
Quoted 2 lines

As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.

Quoted 6 lines

Sorry you miss understand me, I did not explain well.

int osd_req_read_pages(,,,, struct page **pages, int page_count);

Is in exofs.ko. It builds a bio out of **pages then calls osd_req_read(,,bio,) which will then call blk_rq_append_bio()

As I wrote in the previous mail, why does exofs need to build a bio? Why can exofs pass pages to osd ULD?

Quoted 3 lines

So for exofs (filesystem) and pNFS-objects layout driver (Also part of filesystem) I currently need blk_rq_append_bio(), unless we want to do something new for these people.

#13 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 49 lines · FUJITA Tomonori
Quoted 46 lines · On Wed, 11 Feb 2009 16:52:37 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 35 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:

I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice

The API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?

Quoted 8 lines

general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.

I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)

We need to remove the usage of blk_rq_append_bio() in scsi.

Quoted 2 lines

So the first thing we have to decide is what API we need at read/write today there is:

You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.

Quoted 5 lines

void osd_req_read(struct osd_request *or,

	const struct osd_obj_id *, struct bio *data_in, u64 offset);

in exofs I use these two:

Quoted 8 lines
Quoted 6 lines

int osd_req_read_kern(struct osd_request *or,

	const struct osd_obj_id *obj, u64 offset, void *buff, u64 len);
int osd_req_read_pages(struct osd_request *or,
	const struct osd_obj_id *, u64 offset, u64 length,
	struct page **pages, int page_count);

As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.

Sorry you miss understand me, I did not explain well.

Quoted 9 lines
Quoted 4 lines

int osd_req_read_pages(,,,, struct page **pages, int page_count);

Is in exofs.ko. It builds a bio out of **pages then calls osd_req_read(,,bio,) which will then call blk_rq_append_bio()

As I wrote in the previous mail, why does exofs need to build a bio? Why can exofs pass pages to osd ULD?

Sure it can pass pages to ULD but then how ULD maks a request out of pages? And it gets more complicated then that (above multiple times)

But lets try to solve just the pages first.

Boaz

#14 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 55 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 54 lines · FUJITA Tomonori
Quoted 50 lines · On Wed, 11 Feb 2009 16:52:37 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 35 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:

I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice

The API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?

Quoted 8 lines

general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.

I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)

We need to remove the usage of blk_rq_append_bio() in scsi.

Quoted 2 lines

So the first thing we have to decide is what API we need at read/write today there is:

You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.

Quoted 5 lines

void osd_req_read(struct osd_request *or,

	const struct osd_obj_id *, struct bio *data_in, u64 offset);

in exofs I use these two:

Quoted 8 lines
Quoted 6 lines

int osd_req_read_kern(struct osd_request *or,

	const struct osd_obj_id *obj, u64 offset, void *buff, u64 len);
int osd_req_read_pages(struct osd_request *or,
	const struct osd_obj_id *, u64 offset, u64 length,
	struct page **pages, int page_count);

As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.

Sorry you miss understand me, I did not explain well.

int osd_req_read_pages(,,,, struct page **pages, int page_count);

Is in exofs.ko. It builds a bio out of **pages then calls osd_req_read(,,bio,) which will then call blk_rq_append_bio()

Quoted 7 lines
Quoted 4 lines

As I wrote in the previous mail, why does exofs need to build a bio? Why can exofs pass pages to osd ULD?

Sure it can pass pages to ULD but then how ULD maks a request out of pages?

If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).

I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).

Quoted 1 line

And it gets more complicated then that (above multiple times)

I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.

#15 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 11 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 2 lines

Sure it can pass pages to ULD but then how ULD maks a request out of pages?

If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).

What? I don't understand?
blk_rq_map_kern takes a kernel-pointer and length, how to pass array of page* ?

Quoted 5 lines

I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).

Yes adding a new mapping function can help. Please note I need to add (append) pages same as bio_add_pc_page() but on request level.

And yet we had an entry that did exactly that, blk_rq_append_bio(), no?

Quoted 5 lines
Quoted 2 lines

And it gets more complicated then that (above multiple times)

I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.

I have all the time in the world, And yes what James did with blk_rq_map_kern will help with appending other osd segments on top of data.

If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?

Thanks
Boaz

#16 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 9 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 8 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 2 lines

Sure it can pass pages to ULD but then how ULD maks a request out of pages?

Quoted 7 lines
Quoted 3 lines

If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).

What? I don't understand?
blk_rq_map_kern takes a kernel-pointer and length, how to pass array of page* ?

You can call it multiple times. But I guess that you need something new since you need to handle user pages.

Quoted 7 lines
Quoted 4 lines

I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).

Yes adding a new mapping function can help. Please note I need to add (append) pages same as bio_add_pc_page() but on request level.

I think that what you need already exists.

See how st uses blk_rq_map_user() in scsi-misc (keywords: rq_map_data and null_mapped). Please read it before saying something like 'we need to handle kernel buffer'.

Quoted 1 line

And yet we had an entry that did exactly that, blk_rq_append_bio(), no?

Using blk_rq_append_bio in SCSI is unacceptable.

Quoted 11 lines
Quoted 4 lines
Quoted 1 line

And it gets more complicated then that (above multiple times)

I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.

I have all the time in the world, And yes what James did with blk_rq_map_kern will help with appending other osd segments on top of data.

If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?

If you have time, how about making a proposal. ;)

But as I wrote above, I think that blk_rq_map_user() works for you.

maintainer
#17 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 52 lines · FUJITA Tomonori
Quoted 9 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 8 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 2 lines

Sure it can pass pages to ULD but then how ULD maks a request out of pages?

Quoted 7 lines
Quoted 3 lines

If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).

What? I don't understand?
blk_rq_map_kern takes a kernel-pointer and length, how to pass array of page* ?

You can call it multiple times. But I guess that you need something new since you need to handle user pages.

Quoted 7 lines
Quoted 4 lines

I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).

Yes adding a new mapping function can help. Please note I need to add (append) pages same as bio_add_pc_page() but on request level.

I think that what you need already exists.

See how st uses blk_rq_map_user() in scsi-misc (keywords: rq_map_data and null_mapped). Please read it before saying something like 'we need to handle kernel buffer'.

Quoted 1 line

And yet we had an entry that did exactly that, blk_rq_append_bio(), no?

Using blk_rq_append_bio in SCSI is unacceptable.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

And it gets more complicated then that (above multiple times)

I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.

I have all the time in the world, And yes what James did with blk_rq_map_kern will help with appending other osd segments on top of data.

Quoted 7 lines
Quoted 3 lines

If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?

If you have time, how about making a proposal. ;)

But as I wrote above, I think that blk_rq_map_user() works for you.

Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.

Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.

James

#18 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 49 lines · James Bottomley
Quoted 48 lines · FUJITA Tomonori
Quoted 5 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 4 lines · FUJITA Tomonori
Quoted 1 line · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 1 line
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Sure it can pass pages to ULD but then how ULD maks a request out of pages?

If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).

What? I don't understand?
blk_rq_map_kern takes a kernel-pointer and length, how to pass array of page* ?

You can call it multiple times. But I guess that you need something new since you need to handle user pages.

Quoted 6 lines
Quoted 4 lines

I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).

Yes adding a new mapping function can help. Please note I need to add (append) pages same as bio_add_pc_page() but on request level.

I think that what you need already exists.

See how st uses blk_rq_map_user() in scsi-misc (keywords: rq_map_data and null_mapped). Please read it before saying something like 'we need to handle kernel buffer'.

Quoted 1 line

And yet we had an entry that did exactly that, blk_rq_append_bio(), no?

Using blk_rq_append_bio in SCSI is unacceptable.

Quoted 9 lines
Quoted 3 lines
Quoted 1 line

And it gets more complicated then that (above multiple times)

I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.

I have all the time in the world, And yes what James did with blk_rq_map_kern will help with appending other osd segments on top of data.

If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?

If you have time, how about making a proposal. ;)

Quoted 8 lines
Quoted 1 line

But as I wrote above, I think that blk_rq_map_user() works for you.

Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.

You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?

Quoted 6 lines

Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.

I did not know I was reinventing the bio handling infrastructure, I thought I was using it with all available glory, and exported API. I let different users collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted, block_pc style.

Actually I just started to code using blk_rq_map_user(), and setting of all
struct rq_map_data information, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. struct rq_map_data assumes linear memory which I do not have. Also I will need that blk_rq_map_user() will append bios like the change you proposed to blk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awful blk_rq_unmap_user() which forces me to know if I sent via map_usr or map_kern ...)

I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:

	struct requst *blk_pc_make_request(bio);

Of sorts, to make it official, same as generic_make_request() but for block_pc users.

Quoted 4 lines

James

I will try the blk_rq_map_user, I'm sure it will not be pretty, though.

Boaz

#19 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 11 lines · On Wed, 11 Feb 2009 19:55:31 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 6 lines

Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.

You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?

Quoted 8 lines
Quoted 6 lines

Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.

I did not know I was reinventing the bio handling infrastructure, I thought

I think that we call something like the following reinventing the bio handling infrastructure:

static void _abort_unexecuted_bios(struct request *rq)

{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;
		bio_endio(bio, 0);
	}
}
Quoted 1 line

I was using it with all available glory, and exported API. I let different users

include/blkdev.h says:

/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,

			     struct bio *bio);

blk_rq_append_bio is not an exported API. We finally fixed SCSI but you broke it again.

Quoted 9 lines

collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted, block_pc style.

Actually I just started to code using blk_rq_map_user(), and setting of all
struct rq_map_data information, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. struct rq_map_data assumes linear memory which I do not have. Also I will need that blk_rq_map_user() will append bios like the change you proposed to blk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awful blk_rq_unmap_user() which forces me to know if I sent via map_usr or map_kern ...)

I think that having something like blk_rq_map_kern_page with James' extension is another option.

Quoted 2 lines

I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used

As I wrote, you use API that should not be exported. And I really don't think that SCSI ULD needs to play with bio. As James wrote, you reinvent and duplicate some of the block layer. That's a design that we should avoid.

Quoted 4 lines

by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:

	struct requst *blk_pc_make_request(bio);

Of sorts, to make it official, same as generic_make_request() but for block_pc users.

#20 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 25 lines · FUJITA Tomonori
Quoted 10 lines · On Wed, 11 Feb 2009 19:55:31 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 6 lines

Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.

You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?

Quoted 7 lines
Quoted 6 lines

Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.

I did not know I was reinventing the bio handling infrastructure, I thought

I think that we call something like the following reinventing the bio handling infrastructure:

static void _abort_unexecuted_bios(struct request *rq)

Quoted 9 lines
{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;
		bio_endio(bio, 0);
	}
}

No TOMO! The above is because I can start mapping a request even with simple map_kern and then fail and do not execute the request. If so, when I do a put_request() with a bio I'm leaking the bio. The above should be put inside put_request() so not to leak bios, but until then It's here.

It has nothing to do with our discussion.

Quoted 9 lines
Quoted 2 lines

I was using it with all available glory, and exported API. I let different users

include/blkdev.h says:

/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,

Quoted 6 lines
			     struct bio *bio);

blk_rq_append_bio is not an exported API. We finally fixed SCSI but you broke it again.

Fine, but I have a very legitimate need, which is:

Scatter-Gather Memory information was collected in a bio. At some stage I need to make it a request. How?

Quoted 11 lines
Quoted 10 lines

collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted, block_pc style.

Actually I just started to code using blk_rq_map_user(), and setting of all
struct rq_map_data information, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. struct rq_map_data assumes linear memory which I do not have. Also I will need that blk_rq_map_user() will append bios like the change you proposed to blk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awful blk_rq_unmap_user() which forces me to know if I sent via map_usr or map_kern ...)

Quoted 3 lines

I think that having something like blk_rq_map_kern_page with James' extension is another option.

That is a very good option. I will need a way to specify a page_max_count to pre-allocate the right size bio, so no bio re-allocations occurs.

Should I send a proposal for this?

[And yet there is such an exported API at bio level why not just open-code above, Linux Kernel
style]

Quoted 9 lines
Quoted 3 lines

I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used

As I wrote, you use API that should not be exported. And I really don't think that SCSI ULD needs to play with bio. As James wrote, you reinvent and duplicate some of the block layer. That's a design that we should avoid.

And as I wrote, I do not! Show me where? There is not a single line of duplication. I'm serving complex file systems with their crazy memory needs. I need some structure to hold scatter-gather information. The best API around that does just that, with nice stuff like slab pools and pre-allocations, is readily available at bio level. What you are asking me is to duplicate all that. So it is opposite of what you say, you are asking me to duplicate code I'm trying to re-use block layer code.

Quoted 5 lines
Quoted 5 lines

by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:

	struct requst *blk_pc_make_request(bio);

Of sorts, to make it official, same as generic_make_request() but for block_pc users.

TOMO James please answer my question.
What is bad with bio usage as a memory crier? Why can filesystems, diff-engines, DM, MD, you name it, use a bio, only OSD cannot? what is the reason? You say duplication of code? what code in OSD? which code is it duplicating?

Boaz

#21 lore raw

[RFD] blk_rq_map_pages new API

/**
* blk_rq_map_pages - Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
* blk_rq_map_pages can be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that case max_pages can be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */

int blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);

Implementation comments:
- Then when this member is available many places that call blk_rq_map_user() rq_map_data and null_mapped set, and buff == NULL, can be converted to this member.

- Internal block implementation is refactored to not duplicate any code with blk_rq_map_user().

Thanks for any comments
Boaz

#22 lore raw

Re: [RFD] blk_rq_map_pages new API

Quoted 29 lines · Boaz Harrosh

/**
* blk_rq_map_pages - Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
* blk_rq_map_pages can be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that case max_pages can be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */

int blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);

Implementation comments:
- Then when this member is available many places that call blk_rq_map_user() rq_map_data and null_mapped set, and buff == NULL, can be converted to this member.

- Internal block implementation is refactored to not duplicate any code with blk_rq_map_user().

Thanks for any comments
Boaz

--

Quoted 2 lines · FUJITA Tomonori

Please send a patch to implement the proposal.

Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.

If its OK I will implement it ASAP.

Thanks
Boaz

#23 lore raw

Re: [RFD] blk_rq_map_pages new API

Quoted 31 lines · On Thu, 12 Feb 2009 11:19:09 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 23 lines · Boaz Harrosh

/**
* blk_rq_map_pages - Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
* blk_rq_map_pages can be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that case max_pages can be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */

int blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);

Implementation comments:
- Then when this member is available many places that call blk_rq_map_user() rq_map_data and null_mapped set, and buff == NULL, can be converted to this member.

Quoted 6 lines

- Internal block implementation is refactored to not duplicate any code with blk_rq_map_user().

Thanks for any comments
Boaz

--

Quoted 8 lines
Quoted 2 lines · FUJITA Tomonori

Please send a patch to implement the proposal.

Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.

If its OK I will implement it ASAP.

Sorry, I'm not sure until I see the actual code. You always find tricky things when you actually write the code.

The patch should be small. It's not difficult at all to write a patch to see how it works.

#24 lore raw

Re: [RFD] blk_rq_map_pages new API

Quoted 41 lines · On Thu, 12 Feb 2009 18:50:33 +0900 FUJITA Tomonori…
Quoted 38 lines · On Thu, 12 Feb 2009 11:19:09 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 29 lines · Boaz Harrosh

/**
* blk_rq_map_pages - Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
* blk_rq_map_pages can be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that case max_pages can be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */

int blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);

Implementation comments:
- Then when this member is available many places that call blk_rq_map_user() rq_map_data and null_mapped set, and buff == NULL, can be converted to this member.

- Internal block implementation is refactored to not duplicate any code with blk_rq_map_user().

Thanks for any comments
Boaz

--

Quoted 2 lines · FUJITA Tomonori

Please send a patch to implement the proposal.

Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.

Quoted 7 lines
Quoted 1 line

If its OK I will implement it ASAP.

Sorry, I'm not sure until I see the actual code. You always find tricky things when you actually write the code.

The patch should be small. It's not difficult at all to write a patch to see how it works.

BTW, the idea is fine by me:

I have some page frames and want to build a request out of them.

That's what st and osst want. If OSD ULD wants it, adding a new API for it sounds a good idea to me. Then we can remove some features of blk_rq_map_user that I added for st and osst.

#25 lore raw

Re: [RFD] blk_rq_map_pages new API

Quoted 49 lines · FUJITA Tomonori
Quoted 39 lines · On Thu, 12 Feb 2009 18:50:33 +0900 FUJITA Tomonori…
Quoted 36 lines · On Thu, 12 Feb 2009 11:19:09 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 29 lines · Boaz Harrosh

/**
* blk_rq_map_pages - Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
* blk_rq_map_pages can be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that case max_pages can be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */

int blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);

Implementation comments:
- Then when this member is available many places that call blk_rq_map_user() rq_map_data and null_mapped set, and buff == NULL, can be converted to this member.

- Internal block implementation is refactored to not duplicate any code with blk_rq_map_user().

Thanks for any comments
Boaz

--

Quoted 2 lines · FUJITA Tomonori

Please send a patch to implement the proposal.

Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.

Quoted 6 lines
Quoted 1 line

If its OK I will implement it ASAP.

Sorry, I'm not sure until I see the actual code. You always find tricky things when you actually write the code.

The patch should be small. It's not difficult at all to write a patch to see how it works.

Quoted 7 lines

BTW, the idea is fine by me:

I have some page frames and want to build a request out of them.

That's what st and osst want. If OSD ULD wants it, adding a new API for it sounds a good idea to me. Then we can remove some features of blk_rq_map_user that I added for st and osst.

Right, I agree with you totally. I'm on it, I'll try an RFC see how it goes, and we can decide from there.

Thanks TOMO
Boaz

#26 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

In reply to Boaz Harrosh
Quoted 36 lines · On Thu, 12 Feb 2009 10:24:42 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 25 lines · FUJITA Tomonori
Quoted 10 lines · On Wed, 11 Feb 2009 19:55:31 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 6 lines

Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.

You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?

Quoted 7 lines
Quoted 6 lines

Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.

I did not know I was reinventing the bio handling infrastructure, I thought

I think that we call something like the following reinventing the bio handling infrastructure:

static void _abort_unexecuted_bios(struct request *rq)

Quoted 9 lines
{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;
		bio_endio(bio, 0);
	}
}
Quoted 6 lines

No TOMO! The above is because I can start mapping a request even with simple map_kern and then fail and do not execute the request. If so, when I do a put_request() with a bio I'm leaking the bio. The above should be put inside put_request() so not to leak bios, but until then It's here.

It has nothing to do with our discussion.

Seem that you don't understand what I talk about (and James, I think).

Of course, I understand the reason why you need that function.

The point is that OSD should not know how a request links bios.

OSD ULD should focus on OSD protocol processing. It should not know the details of requests, bio, etc. OSD ULD should use the proper helper functions of the block layer if OSD ULD uses the block layer service.

Quoted 11 lines
Quoted 11 lines
Quoted 1 line

I was using it with all available glory, and exported API. I let different users

include/blkdev.h says:

/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,

			     struct bio *bio);
Quoted 8 lines
Quoted 3 lines

blk_rq_append_bio is not an exported API. We finally fixed SCSI but you broke it again.

Fine, but I have a very legitimate need, which is:

Scatter-Gather Memory information was collected in a bio. At some stage I need to make it a request. How?

I don't think that it's not legitimate. See the above.

Quoted 14 lines
Quoted 10 lines
Quoted 9 lines

collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted, block_pc style.

Actually I just started to code using blk_rq_map_user(), and setting of all
struct rq_map_data information, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. struct rq_map_data assumes linear memory which I do not have. Also I will need that blk_rq_map_user() will append bios like the change you proposed to blk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awful blk_rq_unmap_user() which forces me to know if I sent via map_usr or map_kern ...)

Quoted 3 lines

I think that having something like blk_rq_map_kern_page with James' extension is another option.

Quoted 7 lines

That is a very good option. I will need a way to specify a page_max_count to pre-allocate the right size bio, so no bio re-allocations occurs.

Should I send a proposal for this?

[And yet there is such an exported API at bio level why not just open-code above, Linux Kernel
style]

Please send a patch to implement the proposal.

Quoted 22 lines
Quoted 8 lines
Quoted 2 lines

I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used

As I wrote, you use API that should not be exported. And I really don't think that SCSI ULD needs to play with bio. As James wrote, you reinvent and duplicate some of the block layer. That's a design that we should avoid.

And as I wrote, I do not! Show me where? There is not a single line of duplication. I'm serving complex file systems with their crazy memory needs. I need some structure to hold scatter-gather information. The best API around that does just that, with nice stuff like slab pools and pre-allocations, is readily available at bio level. What you are asking me is to duplicate all that. So it is opposite of what you say, you are asking me to duplicate code I'm trying to re-use block layer code.

Quoted 5 lines
Quoted 5 lines

by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:

	struct requst *blk_pc_make_request(bio);

Of sorts, to make it official, same as generic_make_request() but for block_pc users.

Quoted 4 lines

TOMO James please answer my question.
What is bad with bio usage as a memory crier? Why can filesystems, diff-engines, DM, MD, you name it, use a bio, only OSD cannot? what is the reason? You say duplication of code? what code in OSD? which code is it duplicating?

OSD ULD should focus on OSD protocol processing.

#27 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 25 lines · FUJITA Tomonori
Quoted 13 lines · On Thu, 12 Feb 2009 10:24:42 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 12 lines · FUJITA Tomonori

On Wed, 11 Feb 2009 19:55:31 +0200
Boaz Harrosh <[email protected]> wrote:
static void _abort_unexecuted_bios(struct request *rq)

{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;
		bio_endio(bio, 0);
	}
}
Quoted 6 lines

No TOMO! The above is because I can start mapping a request even with simple map_kern and then fail and do not execute the request. If so, when I do a put_request() with a bio I'm leaking the bio. The above should be put inside put_request() so not to leak bios, but until then It's here.

It has nothing to do with our discussion.

Seem that you don't understand what I talk about (and James, I think).

Quoted 9 lines

Of course, I understand the reason why you need that function.

The point is that OSD should not know how a request links bios.

OSD ULD should focus on OSD protocol processing. It should not know the details of requests, bio, etc. OSD ULD should use the proper helper functions of the block layer if OSD ULD uses the block layer service.

What? I'm talking about a BUG, here. What do you suggest that I leak bios until a fix to block is accepted?

And again this is unrelated. Even if I do exactly as you suggest which is use block API minus append_bio I still have that BUG. That must be fixed.

Please stop talking about above code. This is a different thread, unrelated to blk_rq_append_bio() removal.

Quoted 2 lines

OSD ULD should focus on OSD protocol processing.

Which is what it does exactly. You ask me to interfere at OSD level and set policy on memory structures. What I did is let osd_write/read just be a transparent pipe between the upper layer FS code and the low level block layer. Proof of that, any kind of implementation you suggested is that much more code at OSD level? Today it is dead simple.

Boaz

#28 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 30 lines · On Thu, 12 Feb 2009 11:14:02 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 29 lines · FUJITA Tomonori
Quoted 13 lines · On Thu, 12 Feb 2009 10:24:42 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 12 lines · FUJITA Tomonori

On Wed, 11 Feb 2009 19:55:31 +0200
Boaz Harrosh <[email protected]> wrote:
static void _abort_unexecuted_bios(struct request *rq)

{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;
		bio_endio(bio, 0);
	}
}
Quoted 6 lines

No TOMO! The above is because I can start mapping a request even with simple map_kern and then fail and do not execute the request. If so, when I do a put_request() with a bio I'm leaking the bio. The above should be put inside put_request() so not to leak bios, but until then It's here.

It has nothing to do with our discussion.

Seem that you don't understand what I talk about (and James, I think).

Of course, I understand the reason why you need that function.

The point is that OSD should not know how a request links bios.

Quoted 8 lines
Quoted 5 lines

OSD ULD should focus on OSD protocol processing. It should not know the details of requests, bio, etc. OSD ULD should use the proper helper functions of the block layer if OSD ULD uses the block layer service.

What? I'm talking about a BUG, here. What do you suggest that I leak bios until a fix to block is accepted?

Huh? What I said is something like the above function should live in the block layer. Again, OSD ULD should not know about such.

Quoted 7 lines

And again this is unrelated. Even if I do exactly as you suggest which is use block API minus append_bio I still have that BUG. That must be fixed.

Please stop talking about above code. This is a different thread, unrelated to blk_rq_append_bio() removal.

Quoted 1 line
Quoted 8 lines
Quoted 1 line

OSD ULD should focus on OSD protocol processing.

Which is what it does exactly. You ask me to interfere at OSD level and set policy on memory structures. What I did is let osd_write/read just be

a transparent pipe between the upper layer FS code and the low level block layer. Proof of that, any kind of implementation you suggested is that much more code at OSD level? Today it is dead simple.

Why is it so difficult for you to understand the meaning that OSD ULD should not know how request links bios?

#29 lore raw

RFC PATCH 1/1 block: Don't let blk_put_request leak BIOs

If a block ULD had allocated a request and mapped some memory into it, using one of blk_rq_map_xxx routines, but then for some reason failed to execute the request through one of the blk_execute_request routines. Then the associated BIO would leak, unless ULD resorts to low-level loops intimate of block internals.

[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we use WARN_ON_ONCE() in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and only WARN_ON if flag was not set.

I'm sending this before any-tests so people can comment on possible pitfalls.

---
 block/blk-core.c                 |   16 ++++++++++++++++
 drivers/scsi/osd/osd_initiator.c |   17 +----------------
 2 files changed, 17 insertions(+), 16 deletions(-)
block/blk-core.c +16 -0
index a824e49..eac96c2 100644
@@ -1055,6 +1055,21 @@ void part_round_stats(int cpu, struct hd_struct *part)
 EXPORT_SYMBOL_GPL(part_round_stats);
 
 /*
+ * If one of the blk_rq_map_xxx() was called but the request was not
+ * executed by the block layer, then we must release BIOs. Otherwise they
+ * will leak.
+ */
+static void _abort_unexecuted_bios(struct request *req)
+{
+	struct bio *bio;
+
+	while ((bio = req->bio) != NULL) {
+		req->bio = bio->bi_next;
+		bio_endio(bio, 0);
+	}
+}
+
+/*
  * queue lock must be held
  */
 void __blk_put_request(struct request_queue *q, struct request *req)
@@ -1066,6 +1081,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)
 
 	elv_completed_request(q, req);
 
+	_abort_unexecuted_bios(req);
 	/*
 	 * Request may not have originated from ll_rw_blk. if not,
 	 * it didn't come out of our reserved rq pools
drivers/scsi/osd/osd_initiator.c +1 -16
index 0bbbf27..8b1cf72 100644
@@ -335,20 +335,6 @@ struct osd_request *osd_start_request(struct osd_dev *dev, gfp_t gfp)
 }
 EXPORT_SYMBOL(osd_start_request);
 
-/*
- * If osd_finalize_request() was called but the request was not executed through
- * the block layer, then we must release BIOs.
- */
-static void _abort_unexecuted_bios(struct request *rq)
-{
-	struct bio *bio;
-
-	while ((bio = rq->bio) != NULL) {
-		rq->bio = bio->bi_next;
-		bio_endio(bio, 0);
-	}
-}
-
 static void _osd_free_seg(struct osd_request *or __unused,
 	struct _osd_req_data_segment *seg)
 {
@@ -370,11 +356,10 @@ void osd_end_request(struct osd_request *or)
 
 	if (rq) {
 		if (rq->next_rq) {
-			_abort_unexecuted_bios(rq->next_rq);
 			blk_put_request(rq->next_rq);
+			rq->next_rq = NULL;
 		}
 
-		_abort_unexecuted_bios(rq);
 		blk_put_request(rq);
 	}
 	_osd_request_free(or);
Signature · 1 line
1.6.0.1
#30 lore raw

Re: [PATCH] [RFC] block: Don't let blk_put_request leak BIOs

Quoted 14 lines · Boaz Harrosh

If a block ULD had allocated a request and mapped some memory into it, using one of blk_rq_map_xxx routines, but then for some reason failed to execute the request through one of the blk_execute_request routines. Then the associated BIO would leak, unless ULD resorts to low-level loops intimate of block internals.

[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we use WARN_ON_ONCE() in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and only WARN_ON if flag was not set.

Quoted 5 lines

I'm sending this before any-tests so people can comment on possible pitfalls.

---

I've booted a Linux PC with lots of sata disks, connected an iscsi target, ran OSD tests. It looks like it's working which means request->bio is set to NULL after it is used in the regular path.

This needs to sit in Linux next and be tested for a long while.

Jens I'll be waiting for your comment and will send a proper patch for the block bits. We will have to time this with James to see when the OSD bits can be submitted after that, then TOMO's patch for un-exporting blk_req_append_bio can be merged. Or maybe it can all go in one patch through scsi?

Thanks
Boaz

#31 lore raw

Re: [PATCH] [RFC] block: Don't let blk_put_request leak BIOs

Quoted 27 lines · Boaz Harrosh
Quoted 14 lines · Boaz Harrosh

If a block ULD had allocated a request and mapped some memory into it, using one of blk_rq_map_xxx routines, but then for some reason failed to execute the request through one of the blk_execute_request routines. Then the associated BIO would leak, unless ULD resorts to low-level loops intimate of block internals.

[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we use WARN_ON_ONCE() in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and only WARN_ON if flag was not set.

Quoted 5 lines

I'm sending this before any-tests so people can comment on possible pitfalls.

---

I've booted a Linux PC with lots of sata disks, connected an iscsi target, ran OSD tests. It looks like it's working which means request->bio is set to NULL after it is used in the regular path.

This needs to sit in Linux next and be tested for a long while.

Quoted 8 lines

Jens I'll be waiting for your comment and will send a proper patch for the block bits. We will have to time this with James to see when the OSD bits can be submitted after that, then TOMO's patch for un-exporting blk_req_append_bio can be merged. Or maybe it can all go in one patch through scsi?

Thanks
Boaz

Spock to soon this patch is shit, I'll look into it

Sorry for the noise

Boaz

#32 lore raw

RFC PATCH 1/2 block: Don't let blk_put_request leak BIOs

If a block ULD had allocated a request and mapped some memory into it, but then for some reason failed to execute the request through one of the blk_execute_request_xxx routines. Then the associated bio would leak, unless ULD resorts to low-level loops intimate of block internals.

For this to work I have fixed a couple of places in block/ where request->bio != NULL ownership was not honored. And a small cleanup at sg_io() while at it.

[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we use WARN_ON_ONCE() in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and only WARN_ON if flag was not set.
For the duration of linux-next I'm leaving the WARN_ON to catch any problems like found above, and possible memory leaks. Before submission a complimentary patch should remove the WARN_ON. (Or this patch can be rebased)

Please comment on possible pitfalls.

---
 block/blk-core.c   |   17 +++++++++++++++++
 block/blk-merge.c  |    2 ++
 block/scsi_ioctl.c |   21 ++++-----------------
 3 files changed, 23 insertions(+), 17 deletions(-)
block/blk-core.c +17 -0
index a824e49..3c1f920 100644
@@ -1055,6 +1055,22 @@ void part_round_stats(int cpu, struct hd_struct *part)
 EXPORT_SYMBOL_GPL(part_round_stats);
 
 /*
+ * If one of the blk_rq_map_xxx() was called but the request was not
+ * executed by the block layer, then we must release BIOs. Otherwise they
+ * will leak.
+ */
+static void _abort_unexecuted_bios(struct request *req)
+{
+	struct bio *bio;
+
+	WARN_ON(req->bio != NULL);
+	while (unlikely((bio = req->bio) != NULL)) {
+		req->bio = bio->bi_next;
+		bio_endio(bio, 0);
+	}
+}
+
+/*
  * queue lock must be held
  */
 void __blk_put_request(struct request_queue *q, struct request *req)
@@ -1066,6 +1082,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)
 
 	elv_completed_request(q, req);
 
+	_abort_unexecuted_bios(req);
 	/*
 	 * Request may not have originated from ll_rw_blk. if not,
 	 * it didn't come out of our reserved rq pools
block/blk-merge.c +2 -0
index b92f5b0..463e797 100644
@@ -398,6 +398,8 @@ static int attempt_merge(struct request_queue *q, struct request *req,
 	if (blk_rq_cpu_valid(next))
 		req->cpu = next->cpu;
 
+	/* owner-ship of bio passed from next to req */
+	next->bio = NULL;
 	__blk_put_request(q, next);
 	return 1;
 }
block/scsi_ioctl.c +4 -17
index ee9c67d..626ee27 100644
@@ -214,21 +214,10 @@ static int blk_fill_sghdr_rq(struct request_queue *q, struct request *rq,
 	return 0;
 }
 
-/*
- * unmap a request that was previously mapped to this sg_io_hdr. handles
- * both sg and non-sg sg_io_hdr.
- */
-static int blk_unmap_sghdr_rq(struct request *rq, struct sg_io_hdr *hdr)
-{
-	blk_rq_unmap_user(rq->bio);
-	blk_put_request(rq);
-	return 0;
-}
-
 static int blk_complete_sghdr_rq(struct request *rq, struct sg_io_hdr *hdr,
 				 struct bio *bio)
 {
-	int r, ret = 0;
+	int ret = 0;
 
 	/*
 	 * fill in all the output members
@@ -253,12 +242,10 @@ static int blk_complete_sghdr_rq(struct request *rq, struct sg_io_hdr *hdr,
 			ret = -EFAULT;
 	}
 
-	rq->bio = bio;
-	r = blk_unmap_sghdr_rq(rq, hdr);
-	if (ret)
-		r = ret;
+	blk_rq_unmap_user(bio);
+	blk_put_request(rq);
 
-	return r;
+	return ret;
 }
 
 static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
Signature · 1 line
1.6.0.1
#33 lore raw

RFC PATCH 2/2 libosd: Don't let osd abuse block internals, now that it's fixed

blk_put_request will delete any BIOs that where mapped but not executed, so osd_initiator does not have to take care of this situation any more.

This patch is dependent on block patch titled:

[PATCH 1/2] block: Don't let blk_put_request leak BIOs

---
 drivers/scsi/osd/osd_initiator.c |   17 +----------------
 1 files changed, 1 insertions(+), 16 deletions(-)
drivers/scsi/osd/osd_initiator.c +1 -16
index 0bbbf27..8b1cf72 100644
@@ -335,20 +335,6 @@ struct osd_request *osd_start_request(struct osd_dev *dev, gfp_t gfp)
 }
 EXPORT_SYMBOL(osd_start_request);
 
-/*
- * If osd_finalize_request() was called but the request was not executed through
- * the block layer, then we must release BIOs.
- */
-static void _abort_unexecuted_bios(struct request *rq)
-{
-	struct bio *bio;
-
-	while ((bio = rq->bio) != NULL) {
-		rq->bio = bio->bi_next;
-		bio_endio(bio, 0);
-	}
-}
-
 static void _osd_free_seg(struct osd_request *or __unused,
 	struct _osd_req_data_segment *seg)
 {
@@ -370,11 +356,10 @@ void osd_end_request(struct osd_request *or)
 
 	if (rq) {
 		if (rq->next_rq) {
-			_abort_unexecuted_bios(rq->next_rq);
 			blk_put_request(rq->next_rq);
+			rq->next_rq = NULL;
 		}
 
-		_abort_unexecuted_bios(rq);
 		blk_put_request(rq);
 	}
 	_osd_request_free(or);
Signature · 1 line
1.6.0.1
maintainer
#34 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

In reply to Boaz Harrosh
Quoted 59 lines · Boaz Harrosh
Quoted 49 lines · James Bottomley
Quoted 48 lines · FUJITA Tomonori
Quoted 5 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 4 lines · FUJITA Tomonori
Quoted 1 line · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 1 line
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines

Sure it can pass pages to ULD but then how ULD maks a request out of pages?

If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).

What? I don't understand?
blk_rq_map_kern takes a kernel-pointer and length, how to pass array of page* ?

You can call it multiple times. But I guess that you need something new since you need to handle user pages.

Quoted 6 lines
Quoted 4 lines

I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).

Yes adding a new mapping function can help. Please note I need to add (append) pages same as bio_add_pc_page() but on request level.

I think that what you need already exists.

See how st uses blk_rq_map_user() in scsi-misc (keywords: rq_map_data and null_mapped). Please read it before saying something like 'we need to handle kernel buffer'.

Quoted 1 line

And yet we had an entry that did exactly that, blk_rq_append_bio(), no?

Using blk_rq_append_bio in SCSI is unacceptable.

Quoted 9 lines
Quoted 3 lines
Quoted 1 line

And it gets more complicated then that (above multiple times)

I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.

I have all the time in the world, And yes what James did with blk_rq_map_kern will help with appending other osd segments on top of data.

If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?

If you have time, how about making a proposal. ;)

Quoted 8 lines
Quoted 1 line

But as I wrote above, I think that blk_rq_map_user() works for you.

Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.

Quoted 3 lines

You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?

Yes, but we've been trying to cut down on them. In fact with both osd and integrity, the use of bio handlers has been increasing.

Quoted 11 lines
Quoted 6 lines

Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.

I did not know I was reinventing the bio handling infrastructure, I thought I was using it with all available glory, and exported API. I let different users collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted, block_pc style.

Everything in your last sentence is how the block layer handles bios (although it's more sophisticated), so this is the duplication. The problem is that how bios are put together is a block internal thing. Nothing outside of block does that today ... if you construct your own multi-bio requests, we have to remember that drivers/scsi/osd needs updating as well if block ever alters this (say to cope with a new elevator strategy).

Quoted 6 lines

Actually I just started to code using blk_rq_map_user(), and setting of all
struct rq_map_data information, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. struct rq_map_data assumes linear memory which I do not have. Also I will need that blk_rq_map_user() will append bios like the change you proposed to blk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awful blk_rq_unmap_user() which forces me to know if I sent via map_usr or map_kern ...)

blk_rq_map_user does append pages to bios ... that's where I got the changes to blk_rq_map_kern() from.

Quoted 6 lines

I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:

	struct requst *blk_pc_make_request(bio);

Of sorts, to make it official, same as generic_make_request() but for block_pc users.

bios are supposed to be used by filesystems ... they represent the input to block. In principle, it seems nicer that PC requests deal with pages as their natural input ... that way we don't have to worry about exporting the block interfaces for bio->request construction.

Quoted 1 line

I will try the blk_rq_map_user, I'm sure it will not be pretty, though.

Great, thanks ... we can also adjust block interfaces to help.

James

#35 lore raw

Re: [PATCH 2/3] block: unexport blk_rq_append_bio

Quoted 6 lines · James Bottomley

bios are supposed to be used by filesystems ... they represent the input to block. In principle, it seems nicer that PC requests deal with pages as their natural input ... that way we don't have to worry about exporting the block interfaces for bio->request construction.

This is my point. I'm a filesystem and I want a natural route from bio to request, only with a paradigm shift as you call it. For a filesystem the most natural and comfortable vehicle is a bio.

So the interface I would like to have is:

    struct request *blk_make_request(struct request_queue *, struct bio *, gfp_t);

And let block layer take care of it's internals. Off course when crafting above the generic_make_request will use above API so there is absolutely no duplication of code and block layer is free to change as it feels like, only in a single place.

Quoted 6 lines
Quoted 1 line

I will try the blk_rq_map_user, I'm sure it will not be pretty, though.

Great, thanks ... we can also adjust block interfaces to help.

James

I'm on it. It's not as bad as I thought, and it should work for what I have now. I'm touching fast-path block layer code though, so we'll have to test it for a long while.

I have a gut feeling that I will regret it some day, but we'll cross that bridge when we get there.

Thank you for your help and concern.
Boaz

#36 lore raw
In reply to FUJITA Tomonori
Quoted 15 lines · FUJITA Tomonori

This patchset removes scsi_req_map_sg. IOW, this patchset removes 'SCSI-ml does something wrong' comment in the block layer :)

/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,

                            struct bio *bio);

This patchset also removes scsi_execute_async and struct scsi_io_context() and unexport bio_add_pc_page().

This is against scsi-misc and depends on two patchsets:

Quoted 7 lines

'removing scsi_execute_async in st' the second half patchset:

http://marc.info/?l=linux-scsi&m=122900494917385&w=2

'removing scsi_execute_async in osst' the patchset:

http://marc.info/?l=linux-scsi&m=122918478521903&w=2

You can add my Acked-by to all three patches, nice work Tomo!

Signature · 1 line
Jens Axboe
#37 lore raw
Quoted 18 lines · On Mon, 15 Dec 2008 08:28:17 +0100 Jens Axboe <[email protected]> wrote:
Quoted 17 lines · FUJITA Tomonori

This patchset removes scsi_req_map_sg. IOW, this patchset removes 'SCSI-ml does something wrong' comment in the block layer :)

/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,

                            struct bio *bio);

This patchset also removes scsi_execute_async and struct scsi_io_context() and unexport bio_add_pc_page().

This is against scsi-misc and depends on two patchsets:

'removing scsi_execute_async in st' the second half patchset:

Quoted 7 lines
Quoted 5 lines

http://marc.info/?l=linux-scsi&m=122900494917385&w=2

'removing scsi_execute_async in osst' the patchset:

http://marc.info/?l=linux-scsi&m=122918478521903&w=2

You can add my Acked-by to all three patches, nice work Tomo!

Thanks,

I also need your ACKs on the #1-3 patches in the following patchset to push the nice work. :)

http://marc.info/?l=linux-scsi&m=122958057719487&w=2