remove scsi_req_map_sg
Subsystems (3)
cover remove scsi_req_map_sg
This patchset removes scsi_req_map_sg. IOW, this patchset removes 'SCSI-ml does something wrong' comment in the block layer :)
/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,
struct bio *bio);
This patchset also removes scsi_execute_async and struct scsi_io_context() and unexport bio_add_pc_page().
This is against scsi-misc and depends on two patchsets:
'removing scsi_execute_async in st' the second half patchset:
http://marc.info/?l=linux-scsi&m=122900494917385&w=2
'removing scsi_execute_async in osst' the patchset:
PATCH 1/3 remove scsi_req_map_sg
No one uses scsi_execute_async with data transfer now. We can remove scsi_req_map_sg.
Only scsi_eh_lock_door uses scsi_execute_async. scsi_eh_lock_door doesn't handle sense and the callback. So we can remove scsi_io_context too.
--- drivers/scsi/scsi_error.c | 34 +++++-- drivers/scsi/scsi_lib.c | 203 +------------------------------------------- include/scsi/scsi_device.h | 6 -- 3 files changed, 25 insertions(+), 218 deletions(-)
drivers/scsi/scsi_error.c +24 -10
@@ -1453,6 +1453,11 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd) } } +static void eh_lock_door_done (struct request *req,int uptodate) +{ +__blk_put_request (req->q, req); +} +/** * scsi_eh_lock_door - Prevent medium removal for the specified device * @sdev: SCSI device to prevent medium removal
@@ -1475,19 +1480,28 @@ int scsi_decide_disposition(struct scsi_cmnd *scmd)*/ static void scsi_eh_lock_door (struct scsi_device *sdev) { -unsigned char cmnd[MAX_COMMAND_SIZE ]; +struct request *req; - cmnd[0 ] =ALLOW_MEDIUM_REMOVAL ; - cmnd[1 ] =0 ; - cmnd[2 ] =0 ; - cmnd[3 ] =0 ; - cmnd[4 ] =SCSI_REMOVAL_PREVENT ; - cmnd[5 ] =0 ; + req =blk_get_request (sdev->request_queue,READ ,GFP_KERNEL ); +if (!req) +return ; -scsi_execute_async (sdev, cmnd,6 ,DMA_NONE ,NULL ,0 ,0 ,10 *HZ , -5 ,NULL ,NULL ,GFP_KERNEL ); -} + req->cmd[0 ] =ALLOW_MEDIUM_REMOVAL ; + req->cmd[1 ] =0 ; + req->cmd[2 ] =0 ; + req->cmd[3 ] =0 ; + req->cmd[4 ] =SCSI_REMOVAL_PREVENT ; + req->cmd[5 ] =0 ; + req->cmd_len =COMMAND_SIZE (req->cmd[0 ]); + + req->cmd_type =REQ_TYPE_BLOCK_PC ; + req->cmd_flags |=REQ_QUIET ; + req->timeout =10 *HZ ; + req->retries =5 ; + +blk_execute_rq_nowait (req->q,NULL , req,1 , eh_lock_door_done); +}/** * scsi_restart_operations - restart io operations to the specified host.
drivers/scsi/scsi_lib.c +1 -202
232 lines not on this page · raw message
include/scsi/scsi_device.h +0 -6
@@ -370,12 +370,6 @@ extern int scsi_execute_req(struct scsi_device *sdev, const unsigned char *cmd, int data_direction, void *buffer, unsigned bufflen, struct scsi_sense_hdr *, int timeout, int retries, int *resid); -extern int scsi_execute_async (struct scsi_device *sdev, -const unsigned char *cmd,int cmd_len,int data_direction, -void *buffer,unsigned bufflen,int use_sg, -int timeout,int retries,void *privdata, -void (*done)(void *,char *,int ,int ), -gfp_t gfp);static inline int __must_check scsi_device_reprobe (struct scsi_device *sdev) {
Signature · 1 line
1.5.5.GIT
PATCH 2/3 block: unexport blk_rq_append_bio
Now we can unexport blk_rq_append_bio and remove 'SCSI-ml does something wrong' comment in blkdev.h. :)
--- block/blk-map.c | 5 ++--- include/linux/blkdev.h | 6 ------ 2 files changed, 2 insertions(+), 9 deletions(-)
block/blk-map.c +2 -3
@@ -9,8 +9,8 @@#include "blk.h" -int blk_rq_append_bio (struct request_queue *q,struct request *rq, -struct bio *bio) +static int blk_rq_append_bio (struct request_queue *q,struct request *rq, +struct bio *bio) {if (!rq->bio)blk_rq_bio_prep (q, rq, bio);
@@ -24,7 +24,6 @@ int blk_rq_append_bio(struct request_queue *q, struct request *rq, }return 0 ; } -EXPORT_SYMBOL (blk_rq_append_bio);static int __blk_rq_unmap_user (struct bio *bio) {
include/linux/blkdev.h +0 -6
@@ -725,12 +725,6 @@ extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t, struct scsi_ioctl_command __user *);/* -* Temporary export, until SCSI gets fixed up. -*/ -extern int blk_rq_append_bio (struct request_queue *q,struct request *rq, -struct bio *bio); - -/* * A queue has just exitted congestion. Note this in the global counter of * congested queues, and wake up anyone who was waiting for requests to be * put back.
Signature · 1 line
1.5.5.GIT
PATCH 3/3 block: unexport bio_add_pc_page
We can unexport bio_add_pc_page since scsi_req_map_sg has gone.
--- fs/bio.c | 6 +++--- include/linux/bio.h | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-)
fs/bio.c +3 -3
@@ -464,8 +464,9 @@ static int __bio_add_page(struct request_queue *q, struct bio *bio, struct page* smaller than PAGE_SIZE, so it is always possible to add a single * page to an empty bio. This should only be used by REQ_PC bios. */ -int bio_add_pc_page (struct request_queue *q,struct bio *bio,struct page *page, -unsigned int len,unsigned int offset) +static int bio_add_pc_page (struct request_queue *q,struct bio *bio, +struct page *page,unsigned int len, +unsigned int offset) {return __bio_add_page (q, bio, page, len, offset, q->max_hw_sectors); }
@@ -1454,7 +1455,6 @@ EXPORT_SYMBOL(__bio_clone);EXPORT_SYMBOL (bio_clone);EXPORT_SYMBOL (bio_phys_segments);EXPORT_SYMBOL (bio_add_page); -EXPORT_SYMBOL (bio_add_pc_page);EXPORT_SYMBOL (bio_get_nr_vecs);EXPORT_SYMBOL (bio_map_user);EXPORT_SYMBOL (bio_unmap_user);
include/linux/bio.h +0 -2
@@ -351,8 +351,6 @@ extern struct bio *bio_clone(struct bio *, gfp_t);extern void bio_init (struct bio *);extern int bio_add_page (struct bio *,struct page *,unsigned int ,unsigned int ); -extern int bio_add_pc_page (struct request_queue *,struct bio *,struct page *, -unsigned int ,unsigned int );extern int bio_get_nr_vecs (struct block_device *);extern sector_t bio_sector_offset (struct bio *,unsigned short ,unsigned int ); extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
Signature · 1 line
1.5.5.GIT
James Bottomley
maintainerRe: [PATCH 2/3] block: unexport blk_rq_append_bio
In reply to FUJITA TomonoriQuoted 14 lines · FUJITA Tomonori
Now we can unexport
blk_rq_append_bioand remove 'SCSI-ml does something wrong' comment inblkdev.h. :)Signed-off-by FUJITA Tomonori <[email protected]>--- block/blk-map.c | 5 ++--- include/linux/blkdev.h | 6 ------ 2 files changed, 2 insertions(+), 9 deletions(-)block/blk-map.c +0
-0
block/blk-map.c line 13
Quoted 7 lines
block/blk-map.c +2
-2@@ -9,8 +9,8 @@#include "blk.h" -int blk_rq_append_bio (struct request_queue *q,struct request *rq, -struct bio *bio) +static int blk_rq_append_bio (struct request_queue *q,struct request *rq, +struct bio *bio)
There's a current barrier to this: osd_initiator has also become a consumer of blk_rq_append_bio().
It seems to be emulating block internals, so I think the fix is twofold:
1. adjust blk_rq_map_kern to call blk_rq_append_bio() instead of blk_rq_prep_bio() (with an extra failure path).
2. make osd_initiator simply call it for additions.
I can code up a patch to see if it works.
James
James Bottomley
maintainerRe: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 10 lines · James Bottomley
There's a current barrier to this:
osd_initiatorhas also become a consumer ofblk_rq_append_bio().It seems to be emulating block internals, so I think the fix is twofold:
1. adjust
blk_rq_map_kernto callblk_rq_append_bio()instead ofblk_rq_prep_bio()(with an extra failure path).
2. makeosd_initiatorsimply call it for additions.I can code up a patch to see if it works.
So this is the patch to allow blk_rq_map_kern to append to requests, which is what I think is needed.
Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.
James
---
block/blk-map.c +8 -1
@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,int reading =rq_data_dir (rq) ==READ ;int do_copy =0 ;struct bio *bio; +int ret;if (len > (q->max_hw_sectors <<9 ))return -EINVAL ;
@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,if (do_copy) rq->cmd_flags |=REQ_COPY_USER ; -blk_rq_bio_prep (q, rq, bio); + ret =blk_rq_append_bio (q, rq, bio); +if (unlikely (ret)) { +/* request is too big */ +bio_put (bio); +return ret; + } +blk_queue_bounce (q, &rq->bio); rq->buffer = rq->data =NULL ;return 0 ;
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 10 lines · On Tue, 10 Feb 2009 18:19:33 +0000 James Bottomley…
Quoted 9 lines · James Bottomley
There's a current barrier to this:
osd_initiatorhas also become a consumer ofblk_rq_append_bio().It seems to be emulating block internals, so I think the fix is twofold:
1. adjust
blk_rq_map_kernto callblk_rq_append_bio()instead ofblk_rq_prep_bio()(with an extra failure path).
2. makeosd_initiatorsimply call it for additions.
Quoted 7 lines
Quoted 1 line
I can code up a patch to see if it works.
So this is the patch to allow
blk_rq_map_kernto append to requests, which is what I think is needed.Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.
Looks like the osd initiator code uses bio nicely; e.g. using blk_rq_append_bio and bio_map_kern is completely wrong. I'll see the code to fix it...
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
In reply to James BottomleyQuoted 41 lines · James Bottomley
Quoted 10 lines · James Bottomley
There's a current barrier to this:
osd_initiatorhas also become a consumer ofblk_rq_append_bio().It seems to be emulating block internals, so I think the fix is twofold:
1. adjust
blk_rq_map_kernto callblk_rq_append_bio()instead ofblk_rq_prep_bio()(with an extra failure path).
2. makeosd_initiatorsimply call it for additions.I can code up a patch to see if it works.
So this is the patch to allow
blk_rq_map_kernto append to requests, which is what I think is needed.Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.
James
---
block/blk-map.c +1
-1@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,int reading =rq_data_dir (rq) ==READ ;int do_copy =0 ;struct bio *bio; +int ret;if (len > (q->max_hw_sectors <<9 ))return -EINVAL ;@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,if (do_copy) rq->cmd_flags |=REQ_COPY_USER ; -blk_rq_bio_prep (q, rq, bio);
block/blk-map.c int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · line 323
Quoted 10 lines
block/blk-map.c +7
-0+ ret =blk_rq_append_bio (q, rq, bio); +if (unlikely (ret)) { +/* request is too big */ +bio_put (bio); +return ret; + } +blk_queue_bounce (q, &rq->bio); rq->buffer = rq->data =NULL ;return 0 ;
This works, with your patch and below code
I'm passing my tests. I would say it is pretty safe too, since blk_rq_bio_prep would leak the bio if it was called twice on same request, before.
Thanks, I'll send a proper patch after some more testing.
---
git diff --stat -p
drivers/scsi/osd/osd_initiator.c | 12 +++--------- 1 files changed, 3 insertions(+), 9 deletions(-)
drivers/scsi/osd/osd_initiator.c +3 -9
@@ -824,18 +824,12 @@ EXPORT_SYMBOL(osd_req_add_set_attr_list);static int _append_map_kern (struct request *req,void *buff,unsigned len,gfp_t flags) { -struct bio *bio;int ret; - bio =bio_map_kern (req->q, buff, len, flags); -if (IS_ERR (bio)) { -OSD_ERR ("Failed bio_map_kern(%p, %d) => %ld\n" , buff, len, -PTR_ERR (bio)); -return PTR_ERR (bio); - } - ret =blk_rq_append_bio (req->q, req, bio); + ret =blk_rq_map_kern (req->q, req, buff, len, flags);if (ret) { -OSD_ERR ("Failed blk_rq_append_bio(%p) => %d\n" , bio, ret); +OSD_DEBUG ("Failed blk_rq_map_kern(%p, %u) => %d\n" , + buff, len, ret);bio_put (bio); }return ret;
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 78 lines · Boaz Harrosh
Quoted 40 lines · James Bottomley
Quoted 10 lines · James Bottomley
There's a current barrier to this:
osd_initiatorhas also become a consumer ofblk_rq_append_bio().It seems to be emulating block internals, so I think the fix is twofold:
1. adjust
blk_rq_map_kernto callblk_rq_append_bio()instead ofblk_rq_prep_bio()(with an extra failure path).
2. makeosd_initiatorsimply call it for additions.I can code up a patch to see if it works.
So this is the patch to allow
blk_rq_map_kernto append to requests, which is what I think is needed.Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.
James
---
block/blk-map.c +1
-1@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,int reading =rq_data_dir (rq) ==READ ;int do_copy =0 ;struct bio *bio; +int ret;if (len > (q->max_hw_sectors <<9 ))return -EINVAL ;@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,if (do_copy) rq->cmd_flags |=REQ_COPY_USER ; -blk_rq_bio_prep (q, rq, bio);block/blk-map.c int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · line 323
Quoted 10 lines
block/blk-map.c +7
-0+ ret =blk_rq_append_bio (q, rq, bio); +if (unlikely (ret)) { +/* request is too big */ +bio_put (bio); +return ret; + } +blk_queue_bounce (q, &rq->bio); rq->buffer = rq->data =NULL ;return 0 ;This works, with your patch and below code
I'm passing my tests. I would say it is pretty safe too, sinceblk_rq_bio_prepwould leak the bio if it was called twice on same request, before.Thanks, I'll send a proper patch after some more testing.
---
git diff --stat -pdrivers/scsi/osd/osd_initiator.c | 12 +++--------- 1 files changed, 3 insertions(+), 9 deletions(-)drivers/scsi/osd/osd_initiator.c +0
-4@@ -824,18 +824,12 @@ EXPORT_SYMBOL(osd_req_add_set_attr_list);static int _append_map_kern (struct request *req,void *buff,unsigned len,gfp_t flags) { -struct bio *bio;int ret; - bio =bio_map_kern (req->q, buff, len, flags); -if (IS_ERR (bio)) { - OSD_ERR("Failed bio_map_kern(%p, %d) => %ld\n", buff, len,
drivers/scsi/osd/osd_initiator.c EXPORT_SYMBOL(osd_req_add_set_attr_list); · line 833
Quoted 10 lines
drivers/scsi/osd/osd_initiator.c +3
-5- PTR_ERR(bio)); -return PTR_ERR (bio); - } - ret =blk_rq_append_bio (req->q, req, bio); + ret =blk_rq_map_kern (req->q, req, buff, len, flags);if (ret) { -OSD_ERR ("Failed blk_rq_append_bio(%p) => %d\n" , bio, ret); +OSD_DEBUG ("Failed blk_rq_map_kern(%p, %u) => %d\n" , + buff, len, ret);bio_put (bio);
- bio_put(bio);
Sorry
Quoted 3 lines
drivers/scsi/osd/osd_initiator.c +0
-0}return ret;
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
In reply to Boaz HarroshQuoted 82 lines · Boaz Harrosh
Quoted 40 lines · James Bottomley
Quoted 10 lines · James Bottomley
There's a current barrier to this:
osd_initiatorhas also become a consumer ofblk_rq_append_bio().It seems to be emulating block internals, so I think the fix is twofold:
1. adjust
blk_rq_map_kernto callblk_rq_append_bio()instead ofblk_rq_prep_bio()(with an extra failure path).
2. makeosd_initiatorsimply call it for additions.I can code up a patch to see if it works.
So this is the patch to allow
blk_rq_map_kernto append to requests, which is what I think is needed.Unfortunately unwinding osd_initator's bio dependence and putting it back on data buffers looks to be a bit of a longer chore.
James
---
block/blk-map.c +1
-1@@ -289,6 +289,7 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,int reading =rq_data_dir (rq) ==READ ;int do_copy =0 ;struct bio *bio; +int ret;if (len > (q->max_hw_sectors <<9 ))return -EINVAL ;@@ -310,7 +311,13 @@ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf,if (do_copy) rq->cmd_flags |=REQ_COPY_USER ; -blk_rq_bio_prep (q, rq, bio);block/blk-map.c int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, · line 323
Quoted 10 lines
block/blk-map.c +7
-0+ ret =blk_rq_append_bio (q, rq, bio); +if (unlikely (ret)) { +/* request is too big */ +bio_put (bio); +return ret; + } +blk_queue_bounce (q, &rq->bio); rq->buffer = rq->data =NULL ;return 0 ;This works, with your patch and below code
I'm passing my tests. I would say it is pretty safe too, sinceblk_rq_bio_prepwould leak the bio if it was called twice on same request, before.Thanks, I'll send a proper patch after some more testing.
---
git diff --stat -pdrivers/scsi/osd/osd_initiator.c | 12 +++--------- 1 files changed, 3 insertions(+), 9 deletions(-)drivers/scsi/osd/osd_initiator.c +0
-8@@ -824,18 +824,12 @@ EXPORT_SYMBOL(osd_req_add_set_attr_list);static int _append_map_kern (struct request *req,void *buff,unsigned len,gfp_t flags) { -struct bio *bio;int ret; - bio =bio_map_kern (req->q, buff, len, flags); -if (IS_ERR (bio)) { -OSD_ERR ("Failed bio_map_kern(%p, %d) => %ld\n" , buff, len, -PTR_ERR (bio)); -return PTR_ERR (bio); - } - ret =blk_rq_append_bio (req->q, req, bio);
drivers/scsi/osd/osd_initiator.c EXPORT_SYMBOL(osd_req_add_set_attr_list);
Quoted 10 lines
drivers/scsi/osd/osd_initiator.c +3
-2+ ret =blk_rq_map_kern (req->q, req, buff, len, flags);if (ret) { -OSD_ERR ("Failed blk_rq_append_bio(%p) => %d\n" , bio, ret); +OSD_DEBUG ("Failed blk_rq_map_kern(%p, %u) => %d\n" , + buff, len, ret);bio_put (bio); }return ret; --
I spoke too soon. There is one more place that uses blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS generic_make_request() route.
I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs. blk_rq_append_bio() was the last way to associate a bio with a request. (except for privileged block-filesystems)
So the first thing we have to decide is what API we need at read/write today there is:
void osd_req_read(struct osd_request *or,
const struct osd_obj_id *, struct bio *data_in, u64 offset);
in exofs I use these two:
int osd_req_read_kern(struct osd_request *or,
const struct osd_obj_id *obj, u64 offset, void *buff, u64 len); int osd_req_read_pages(struct osd_request *or, const struct osd_obj_id *, u64 offset, u64 length, struct page **pages, int page_count);
(Same for write)
pNFS layout driver is very similar.
Thanks
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:
I spoke too soon. There is one more place that uses
blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a nice
The API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?
Quoted 8 lines
general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS
generic_make_request()route.I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs.
blk_rq_append_bio()was the last way to associate a bio with a request. (except for privileged block-filesystems)
We need to remove the usage of blk_rq_append_bio() in scsi.
Quoted 2 lines
So the first thing we have to decide is what API we need at read/write today there is:
You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.
Quoted 11 lines
void osd_req_read(struct
osd_request*or,const struct osd_obj_id *, struct bio *data_in, u64 offset);in exofs I use these two:
int osd_req_read_kern(struct
osd_request*or,const struct osd_obj_id *obj, u64 offset, void *buff, u64 len); int osd_req_read_pages(struct osd_request *or, const struct osd_obj_id *, u64 offset, u64 length, struct page **pages, int page_count);
As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 45 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:
I spoke too soon. There is one more place that uses
blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a niceThe API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?
Quoted 8 lines
general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS
generic_make_request()route.I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs.
blk_rq_append_bio()was the last way to associate a bio with a request. (except for privileged block-filesystems)We need to remove the usage of
blk_rq_append_bio()in scsi.Quoted 2 lines
So the first thing we have to decide is what API we need at read/write today there is:
You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.
Quoted 11 lines
void osd_req_read(struct
osd_request*or,const struct osd_obj_id *, struct bio *data_in, u64 offset);in exofs I use these two:
int osd_req_read_kern(struct
osd_request*or,const struct osd_obj_id *obj, u64 offset, void *buff, u64 len); int osd_req_read_pages(struct osd_request *or, const struct osd_obj_id *, u64 offset, u64 length, struct page **pages, int page_count);
Quoted 2 lines
As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.
Sorry you miss understand me, I did not explain well.
int osd_req_read_pages(,,,, struct page **pages, int page_count);
Is in exofs.ko. It builds a bio out of **pages then calls osd_req_read(,,bio,) which will then call blk_rq_append_bio()
So for exofs (filesystem) and pNFS-objects layout driver (Also part of filesystem) I currently need blk_rq_append_bio(), unless we want to do something new for these people.
Thanks
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 49 lines · On Wed, 11 Feb 2009 16:52:37 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 45 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:
I spoke too soon. There is one more place that uses
blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a niceThe API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?
Quoted 8 lines
general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS
generic_make_request()route.I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs.
blk_rq_append_bio()was the last way to associate a bio with a request. (except for privileged block-filesystems)We need to remove the usage of
blk_rq_append_bio()in scsi.Quoted 2 lines
So the first thing we have to decide is what API we need at read/write today there is:
You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.
Quoted 11 lines
void osd_req_read(struct
osd_request*or,const struct osd_obj_id *, struct bio *data_in, u64 offset);in exofs I use these two:
int osd_req_read_kern(struct
osd_request*or,const struct osd_obj_id *obj, u64 offset, void *buff, u64 len); int osd_req_read_pages(struct osd_request *or, const struct osd_obj_id *, u64 offset, u64 length, struct page **pages, int page_count);Quoted 2 lines
As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.
Quoted 6 lines
Sorry you miss understand me, I did not explain well.
int
osd_req_read_pages(,,,, struct page **pages, int page_count);Is in exofs.ko. It builds a bio out of **pages then calls
osd_req_read(,,bio,)which will then callblk_rq_append_bio()
As I wrote in the previous mail, why does exofs need to build a bio? Why can exofs pass pages to osd ULD?
Quoted 3 lines
So for exofs (filesystem) and pNFS-objects layout driver (Also part of filesystem) I currently need
blk_rq_append_bio(), unless we want to do something new for these people.
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 49 lines · FUJITA Tomonori
Quoted 46 lines · On Wed, 11 Feb 2009 16:52:37 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 35 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:
I spoke too soon. There is one more place that uses
blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a niceThe API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?
Quoted 8 lines
general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS
generic_make_request()route.I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs.
blk_rq_append_bio()was the last way to associate a bio with a request. (except for privileged block-filesystems)We need to remove the usage of
blk_rq_append_bio()in scsi.Quoted 2 lines
So the first thing we have to decide is what API we need at read/write today there is:
You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.
Quoted 5 lines
void osd_req_read(struct
osd_request*or,const struct osd_obj_id *, struct bio *data_in, u64 offset);in exofs I use these two:
Quoted 8 lines
Quoted 6 lines
int osd_req_read_kern(struct
osd_request*or,const struct osd_obj_id *obj, u64 offset, void *buff, u64 len); int osd_req_read_pages(struct osd_request *or, const struct osd_obj_id *, u64 offset, u64 length, struct page **pages, int page_count);As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.
Sorry you miss understand me, I did not explain well.
Quoted 9 lines
Quoted 4 lines
int
osd_req_read_pages(,,,, struct page **pages, int page_count);Is in exofs.ko. It builds a bio out of **pages then calls
osd_req_read(,,bio,)which will then callblk_rq_append_bio()As I wrote in the previous mail, why does exofs need to build a bio? Why can exofs pass pages to osd ULD?
Sure it can pass pages to ULD but then how ULD maks a request out of pages? And it gets more complicated then that (above multiple times)
But lets try to solve just the pages first.
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 55 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 54 lines · FUJITA Tomonori
Quoted 50 lines · On Wed, 11 Feb 2009 16:52:37 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 35 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 11:15:53 +0200 Boaz Harrosh <[email protected]> wrote:
I spoke too soon. There is one more place that uses
blk_rq_append_bio, that is the place that adds the read/write bio that is received in osd_req_write/read. The reason I receive a bio at these is because mainly I need a way to accept struct page* arrays, as well as kernel & user pointers. A bio is a niceThe API using an array of pointers to page can handle everything. So why do the users of ULD (some rare file systems, I guess), osd initiator, need to build bios and pass them?
Quoted 8 lines
general carrier for any type of memory. Given a bio at hand there are no ways left to prepare a request from it save the FS
generic_make_request()route.I was thinking of using struct sg_iovec* at one stage but they look very scary when used with page*, and mapping a page to a pointer but not doing cache sync and all that jazz. A bio is a very nice carrier of a scatter-gather list of memory. It has all the API for any needs.
blk_rq_append_bio()was the last way to associate a bio with a request. (except for privileged block-filesystems)We need to remove the usage of
blk_rq_append_bio()in scsi.Quoted 2 lines
So the first thing we have to decide is what API we need at read/write today there is:
You are talking about the API for osd file systems (or something related with osd), right? If so, I think that you can do whatever you want to do now. You can make a mistake since it's in-kernel API.
Quoted 5 lines
void osd_req_read(struct
osd_request*or,const struct osd_obj_id *, struct bio *data_in, u64 offset);in exofs I use these two:
Quoted 8 lines
Quoted 6 lines
int osd_req_read_kern(struct
osd_request*or,const struct osd_obj_id *obj, u64 offset, void *buff, u64 len); int osd_req_read_pages(struct osd_request *or, const struct osd_obj_id *, u64 offset, u64 length, struct page **pages, int page_count);As I wrote above, if you have an interface handling 'struct page **pages', then there should be ok.
Sorry you miss understand me, I did not explain well.
int
osd_req_read_pages(,,,, struct page **pages, int page_count);Is in exofs.ko. It builds a bio out of **pages then calls
osd_req_read(,,bio,)which will then callblk_rq_append_bio()
Quoted 7 lines
Quoted 4 lines
As I wrote in the previous mail, why does exofs need to build a bio? Why can exofs pass pages to osd ULD?
Sure it can pass pages to ULD but then how ULD maks a request out of pages?
If you extend blk_rq_map_kern as James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).
I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended blk_rq_map_user).
Quoted 1 line
And it gets more complicated then that (above multiple times)
I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 11 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 2 lines
Sure it can pass pages to ULD but then how ULD maks a request out of pages?
If you extend
blk_rq_map_kernas James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).
What? I don't understand?blk_rq_map_kern takes a kernel-pointer and length, how to pass array of page* ?
Quoted 5 lines
I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended
blk_rq_map_user).
Yes adding a new mapping function can help. Please note I need to add (append) pages same as bio_add_pc_page() but on request level.
And yet we had an entry that did exactly that, blk_rq_append_bio(), no?
Quoted 5 lines
Quoted 2 lines
And it gets more complicated then that (above multiple times)
I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.
I have all the time in the world, And yes what James did with blk_rq_map_kern will help with appending other osd segments on top of data.
If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?
Thanks
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 9 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 8 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 2 lines
Sure it can pass pages to ULD but then how ULD maks a request out of pages?
Quoted 7 lines
Quoted 3 lines
If you extend
blk_rq_map_kernas James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).What? I don't understand?
blk_rq_map_kerntakes a kernel-pointer and length, how to pass array of page* ?
You can call it multiple times. But I guess that you need something new since you need to handle user pages.
Quoted 7 lines
Quoted 4 lines
I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended
blk_rq_map_user).Yes adding a new mapping function can help. Please note I need to add (append) pages same as
bio_add_pc_page()but on request level.
I think that what you need already exists.
See how st uses blk_rq_map_user() in scsi-misc (keywords: rq_map_data and null_mapped). Please read it before saying something like 'we need to handle kernel buffer'.
Quoted 1 line
And yet we had an entry that did exactly that,
blk_rq_append_bio(), no?
Using blk_rq_append_bio in SCSI is unacceptable.
Quoted 11 lines
Quoted 4 lines
Quoted 1 line
And it gets more complicated then that (above multiple times)
I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.
I have all the time in the world, And yes what James did with
blk_rq_map_kernwill help with appending other osd segments on top of data.If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?
If you have time, how about making a proposal. ;)
But as I wrote above, I think that blk_rq_map_user() works for you.
James Bottomley
maintainerRe: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 52 lines · FUJITA Tomonori
Quoted 9 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 8 lines · FUJITA Tomonori
Quoted 4 lines · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 2 lines
Sure it can pass pages to ULD but then how ULD maks a request out of pages?
Quoted 7 lines
Quoted 3 lines
If you extend
blk_rq_map_kernas James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).What? I don't understand?
blk_rq_map_kerntakes a kernel-pointer and length, how to pass array of page* ?You can call it multiple times. But I guess that you need something new since you need to handle user pages.
Quoted 7 lines
Quoted 4 lines
I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended
blk_rq_map_user).Yes adding a new mapping function can help. Please note I need to add (append) pages same as
bio_add_pc_page()but on request level.I think that what you need already exists.
See how st uses
blk_rq_map_user()in scsi-misc (keywords:rq_map_dataandnull_mapped). Please read it before saying something like 'we need to handle kernel buffer'.Quoted 1 line
And yet we had an entry that did exactly that,
blk_rq_append_bio(), no?Using
blk_rq_append_bioin SCSI is unacceptable.Quoted 8 lines
Quoted 4 lines
Quoted 1 line
And it gets more complicated then that (above multiple times)
I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.
I have all the time in the world, And yes what James did with
blk_rq_map_kernwill help with appending other osd segments on top of data.
Quoted 7 lines
Quoted 3 lines
If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?
If you have time, how about making a proposal. ;)
But as I wrote above, I think that
blk_rq_map_user()works for you.
Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.
Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.
James
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 49 lines · James Bottomley
Quoted 48 lines · FUJITA Tomonori
Quoted 5 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 4 lines · FUJITA Tomonori
Quoted 1 line · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 1 line
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Sure it can pass pages to ULD but then how ULD maks a request out of pages?
If you extend
blk_rq_map_kernas James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).What? I don't understand?
blk_rq_map_kerntakes a kernel-pointer and length, how to pass array of page* ?You can call it multiple times. But I guess that you need something new since you need to handle user pages.
Quoted 6 lines
Quoted 4 lines
I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended
blk_rq_map_user).Yes adding a new mapping function can help. Please note I need to add (append) pages same as
bio_add_pc_page()but on request level.I think that what you need already exists.
See how st uses
blk_rq_map_user()in scsi-misc (keywords:rq_map_dataandnull_mapped). Please read it before saying something like 'we need to handle kernel buffer'.Quoted 1 line
And yet we had an entry that did exactly that,
blk_rq_append_bio(), no?Using
blk_rq_append_bioin SCSI is unacceptable.Quoted 9 lines
Quoted 3 lines
Quoted 1 line
And it gets more complicated then that (above multiple times)
I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.
I have all the time in the world, And yes what James did with
blk_rq_map_kernwill help with appending other osd segments on top of data.If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?
If you have time, how about making a proposal. ;)
Quoted 8 lines
Quoted 1 line
But as I wrote above, I think that
blk_rq_map_user()works for you.Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.
You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?
Quoted 6 lines
Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.
I did not know I was reinventing the bio handling infrastructure, I thought I was using it with all available glory, and exported API. I let different users collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted, block_pc style.
Actually I just started to code using blk_rq_map_user(), and setting of all
struct rq_map_data information, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. struct rq_map_data assumes linear memory which I do not have. Also I will need that blk_rq_map_user() will append bios like the change you proposed to blk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awful blk_rq_unmap_user() which forces me to know if I sent via map_usr or map_kern ...)
I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:
struct requst *blk_pc_make_request(bio);
Of sorts, to make it official, same as generic_make_request() but for block_pc users.
Quoted 4 lines
James
I will try the blk_rq_map_user, I'm sure it will not be pretty, though.
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 11 lines · On Wed, 11 Feb 2009 19:55:31 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 6 lines
Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.
You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?
Quoted 8 lines
Quoted 6 lines
Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.
I did not know I was reinventing the bio handling infrastructure, I thought
I think that we call something like the following reinventing the bio handling infrastructure:
static void _abort_unexecuted_bios(struct request *rq)
{
struct bio *bio;
while ((bio = rq->bio) != NULL) {
rq->bio = bio->bi_next;
bio_endio(bio, 0);
}
}Quoted 1 line
I was using it with all available glory, and exported API. I let different users
include/blkdev.h says:
/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(struct request_queue *q, struct request *rq,
struct bio *bio);
blk_rq_append_bio is not an exported API. We finally fixed SCSI but you broke it again.
Quoted 9 lines
collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted,
block_pcstyle.Actually I just started to code using
blk_rq_map_user(), and setting of all
structrq_map_datainformation, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. structrq_map_dataassumes linear memory which I do not have. Also I will need thatblk_rq_map_user()will append bios like the change you proposed toblk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awfulblk_rq_unmap_user()which forces me to know if I sent viamap_usrormap_kern...)
I think that having something like blk_rq_map_kern_page with James' extension is another option.
Quoted 2 lines
I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used
As I wrote, you use API that should not be exported. And I really don't think that SCSI ULD needs to play with bio. As James wrote, you reinvent and duplicate some of the block layer. That's a design that we should avoid.
Quoted 4 lines
by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:
struct requst *blk_pc_make_request(bio);Of sorts, to make it official, same as
generic_make_request()but forblock_pcusers.
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 25 lines · FUJITA Tomonori
Quoted 10 lines · On Wed, 11 Feb 2009 19:55:31 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 6 lines
Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.
You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?
Quoted 7 lines
Quoted 6 lines
Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.
I did not know I was reinventing the bio handling infrastructure, I thought
I think that we call something like the following reinventing the bio handling infrastructure:
static void
_abort_unexecuted_bios(struct request *rq)
Quoted 9 lines
{
struct bio *bio;
while ((bio = rq->bio) != NULL) {
rq->bio = bio->bi_next;
bio_endio(bio, 0);
}
}No TOMO! The above is because I can start mapping a request even with simple map_kern and then fail and do not execute the request. If so, when I do a put_request() with a bio I'm leaking the bio. The above should be put inside put_request() so not to leak bios, but until then It's here.
It has nothing to do with our discussion.
Quoted 9 lines
Quoted 2 lines
I was using it with all available glory, and exported API. I let different users
include/blkdev.hsays:/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(structrequest_queue*q, struct request *rq,
Quoted 6 lines
struct bio *bio);
blk_rq_append_biois not an exported API. We finally fixed SCSI but you broke it again.
Fine, but I have a very legitimate need, which is:
Scatter-Gather Memory information was collected in a bio. At some stage I need to make it a request. How?
Quoted 11 lines
Quoted 10 lines
collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted,
block_pcstyle.Actually I just started to code using
blk_rq_map_user(), and setting of all
structrq_map_datainformation, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. structrq_map_dataassumes linear memory which I do not have. Also I will need thatblk_rq_map_user()will append bios like the change you proposed toblk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awfulblk_rq_unmap_user()which forces me to know if I sent viamap_usrormap_kern...)
Quoted 3 lines
I think that having something like
blk_rq_map_kern_pagewith James' extension is another option.
That is a very good option. I will need a way to specify a page_max_count to pre-allocate the right size bio, so no bio re-allocations occurs.
Should I send a proposal for this?
[And yet there is such an exported API at bio level why not just open-code above, Linux Kernel
style]
Quoted 9 lines
Quoted 3 lines
I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used
As I wrote, you use API that should not be exported. And I really don't think that SCSI ULD needs to play with bio. As James wrote, you reinvent and duplicate some of the block layer. That's a design that we should avoid.
And as I wrote, I do not! Show me where? There is not a single line of duplication. I'm serving complex file systems with their crazy memory needs. I need some structure to hold scatter-gather information. The best API around that does just that, with nice stuff like slab pools and pre-allocations, is readily available at bio level. What you are asking me is to duplicate all that. So it is opposite of what you say, you are asking me to duplicate code I'm trying to re-use block layer code.
Quoted 5 lines
Quoted 5 lines
by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:
struct requst *blk_pc_make_request(bio);Of sorts, to make it official, same as
generic_make_request()but forblock_pcusers.
TOMO James please answer my question.
What is bad with bio usage as a memory crier? Why can filesystems, diff-engines, DM, MD, you name it, use a bio, only OSD cannot? what is the reason? You say duplication of code? what code in OSD? which code is it duplicating?
Boaz
[RFD] blk_rq_map_pages new API
/**
* blk_rq_map_pages - Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
* blk_rq_map_pages can be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that case max_pages can be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */
int blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);
Implementation comments:
- Then when this member is available many places that call blk_rq_map_user() rq_map_data and null_mapped set, and buff == NULL, can be converted to this member.
- Internal block implementation is refactored to not duplicate any code with blk_rq_map_user().
Thanks for any comments
Boaz
Re: [RFD] blk_rq_map_pages new API
Quoted 29 lines · Boaz Harrosh
/**
*blk_rq_map_pages- Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
*blk_rq_map_pagescan be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that casemax_pagescan be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */int
blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);Implementation comments:
- Then when this member is available many places that callblk_rq_map_user()rq_map_dataandnull_mappedset, and buff == NULL, can be converted to this member.- Internal block implementation is refactored to not duplicate any code with
blk_rq_map_user().Thanks for any comments
Boaz--
Quoted 2 lines · FUJITA Tomonori
Please send a patch to implement the proposal.
Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.
If its OK I will implement it ASAP.
Thanks
Boaz
Re: [RFD] blk_rq_map_pages new API
Quoted 31 lines · On Thu, 12 Feb 2009 11:19:09 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 23 lines · Boaz Harrosh
/**
*blk_rq_map_pages- Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
*blk_rq_map_pagescan be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that casemax_pagescan be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */int
blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);Implementation comments:
- Then when this member is available many places that callblk_rq_map_user()rq_map_dataandnull_mappedset, and buff == NULL, can be converted to this member.Quoted 6 lines
- Internal block implementation is refactored to not duplicate any code with
blk_rq_map_user().Thanks for any comments
Boaz--
Quoted 8 lines
Quoted 2 lines · FUJITA Tomonori
Please send a patch to implement the proposal.
Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.
If its OK I will implement it ASAP.
Sorry, I'm not sure until I see the actual code. You always find tricky things when you actually write the code.
The patch should be small. It's not difficult at all to write a patch to see how it works.
Re: [RFD] blk_rq_map_pages new API
Quoted 41 lines · On Thu, 12 Feb 2009 18:50:33 +0900 FUJITA Tomonori…
Quoted 38 lines · On Thu, 12 Feb 2009 11:19:09 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 29 lines · Boaz Harrosh
/**
*blk_rq_map_pages- Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
*blk_rq_map_pagescan be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that casemax_pagescan be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */int
blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);Implementation comments:
- Then when this member is available many places that callblk_rq_map_user()rq_map_dataandnull_mappedset, and buff == NULL, can be converted to this member.- Internal block implementation is refactored to not duplicate any code with
blk_rq_map_user().Thanks for any comments
Boaz--
Quoted 2 lines · FUJITA Tomonori
Please send a patch to implement the proposal.
Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.
Quoted 7 lines
Quoted 1 line
If its OK I will implement it ASAP.
Sorry, I'm not sure until I see the actual code. You always find tricky things when you actually write the code.
The patch should be small. It's not difficult at all to write a patch to see how it works.
BTW, the idea is fine by me:
I have some page frames and want to build a request out of them.
That's what st and osst want. If OSD ULD wants it, adding a new API for it sounds a good idea to me. Then we can remove some features of blk_rq_map_user that I added for st and osst.
Re: [RFD] blk_rq_map_pages new API
Quoted 49 lines · FUJITA Tomonori
Quoted 39 lines · On Thu, 12 Feb 2009 18:50:33 +0900 FUJITA Tomonori…
Quoted 36 lines · On Thu, 12 Feb 2009 11:19:09 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 29 lines · Boaz Harrosh
/**
*blk_rq_map_pages- Receives an array of pages and appends them to a request structure
*
* @rq: The request to map
* @pages_info: A structure that specifies the array of pages, offset and more that need to be
* map to the request. the @pages_info->null_mapped is assumed to be 1 and is ignored.
* @length: Total bytes to map. Array of pages can be larger, stop mapping after @length bytes mapped.
* @max_pages: When allocating the internal bio use @max_pages as an hint that says the amount of anticipated
* pages that will be mapped. This member is optional and can be zero.
*
*blk_rq_map_pagescan be called multiple times so the user does not need to allocate a contiguous array
* of struct page pointers but can call this routine multiple times. In that casemax_pagescan be set
* so no bio re-allocation occurs.
* There is no unmap function for this mapping, the request is completed in the regular way. */int
blk_rq_map_pages(struct request rq, struct *rq_map_data pages_info, unsigned length, unsigned max_pages);Implementation comments:
- Then when this member is available many places that callblk_rq_map_user()rq_map_dataandnull_mappedset, and buff == NULL, can be converted to this member.- Internal block implementation is refactored to not duplicate any code with
blk_rq_map_user().Thanks for any comments
Boaz--
Quoted 2 lines · FUJITA Tomonori
Please send a patch to implement the proposal.
Is the above API accepted by you? Is it accepted by Jens? Should I also attempt first comment above.
Quoted 6 lines
Quoted 1 line
If its OK I will implement it ASAP.
Sorry, I'm not sure until I see the actual code. You always find tricky things when you actually write the code.
The patch should be small. It's not difficult at all to write a patch to see how it works.
Quoted 7 lines
BTW, the idea is fine by me:
I have some page frames and want to build a request out of them.
That's what st and osst want. If OSD ULD wants it, adding a new API for it sounds a good idea to me. Then we can remove some features of
blk_rq_map_userthat I added for st and osst.
Right, I agree with you totally. I'm on it, I'll try an RFC see how it goes, and we can decide from there.
Thanks TOMO
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
In reply to Boaz HarroshQuoted 36 lines · On Thu, 12 Feb 2009 10:24:42 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 25 lines · FUJITA Tomonori
Quoted 10 lines · On Wed, 11 Feb 2009 19:55:31 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 6 lines
Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.
You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?
Quoted 7 lines
Quoted 6 lines
Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.
I did not know I was reinventing the bio handling infrastructure, I thought
I think that we call something like the following reinventing the bio handling infrastructure:
static void
_abort_unexecuted_bios(struct request *rq)Quoted 9 lines
{ struct bio *bio; while ((bio = rq->bio) != NULL) { rq->bio = bio->bi_next; bio_endio(bio, 0); } }
Quoted 6 lines
No TOMO! The above is because I can start mapping a request even with simple
map_kernand then fail and do not execute the request. If so, when I do aput_request()with a bio I'm leaking the bio. The above should be put insideput_request()so not to leak bios, but until then It's here.It has nothing to do with our discussion.
Seem that you don't understand what I talk about (and James, I think).
Of course, I understand the reason why you need that function.
The point is that OSD should not know how a request links bios.
OSD ULD should focus on OSD protocol processing. It should not know the details of requests, bio, etc. OSD ULD should use the proper helper functions of the block layer if OSD ULD uses the block layer service.
Quoted 11 lines
Quoted 11 lines
Quoted 1 line
I was using it with all available glory, and exported API. I let different users
include/blkdev.hsays:/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(structrequest_queue*q, struct request *rq,struct bio *bio);
Quoted 8 lines
Quoted 3 lines
blk_rq_append_biois not an exported API. We finally fixed SCSI but you broke it again.Fine, but I have a very legitimate need, which is:
Scatter-Gather Memory information was collected in a bio. At some stage I need to make it a request. How?
I don't think that it's not legitimate. See the above.
Quoted 14 lines
Quoted 10 lines
Quoted 9 lines
collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted,
block_pcstyle.Actually I just started to code using
blk_rq_map_user(), and setting of all
structrq_map_datainformation, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. structrq_map_dataassumes linear memory which I do not have. Also I will need thatblk_rq_map_user()will append bios like the change you proposed toblk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awfulblk_rq_unmap_user()which forces me to know if I sent viamap_usrormap_kern...)Quoted 3 lines
I think that having something like
blk_rq_map_kern_pagewith James' extension is another option.
Quoted 7 lines
That is a very good option. I will need a way to specify a
page_max_countto pre-allocate the right size bio, so no bio re-allocations occurs.Should I send a proposal for this?
[And yet there is such an exported API at bio level why not just open-code above, Linux Kernel
style]
Please send a patch to implement the proposal.
Quoted 22 lines
Quoted 8 lines
Quoted 2 lines
I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used
As I wrote, you use API that should not be exported. And I really don't think that SCSI ULD needs to play with bio. As James wrote, you reinvent and duplicate some of the block layer. That's a design that we should avoid.
And as I wrote, I do not! Show me where? There is not a single line of duplication. I'm serving complex file systems with their crazy memory needs. I need some structure to hold scatter-gather information. The best API around that does just that, with nice stuff like slab pools and pre-allocations, is readily available at bio level. What you are asking me is to duplicate all that. So it is opposite of what you say, you are asking me to duplicate code I'm trying to re-use block layer code.
Quoted 5 lines
Quoted 5 lines
by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:
struct requst *blk_pc_make_request(bio);Of sorts, to make it official, same as
generic_make_request()but forblock_pcusers.
Quoted 4 lines
TOMO James please answer my question.
What is bad with bio usage as a memory crier? Why can filesystems, diff-engines, DM, MD, you name it, use a bio, only OSD cannot? what is the reason? You say duplication of code? what code in OSD? which code is it duplicating?
OSD ULD should focus on OSD protocol processing.
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 25 lines · FUJITA Tomonori
Quoted 13 lines · On Thu, 12 Feb 2009 10:24:42 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 12 lines · FUJITA Tomonori
On Wed, 11 Feb 2009 19:55:31 +0200
Boaz Harrosh <[email protected]> wrote:
static void_abort_unexecuted_bios(struct request *rq){ struct bio *bio; while ((bio = rq->bio) != NULL) { rq->bio = bio->bi_next; bio_endio(bio, 0); } }Quoted 6 lines
No TOMO! The above is because I can start mapping a request even with simple
map_kernand then fail and do not execute the request. If so, when I do aput_request()with a bio I'm leaking the bio. The above should be put insideput_request()so not to leak bios, but until then It's here.It has nothing to do with our discussion.
Seem that you don't understand what I talk about (and James, I think).
Quoted 9 lines
Of course, I understand the reason why you need that function.
The point is that OSD should not know how a request links bios.
OSD ULD should focus on OSD protocol processing. It should not know the details of requests, bio, etc. OSD ULD should use the proper helper functions of the block layer if OSD ULD uses the block layer service.
What? I'm talking about a BUG, here. What do you suggest that I leak bios until a fix to block is accepted?
And again this is unrelated. Even if I do exactly as you suggest which is use block API minus append_bio I still have that BUG. That must be fixed.
Please stop talking about above code. This is a different thread, unrelated to blk_rq_append_bio() removal.
Quoted 2 lines
OSD ULD should focus on OSD protocol processing.
Which is what it does exactly. You ask me to interfere at OSD level and set policy on memory structures. What I did is let osd_write/read just be a transparent pipe between the upper layer FS code and the low level block layer. Proof of that, any kind of implementation you suggested is that much more code at OSD level? Today it is dead simple.
Boaz
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 30 lines · On Thu, 12 Feb 2009 11:14:02 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 29 lines · FUJITA Tomonori
Quoted 13 lines · On Thu, 12 Feb 2009 10:24:42 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 12 lines · FUJITA Tomonori
On Wed, 11 Feb 2009 19:55:31 +0200
Boaz Harrosh <[email protected]> wrote:
static void_abort_unexecuted_bios(struct request *rq){ struct bio *bio; while ((bio = rq->bio) != NULL) { rq->bio = bio->bi_next; bio_endio(bio, 0); } }Quoted 6 lines
No TOMO! The above is because I can start mapping a request even with simple
map_kernand then fail and do not execute the request. If so, when I do aput_request()with a bio I'm leaking the bio. The above should be put insideput_request()so not to leak bios, but until then It's here.It has nothing to do with our discussion.
Seem that you don't understand what I talk about (and James, I think).
Of course, I understand the reason why you need that function.
The point is that OSD should not know how a request links bios.
Quoted 8 lines
Quoted 5 lines
OSD ULD should focus on OSD protocol processing. It should not know the details of requests, bio, etc. OSD ULD should use the proper helper functions of the block layer if OSD ULD uses the block layer service.
What? I'm talking about a BUG, here. What do you suggest that I leak bios until a fix to block is accepted?
Huh? What I said is something like the above function should live in the block layer. Again, OSD ULD should not know about such.
Quoted 7 lines
And again this is unrelated. Even if I do exactly as you suggest which is use block API minus
append_bioI still have that BUG. That must be fixed.Please stop talking about above code. This is a different thread, unrelated to
blk_rq_append_bio()removal.Quoted 1 line
Quoted 8 lines
Quoted 1 line
OSD ULD should focus on OSD protocol processing.
Which is what it does exactly. You ask me to interfere at OSD level and set policy on memory structures. What I did is let osd_write/read just be
a transparent pipe between the upper layer FS code and the low level block layer. Proof of that, any kind of implementation you suggested is that much more code at OSD level? Today it is dead simple.
Why is it so difficult for you to understand the meaning that OSD ULD should not know how request links bios?
RFC PATCH 1/1 block: Don't let blk_put_request leak BIOs
If a block ULD had allocated a request and mapped some memory into it, using one of blk_rq_map_xxx routines, but then for some reason failed to execute the request through one of the blk_execute_request routines. Then the associated BIO would leak, unless ULD resorts to low-level loops intimate of block internals.
[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we use WARN_ON_ONCE() in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and only WARN_ON if flag was not set.
I'm sending this before any-tests so people can comment on possible pitfalls.
--- block/blk-core.c | 16 ++++++++++++++++ drivers/scsi/osd/osd_initiator.c | 17 +---------------- 2 files changed, 17 insertions(+), 16 deletions(-)
block/blk-core.c +16 -0
@@ -1055,6 +1055,21 @@ void part_round_stats(int cpu, struct hd_struct *part)EXPORT_SYMBOL_GPL (part_round_stats);/* +* If one of the blk_rq_map_xxx() was called but the request was not +* executed by the block layer, then we must release BIOs. Otherwise they +* will leak. +*/ +static void _abort_unexecuted_bios (struct request *req) +{ +struct bio *bio; + +while ((bio = req->bio) !=NULL ) { + req->bio = bio->bi_next; +bio_endio (bio,0 ); + } +} + +/* * queue lock must be held */ void __blk_put_request (struct request_queue *q,struct request *req)
@@ -1066,6 +1081,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)elv_completed_request (q, req); +_abort_unexecuted_bios (req);/* * Request may not have originated from ll_rw_blk. if not, * it didn't come out of our reserved rq pools
drivers/scsi/osd/osd_initiator.c +1 -16
@@ -335,20 +335,6 @@ struct osd_request *osd_start_request(struct osd_dev *dev, gfp_t gfp) }EXPORT_SYMBOL (osd_start_request); -/* -* If osd_finalize_request() was called but the request was not executed through -* the block layer, then we must release BIOs. -*/ -static void _abort_unexecuted_bios (struct request *rq) -{ -struct bio *bio; - -while ((bio = rq->bio) !=NULL ) { - rq->bio = bio->bi_next; -bio_endio (bio,0 ); - } -} - static void _osd_free_seg(struct osd_request *or __unused,struct _osd_req_data_segment *seg) {
@@ -370,11 +356,10 @@ void osd_end_request(struct osd_request *or)if (rq) {if (rq->next_rq) { -_abort_unexecuted_bios (rq->next_rq);blk_put_request (rq->next_rq); + rq->next_rq =NULL ; } -_abort_unexecuted_bios (rq);blk_put_request (rq); }_osd_request_free (or);
Signature · 1 line
1.6.0.1
Re: [PATCH] [RFC] block: Don't let blk_put_request leak BIOs
Quoted 14 lines · Boaz Harrosh
If a block ULD had allocated a request and mapped some memory into it, using one of
blk_rq_map_xxxroutines, but then for some reason failed to execute the request through one of theblk_execute_requestroutines. Then the associated BIO would leak, unless ULD resorts to low-level loops intimate of block internals.[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we useWARN_ON_ONCE()in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and onlyWARN_ONif flag was not set.
Quoted 5 lines
I'm sending this before any-tests so people can comment on possible pitfalls.
Signed-off-by Boaz Harrosh <[email protected]>---
I've booted a Linux PC with lots of sata disks, connected an iscsi target, ran OSD tests. It looks like it's working which means request->bio is set to NULL after it is used in the regular path.
This needs to sit in Linux next and be tested for a long while.
Jens I'll be waiting for your comment and will send a proper patch for the block bits. We will have to time this with James to see when the OSD bits can be submitted after that, then TOMO's patch for un-exporting blk_req_append_bio can be merged. Or maybe it can all go in one patch through scsi?
Thanks
Boaz
Re: [PATCH] [RFC] block: Don't let blk_put_request leak BIOs
Quoted 27 lines · Boaz Harrosh
Quoted 14 lines · Boaz Harrosh
If a block ULD had allocated a request and mapped some memory into it, using one of
blk_rq_map_xxxroutines, but then for some reason failed to execute the request through one of theblk_execute_requestroutines. Then the associated BIO would leak, unless ULD resorts to low-level loops intimate of block internals.[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we useWARN_ON_ONCE()in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and onlyWARN_ONif flag was not set.Quoted 5 lines
I'm sending this before any-tests so people can comment on possible pitfalls.
Signed-off-by Boaz Harrosh <[email protected]>---
I've booted a Linux PC with lots of sata disks, connected an iscsi target, ran OSD tests. It looks like it's working which means
request->biois set to NULL after it is used in the regular path.This needs to sit in Linux next and be tested for a long while.
Quoted 8 lines
Jens I'll be waiting for your comment and will send a proper patch for the block bits. We will have to time this with James to see when the OSD bits can be submitted after that, then TOMO's patch for un-exporting
blk_req_append_biocan be merged. Or maybe it can all go in one patch through scsi?Thanks
Boaz
Spock to soon this patch is shit, I'll look into it
Sorry for the noise
Boaz
RFC PATCH 1/2 block: Don't let blk_put_request leak BIOs
If a block ULD had allocated a request and mapped some memory into it, but then for some reason failed to execute the request through one of the blk_execute_request_xxx routines. Then the associated bio would leak, unless ULD resorts to low-level loops intimate of block internals.
For this to work I have fixed a couple of places in block/ where request->bio != NULL ownership was not honored. And a small cleanup at sg_io() while at it.
[RFC]
This code will also catch situations where LLD failed to complete the request before aborting it. Such situations are a BUG. Should we use WARN_ON_ONCE() in that case. The situation above is possible and can happen normally in memory pressure situations so maybe we should devise a bit-flag that ULD denotes that the request was aborted and only WARN_ON if flag was not set.
For the duration of linux-next I'm leaving the WARN_ON to catch any problems like found above, and possible memory leaks. Before submission a complimentary patch should remove the WARN_ON. (Or this patch can be rebased)
Please comment on possible pitfalls.
--- block/blk-core.c | 17 +++++++++++++++++ block/blk-merge.c | 2 ++ block/scsi_ioctl.c | 21 ++++----------------- 3 files changed, 23 insertions(+), 17 deletions(-)
block/blk-core.c +17 -0
@@ -1055,6 +1055,22 @@ void part_round_stats(int cpu, struct hd_struct *part)EXPORT_SYMBOL_GPL (part_round_stats);/* +* If one of the blk_rq_map_xxx() was called but the request was not +* executed by the block layer, then we must release BIOs. Otherwise they +* will leak. +*/ +static void _abort_unexecuted_bios (struct request *req) +{ +struct bio *bio; + +WARN_ON (req->bio !=NULL ); +while (unlikely ((bio = req->bio) !=NULL )) { + req->bio = bio->bi_next; +bio_endio (bio,0 ); + } +} + +/* * queue lock must be held */ void __blk_put_request (struct request_queue *q,struct request *req)
@@ -1066,6 +1082,7 @@ void __blk_put_request(struct request_queue *q, struct request *req)elv_completed_request (q, req); +_abort_unexecuted_bios (req);/* * Request may not have originated from ll_rw_blk. if not, * it didn't come out of our reserved rq pools
block/blk-merge.c +2 -0
@@ -398,6 +398,8 @@ static int attempt_merge(struct request_queue *q, struct request *req,if (blk_rq_cpu_valid (next)) req->cpu = next->cpu; +/* owner-ship of bio passed from next to req */ + next->bio =NULL ;__blk_put_request (q, next);return 1 ; }
block/scsi_ioctl.c +4 -17
@@ -214,21 +214,10 @@ static int blk_fill_sghdr_rq(struct request_queue *q, struct request *rq,return 0 ; } -/* -* unmap a request that was previously mapped to this sg_io_hdr. handles -* both sg and non-sg sg_io_hdr. -*/ -static int blk_unmap_sghdr_rq (struct request *rq,struct sg_io_hdr *hdr) -{ -blk_rq_unmap_user (rq->bio); -blk_put_request (rq); -return 0 ; -} -static int blk_complete_sghdr_rq (struct request *rq,struct sg_io_hdr *hdr,struct bio *bio) { -int r, ret =0 ; +int ret =0 ;/* * fill in all the output members
@@ -253,12 +242,10 @@ static int blk_complete_sghdr_rq(struct request *rq, struct sg_io_hdr *hdr, ret = -EFAULT ; } - rq->bio = bio; - r =blk_unmap_sghdr_rq (rq, hdr); -if (ret) - r = ret; +blk_rq_unmap_user (bio); +blk_put_request (rq); -return r; +return ret; } static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
Signature · 1 line
1.6.0.1
RFC PATCH 2/2 libosd: Don't let osd abuse block internals, now that it's fixed
blk_put_request will delete any BIOs that where mapped but not executed, so osd_initiator does not have to take care of this situation any more.
This patch is dependent on block patch titled:
[PATCH 1/2] block: Don't let blk_put_request leak BIOs
--- drivers/scsi/osd/osd_initiator.c | 17 +---------------- 1 files changed, 1 insertions(+), 16 deletions(-)
drivers/scsi/osd/osd_initiator.c +1 -16
@@ -335,20 +335,6 @@ struct osd_request *osd_start_request(struct osd_dev *dev, gfp_t gfp) }EXPORT_SYMBOL (osd_start_request); -/* -* If osd_finalize_request() was called but the request was not executed through -* the block layer, then we must release BIOs. -*/ -static void _abort_unexecuted_bios (struct request *rq) -{ -struct bio *bio; - -while ((bio = rq->bio) !=NULL ) { - rq->bio = bio->bi_next; -bio_endio (bio,0 ); - } -} - static void _osd_free_seg(struct osd_request *or __unused,struct _osd_req_data_segment *seg) {
@@ -370,11 +356,10 @@ void osd_end_request(struct osd_request *or)if (rq) {if (rq->next_rq) { -_abort_unexecuted_bios (rq->next_rq);blk_put_request (rq->next_rq); + rq->next_rq =NULL ; } -_abort_unexecuted_bios (rq);blk_put_request (rq); }_osd_request_free (or);
Signature · 1 line
1.6.0.1
James Bottomley
maintainerRe: [PATCH 2/3] block: unexport blk_rq_append_bio
In reply to Boaz HarroshQuoted 59 lines · Boaz Harrosh
Quoted 49 lines · James Bottomley
Quoted 48 lines · FUJITA Tomonori
Quoted 5 lines · On Wed, 11 Feb 2009 17:41:35 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 4 lines · FUJITA Tomonori
Quoted 1 line · On Wed, 11 Feb 2009 17:07:16 +0200 Boaz Harrosh <[email protected]> wrote:
Quoted 1 line
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
Sure it can pass pages to ULD but then how ULD maks a request out of pages?
If you extend
blk_rq_map_kernas James did, your ULD make a request out of passed pages, that is, the block layer properly builds a request with bio(s).What? I don't understand?
blk_rq_map_kerntakes a kernel-pointer and length, how to pass array of page* ?You can call it multiple times. But I guess that you need something new since you need to handle user pages.
Quoted 6 lines
Quoted 4 lines
I think that another possible option is adding a new mapping function that can handle multiple bios for kernel buffers (as Mike extended
blk_rq_map_user).Yes adding a new mapping function can help. Please note I need to add (append) pages same as
bio_add_pc_page()but on request level.I think that what you need already exists.
See how st uses
blk_rq_map_user()in scsi-misc (keywords:rq_map_dataandnull_mapped). Please read it before saying something like 'we need to handle kernel buffer'.Quoted 1 line
And yet we had an entry that did exactly that,
blk_rq_append_bio(), no?Using
blk_rq_append_bioin SCSI is unacceptable.Quoted 9 lines
Quoted 3 lines
Quoted 1 line
And it gets more complicated then that (above multiple times)
I don't think so. If you don't have time to work on it, please let me know. I'll fix OSD ULD.
I have all the time in the world, And yes what James did with
blk_rq_map_kernwill help with appending other osd segments on top of data.If you propose the appropriate new API for block request level I can implement that in block, and also in OSD. What other entry you want to add/change that solve my need?
If you have time, how about making a proposal. ;)
Quoted 8 lines
Quoted 1 line
But as I wrote above, I think that
blk_rq_map_user()works for you.Part of the problem, I think, is that a typical filesystem either works with pages (ext3) or bios (btrfs) and puts them in block at the top. Block then accumulates the pages to bios, elevates the bios into requests and spits those out to SCSI ... this is why we want to eliminate bio handlers from SCSI.
Quoted 3 lines
You are too fast for me, sorry, I did not get that: "... this is why" logical jump. I thought we where getting read of scatter-gather handlers from SCSI. We never had any bio handlers in scsi, did we?
Yes, but we've been trying to cut down on them. In fact with both osd and integrity, the use of bio handlers has been increasing.
Quoted 11 lines
Quoted 6 lines
Your problem is that you phrase your osd fs/pnfs in terms of bios, so then you need to emulate the block bio handlers internally (because you're bypassing block) in your ULD. However, what we're saying is if you speak pages instead, you can utilise the standard block pc handlers, which are designed to speak pages, without reinventing the bio handling infrastructure.
I did not know I was reinventing the bio handling infrastructure, I thought I was using it with all available glory, and exported API. I let different users collect their memory information inside a bio, then in one simple call the bio is elevated to a request and the request is submitted,
block_pcstyle.
Everything in your last sentence is how the block layer handles bios (although it's more sophisticated), so this is the duplication. The problem is that how bios are put together is a block internal thing. Nothing outside of block does that today ... if you construct your own multi-bio requests, we have to remember that drivers/scsi/osd needs updating as well if block ever alters this (say to cope with a new elevator strategy).
Quoted 6 lines
Actually I just started to code using
blk_rq_map_user(), and setting of all
structrq_map_datainformation, and that feels a little like bio code duplication. Because I need a collection (link-list) of them. structrq_map_dataassumes linear memory which I do not have. Also I will need thatblk_rq_map_user()will append bios like the change you proposed toblk_rq_map_kern(), I guess it is doable, only it's lots of more work. (And that awfulblk_rq_unmap_user()which forces me to know if I sent viamap_usrormap_kern...)
blk_rq_map_user does append pages to bios ... that's where I got the changes to blk_rq_map_kern() from.
Quoted 6 lines
I have one question. If bio API is only to be used by block layer internal, why is it exported at all? Seems a bio has nice API for collecting memory information and is used by all kind of block users like filesystems, DM, MD, data integrity and others. I would not mind a:
struct requst *blk_pc_make_request(bio);Of sorts, to make it official, same as
generic_make_request()but forblock_pcusers.
bios are supposed to be used by filesystems ... they represent the input to block. In principle, it seems nicer that PC requests deal with pages as their natural input ... that way we don't have to worry about exporting the block interfaces for bio->request construction.
Quoted 1 line
I will try the
blk_rq_map_user, I'm sure it will not be pretty, though.
Great, thanks ... we can also adjust block interfaces to help.
James
Re: [PATCH 2/3] block: unexport blk_rq_append_bio
Quoted 6 lines · James Bottomley
bios are supposed to be used by filesystems ... they represent the input to block. In principle, it seems nicer that PC requests deal with pages as their natural input ... that way we don't have to worry about exporting the block interfaces for
bio->requestconstruction.
This is my point. I'm a filesystem and I want a natural route from bio to request, only with a paradigm shift as you call it. For a filesystem the most natural and comfortable vehicle is a bio.
So the interface I would like to have is:
struct request *blk_make_request(struct request_queue *, struct bio *, gfp_t);
And let block layer take care of it's internals. Off course when crafting above the generic_make_request will use above API so there is absolutely no duplication of code and block layer is free to change as it feels like, only in a single place.
Quoted 6 lines
Quoted 1 line
I will try the
blk_rq_map_user, I'm sure it will not be pretty, though.Great, thanks ... we can also adjust block interfaces to help.
James
I'm on it. It's not as bad as I thought, and it should work for what I have now. I'm touching fast-path block layer code though, so we'll have to test it for a long while.
I have a gut feeling that I will regret it some day, but we'll cross that bridge when we get there.
Thank you for your help and concern.
Boaz
Quoted 15 lines · FUJITA Tomonori
This patchset removes
scsi_req_map_sg. IOW, this patchset removes 'SCSI-ml does something wrong' comment in the block layer :)/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(structrequest_queue*q, struct request *rq,struct bio *bio);This patchset also removes
scsi_execute_asyncand structscsi_io_context()and unexportbio_add_pc_page().This is against scsi-misc and depends on two patchsets:
Quoted 7 lines
'removing
scsi_execute_asyncin st' the second half patchset:http://marc.info/?l=linux-scsi&m=122900494917385&w=2
'removing
scsi_execute_asyncin osst' the patchset:
You can add my Acked-by to all three patches, nice work Tomo!
Signature · 1 line
Jens Axboe
Quoted 18 lines · On Mon, 15 Dec 2008 08:28:17 +0100 Jens Axboe <[email protected]> wrote:
Quoted 17 lines · FUJITA Tomonori
This patchset removes
scsi_req_map_sg. IOW, this patchset removes 'SCSI-ml does something wrong' comment in the block layer :)/*
* Temporary export, until SCSI gets fixed up.
*/
extern int blk_rq_append_bio(structrequest_queue*q, struct request *rq,struct bio *bio);This patchset also removes
scsi_execute_asyncand structscsi_io_context()and unexportbio_add_pc_page().This is against scsi-misc and depends on two patchsets:
'removing
scsi_execute_asyncin st' the second half patchset:
Quoted 7 lines
Quoted 5 lines
http://marc.info/?l=linux-scsi&m=122900494917385&w=2
'removing
scsi_execute_asyncin osst' the patchset:You can add my Acked-by to all three patches, nice work Tomo!
Thanks,
I also need your ACKs on the #1-3 patches in the following patchset to push the nice work. :)