IO scheduler based IO controller V3
Subsystems (11)
PATCH 14/20 blkio_cgroup patches from Ryo to track async bios.
In reply to Vivek Goyalo blkio_cgroup patches from Ryo to track async bios.
o Fernando is also working on another IO tracking mechanism. We are not particular about any IO tracking mechanism. This patchset can make use of any mechanism which makes it to upstream. For the time being making use of Ryo's posting.
Based on 2.6.30-rc3-git3
--- block/blk-ioc.c | 37 +++--- fs/buffer.c | 2 + fs/direct-io.c | 2 + include/linux/biotrack.h | 97 +++++++++++++ include/linux/cgroup_subsys.h | 6 + include/linux/iocontext.h | 1 + include/linux/memcontrol.h | 6 + include/linux/mmzone.h | 4 +- include/linux/page_cgroup.h | 31 ++++- init/Kconfig | 15 ++ mm/Makefile | 4 +- mm/biotrack.c | 300 +++++++++++++++++++++++++++++++++++++++++ mm/bounce.c | 2 + mm/filemap.c | 2 + mm/memcontrol.c | 6 + mm/memory.c | 5 + mm/page-writeback.c | 2 + mm/page_cgroup.c | 17 ++- mm/swap_state.c | 2 + 19 files changed, 511 insertions(+), 30 deletions(-) create mode 100644 include/linux/biotrack.h create mode 100644 mm/biotrack.c
block/blk-ioc.c +21 -16
@@ -84,27 +84,32 @@ void exit_io_context(void) } } +void init_io_context (struct io_context *ioc) +{ +atomic_set (&ioc->refcount,1 ); +atomic_set (&ioc->nr_tasks,1 ); +spin_lock_init (&ioc->lock); + ioc->ioprio_changed =0 ; + ioc->ioprio =0 ; +#ifdef CONFIG_GROUP_IOSCHED + ioc->cgroup_changed =0 ; +#endif + ioc->last_waited = jiffies;/* doesn't matter... */ + ioc->nr_batch_requests =0 ;/* because this is 0 */ + ioc->aic =NULL ; +INIT_RADIX_TREE (&ioc->radix_root,GFP_ATOMIC | __GFP_HIGH); +INIT_HLIST_HEAD (&ioc->cic_list); + ioc->ioc_data =NULL ; +} + +struct io_context *alloc_io_context (gfp_t gfp_flags,int node) {struct io_context *ret; ret =kmem_cache_alloc_node (iocontext_cachep, gfp_flags, node); -if (ret) { -atomic_set (&ret->refcount,1 ); -atomic_set (&ret->nr_tasks,1 ); -spin_lock_init (&ret->lock); - ret->ioprio_changed =0 ; - ret->ioprio =0 ; -#ifdef CONFIG_GROUP_IOSCHED - ret->cgroup_changed =0 ; -#endif - ret->last_waited = jiffies;/* doesn't matter... */ - ret->nr_batch_requests =0 ;/* because this is 0 */ - ret->aic =NULL ; -INIT_RADIX_TREE (&ret->radix_root,GFP_ATOMIC | __GFP_HIGH); -INIT_HLIST_HEAD (&ret->cic_list); - ret->ioc_data =NULL ; - } +if (ret) +init_io_context (ret);return ret; }
fs/buffer.c +2 -0
@@ -36,6 +36,7 @@#include <linux/buffer_head.h> #include <linux/task_io_accounting_ops.h> #include <linux/bio.h> +#include <linux/biotrack.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/bitops.h>
@@ -668,6 +669,7 @@ static void __set_page_dirty(struct page *page,if (page->mapping) {/* Race with truncate? */ WARN_ON_ONCE (warn && !PageUptodate (page));account_page_dirtied (page, mapping); +blkio_cgroup_reset_owner_pagedirty (page, current->mm);radix_tree_tag_set (&mapping->page_tree,page_index (page),PAGECACHE_TAG_DIRTY ); }
fs/direct-io.c +2 -0
@@ -33,6 +33,7 @@#include <linux/err.h> #include <linux/blkdev.h> #include <linux/buffer_head.h> +#include <linux/biotrack.h> #include <linux/rwsem.h> #include <linux/uio.h> #include <asm/atomic.h>
@@ -797,6 +798,7 @@ static int do_direct_IO(struct dio *dio) ret =PTR_ERR (page); goto out; } +blkio_cgroup_reset_owner (page, current->mm);while (block_in_page < blocks_per_page) {unsigned offset_in_page = block_in_page << blkbits;
include/linux/biotrack.h new file +97 -0
97 lines not on this page · raw message
include/linux/cgroup_subsys.h +6 -0
@@ -43,6 +43,12 @@ SUBSYS(mem_cgroup)/* */ +#ifdef CONFIG_CGROUP_BLKIO +SUBSYS(blkio_cgroup) +#endif + +/* */ +#ifdef CONFIG_CGROUP_DEVICE SUBSYS(devices)#endif
include/linux/iocontext.h +1 -0
@@ -105,6 +105,7 @@ int put_io_context(struct io_context *ioc);void exit_io_context (void );struct io_context *get_io_context (gfp_t gfp_flags,int node);struct io_context *alloc_io_context (gfp_t gfp_flags,int node); +void init_io_context (struct io_context *ioc);void copy_io_context (struct io_context **pdst,struct io_context **psrc);#else static inline void exit_io_context (void )
include/linux/memcontrol.h +6 -0
@@ -37,6 +37,8 @@ struct mm_struct;* (Of course, if memcg does memory allocation in future, GFP_KERNEL is sane.) */ +extern void __init_mem_page_cgroup (struct page_cgroup *pc); +extern int mem_cgroup_newpage_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask);/* for swap handling */
@@ -120,6 +122,10 @@ extern bool mem_cgroup_oom_called(struct task_struct *task);#else /* CONFIG_CGROUP_MEM_RES_CTLR */ struct mem_cgroup ; +static inline void __init_mem_page_cgroup (struct page_cgroup *pc) +{ +} +static inline int mem_cgroup_newpage_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) {
include/linux/mmzone.h +2 -2
@@ -607,7 +607,7 @@ typedef struct pglist_data {int nr_zones;#ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */ struct page *node_mem_map; -#ifdef CONFIG_CGROUP_MEM_RES_CTLR +#ifdef CONFIG_CGROUP_PAGE struct page_cgroup *node_page_cgroup;#endif #endif
@@ -958,7 +958,7 @@ struct mem_section {/* See declaration of similar field in struct zone */ unsigned long *pageblock_flags; -#ifdef CONFIG_CGROUP_MEM_RES_CTLR +#ifdef CONFIG_CGROUP_PAGE /* * If !SPARSEMEM, pgdat doesn't have page_cgroup pointer. We use * section. (see memcontrol.h/page_cgroup.h about this.)
include/linux/page_cgroup.h +28 -3
@@ -1,7 +1,7 @@#ifndef __LINUX_PAGE_CGROUP_H#define __LINUX_PAGE_CGROUP_H -#ifdef CONFIG_CGROUP_MEM_RES_CTLR +#ifdef CONFIG_CGROUP_PAGE #include <linux/bit_spinlock.h> /* * Page Cgroup can be considered as an extended mem_map.
@@ -12,9 +12,11 @@*/ struct page_cgroup {unsigned long flags; -struct mem_cgroup *mem_cgroup;struct page *page; +#ifdef CONFIG_CGROUP_MEM_RES_CTLR +struct mem_cgroup *mem_cgroup;struct list_head lru;/* per cgroup LRU list */ +#endif };void __meminit pgdat_page_cgroup_init (struct pglist_data *pgdat);
@@ -71,7 +73,7 @@ static inline void unlock_page_cgroup(struct page_cgroup *pc) bit_spin_unlock(PCG_LOCK, &pc->flags); } -#else /* CONFIG_CGROUP_MEM_RES_CTLR */ +#else /* CONFIG_CGROUP_PAGE */ struct page_cgroup ;static inline void __meminit pgdat_page_cgroup_init (struct pglist_data *pgdat)
@@ -122,4 +124,27 @@ static inline void swap_cgroup_swapoff(int type) }#endif + +#ifdef CONFIG_CGROUP_BLKIO +/* +* use lower 16 bits for flags and reserve the rest for the page tracking id +*/ +#define PCG_TRACKING_ID_SHIFT (16) +#define PCG_TRACKING_ID_BITS \ + (8 * sizeof(unsigned long) - PCG_TRACKING_ID_SHIFT) + +/* NOTE: must be called with page_cgroup() held */ +static inline unsigned long page_cgroup_get_id (struct page_cgroup *pc) +{ +return pc->flags >>PCG_TRACKING_ID_SHIFT ; +} + +/* NOTE: must be called with page_cgroup() held */ +static inline void page_cgroup_set_id (struct page_cgroup *pc,unsigned long id) +{ +WARN_ON (id >= (1UL <<PCG_TRACKING_ID_BITS )); + pc->flags &= (1UL <<PCG_TRACKING_ID_SHIFT ) -1 ; + pc->flags |= (unsigned long )(id <<PCG_TRACKING_ID_SHIFT ); +} +#endif #endif
init/Kconfig +15 -0
@@ -616,6 +616,21 @@ config GROUP_IOSCHED endif # CGROUPS +config CGROUP_BLKIO + bool "Block I/O cgroup subsystem" + depends on CGROUPS && BLOCK + select MM_OWNER + help + Provides a Resource Controller which enables to track the onwner + of every Block I/O requests. + The information this subsystem provides can be used from any + kind of module such as dm-ioband device mapper modules or + the cfq-scheduler. + +config CGROUP_PAGE + def_bool y + depends on CGROUP_MEM_RES_CTLR || CGROUP_BLKIO + config MM_OWNER bool
mm/Makefile +3 -1
@@ -37,4 +37,6 @@ else obj-$(CONFIG_SMP) += allocpercpu.o endif obj-$(CONFIG_QUICKLIST) += quicklist.o -obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o page_cgroup.o +obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o +obj-$(CONFIG_CGROUP_PAGE) += page_cgroup.o +obj-$(CONFIG_CGROUP_BLKIO) += biotrack.o
mm/biotrack.c new file +300 -0
300 lines not on this page · raw message
mm/bounce.c +2 -0
@@ -14,6 +14,7 @@#include <linux/hash.h> #include <linux/highmem.h> #include <linux/blktrace_api.h> +#include <linux/biotrack.h> #include <trace/block.h> #include <asm/tlbflush.h>
@@ -212,6 +213,7 @@ static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig, to->bv_len = from->bv_len; to->bv_offset = from->bv_offset;inc_zone_page_state (to->bv_page,NR_BOUNCE ); +blkio_cgroup_copy_owner (to->bv_page, page);if (rw ==WRITE ) {char *vto, *vfrom;
mm/filemap.c +2 -0
@@ -33,6 +33,7 @@#include <linux/cpuset.h> #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ #include <linux/memcontrol.h> +#include <linux/biotrack.h> #include <linux/mm_inline.h> /* for page_is_file_cache() */ #include "internal.h"
@@ -464,6 +465,7 @@ int add_to_page_cache_locked(struct page *page, struct address_space *mapping, gfp_mask & GFP_RECLAIM_MASK);if (error) goto out; +blkio_cgroup_set_owner (page, current->mm); error =radix_tree_preload (gfp_mask & ~__GFP_HIGHMEM);if (error ==0 ) {
mm/memcontrol.c +6 -0
@@ -128,6 +128,12 @@ struct mem_cgroup_lru_info {struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES ]; }; +void __meminit __init_mem_page_cgroup (struct page_cgroup *pc) +{ + pc->mem_cgroup =NULL ; +INIT_LIST_HEAD (&pc->lru); +} +/* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide
mm/memory.c +5 -0
@@ -51,6 +51,7 @@#include <linux/init.h> #include <linux/writeback.h> #include <linux/memcontrol.h> +#include <linux/biotrack.h> #include <linux/mmu_notifier.h> #include <linux/kallsyms.h> #include <linux/swapops.h>
@@ -2064,6 +2065,7 @@ gotten:*/ ptep_clear_flush_notify (vma, address, page_table);page_add_new_anon_rmap (new_page, vma, address); +blkio_cgroup_set_owner (new_page, mm);set_pte_at (mm, address, page_table, entry);update_mmu_cache (vma, address, entry);if (old_page) {
@@ -2529,6 +2531,7 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,flush_icache_page (vma, page);set_pte_at (mm, address, page_table, pte);page_add_anon_rmap (page, vma, address); +blkio_cgroup_reset_owner (page, mm);/* It's better to call commit-charge after rmap is established */ mem_cgroup_commit_charge_swapin (page, ptr);
@@ -2593,6 +2596,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, goto release;inc_mm_counter (mm, anon_rss);page_add_new_anon_rmap (page, vma, address); +blkio_cgroup_set_owner (page, mm);set_pte_at (mm, address, page_table, entry);/* No need to invalidate - it was non-present before */
@@ -2740,6 +2744,7 @@ static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,if (anon) {inc_mm_counter (mm, anon_rss);page_add_new_anon_rmap (page, vma, address); +blkio_cgroup_set_owner (page, mm); }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (page);
mm/page-writeback.c +2 -0
@@ -23,6 +23,7 @@#include <linux/init.h> #include <linux/backing-dev.h> #include <linux/task_io_accounting_ops.h> +#include <linux/biotrack.h> #include <linux/blkdev.h> #include <linux/mpage.h> #include <linux/rmap.h>
@@ -1243,6 +1244,7 @@ int __set_page_dirty_nobuffers(struct page *page)BUG_ON (mapping2 != mapping);WARN_ON_ONCE (!PagePrivate (page) && !PageUptodate (page));account_page_dirtied (page, mapping); +blkio_cgroup_reset_owner_pagedirty (page, current->mm);radix_tree_tag_set (&mapping->page_tree,page_index (page),PAGECACHE_TAG_DIRTY ); }
mm/page_cgroup.c +9 -8
@@ -9,14 +9,15 @@#include <linux/vmalloc.h> #include <linux/cgroup.h> #include <linux/swapops.h> +#include <linux/biotrack.h> static void __meminit __init_page_cgroup (struct page_cgroup *pc,unsigned long pfn) { pc->flags =0 ; - pc->mem_cgroup =NULL ; pc->page =pfn_to_page (pfn); -INIT_LIST_HEAD (&pc->lru); +__init_mem_page_cgroup (pc); +__init_blkio_page_cgroup (pc); }static unsigned long total_usage;
@@ -74,7 +75,7 @@ void __init page_cgroup_init(void)int nid, fail; -if (mem_cgroup_disabled ()) +if (mem_cgroup_disabled () &&blkio_cgroup_disabled ())return ;for_each_online_node (nid) {
@@ -83,12 +84,12 @@ void __init page_cgroup_init(void) goto fail; }printk (KERN_INFO "allocated %ld bytes of page_cgroup\n" , total_usage); -printk (KERN_INFO "please try cgroup_disable=memory option if you" +printk (KERN_INFO "please try cgroup_disable=memory,blkio option if you" " don't want\n" );return ; fail:printk (KERN_CRIT "allocation of page_cgroup was failed.\n" ); -printk (KERN_CRIT "please try cgroup_disable=memory boot option\n" ); +printk (KERN_CRIT "please try cgroup_disable=memory,blkio boot options\n" );panic ("Out of memory" ); }
@@ -248,7 +249,7 @@ void __init page_cgroup_init(void)unsigned long pfn;int fail =0 ; -if (mem_cgroup_disabled ()) +if (mem_cgroup_disabled () &&blkio_cgroup_disabled ())return ;for (pfn =0 ; !fail && pfn < max_pfn; pfn +=PAGES_PER_SECTION ) {
@@ -263,8 +264,8 @@ void __init page_cgroup_init(void)hotplug_memory_notifier (page_cgroup_callback,0 ); }printk (KERN_INFO "allocated %ld bytes of page_cgroup\n" , total_usage); -printk (KERN_INFO "please try cgroup_disable=memory option if you don't" -" want\n" ); +printk (KERN_INFO "please try cgroup_disable=memory,blkio option" +" if you don't want\n" ); }void __meminit pgdat_page_cgroup_init (struct pglist_data *pgdat)
mm/swap_state.c +2 -0
@@ -18,6 +18,7 @@#include <linux/pagevec.h> #include <linux/migrate.h> #include <linux/page_cgroup.h> +#include <linux/biotrack.h> #include <asm/pgtable.h>
@@ -308,6 +309,7 @@ struct page *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask,*/ __set_page_locked (new_page);SetPageSwapBacked (new_page); +blkio_cgroup_set_owner (new_page, current->mm); err =add_to_swap_cache (new_page, entry, gfp_mask &GFP_KERNEL );if (likely (!err)) {/*
Signature · 1 line
1.6.0.1
PATCH 15/20 io-controller: map async requests to appropriate cgroup
In reply to Vivek Goyalo So far we were assuming that a bio/rq belongs to the task who is submitting it. It did not hold good in case of async writes. This patch makes use of blkio_cgroup pataches to attribute the aysnc writes to right group instead of task submitting the bio.
o For sync requests, we continue to assume that io belongs to the task submitting it. Only in case of async requests, we make use of io tracking patches to track the owner cgroup.
o So far cfq always caches the async queue pointer. With async requests now not necessarily being tied to submitting task io context, caching the pointer will not help for async queues. This patch introduces a new config option CONFIG_TRACK_ASYNC_CONTEXT. If this option is not set, cfq retains old behavior where async queue pointer is cached in task context. If it is not set, async queue pointer is not cached and we take help of bio tracking patches to determine group bio belongs to and then map it to async queue of that group.
--- block/Kconfig.iosched | 16 +++++ block/as-iosched.c | 2 +- block/blk-core.c | 7 +- block/cfq-iosched.c | 152 ++++++++++++++++++++++++++++++++++++---------- block/deadline-iosched.c | 2 +- block/elevator-fq.c | 101 ++++++++++++++++++++++++++----- block/elevator-fq.h | 23 ++++++- block/elevator.c | 15 +++-- include/linux/elevator.h | 21 ++++++- 9 files changed, 272 insertions(+), 67 deletions(-)
block/Kconfig.iosched +16 -0
@@ -124,6 +124,22 @@ config DEFAULT_IOSCHED default "cfq" if DEFAULT_CFQ default "noop" if DEFAULT_NOOP +config TRACK_ASYNC_CONTEXT + bool "Determine async request context from bio" + depends on GROUP_IOSCHED + select CGROUP_BLKIO + default n + ---help--- + Normally async request is attributed to the task submitting the + request. With group ioscheduling, for accurate accounting of + async writes, one needs to map the request to original task/cgroup + which originated the request and not the submitter of the request. + + Currently there are generic io tracking patches to provide facility + to map bio to original owner. If this option is set, for async + request, original owner of the bio is decided by using io tracking + patches otherwise we continue to attribute the request to the + submitting thread. endmenu endif
block/as-iosched.c +1 -1
@@ -1412,7 +1412,7 @@ as_merge(struct request_queue *q, struct request **req, struct bio *bio) {sector_t rb_key = bio->bi_sector +bio_sectors (bio);struct request *__rq; -struct as_queue *asq =elv_get_sched_queue_current (q); +struct as_queue *asq =elv_get_sched_queue_bio (q, bio);if (!asq)return ELEVATOR_NO_MERGE ;
block/blk-core.c +4 -3
@@ -643,7 +643,8 @@ static inline void blk_free_request(struct request_queue *q, struct request *rq) }static struct request * -blk_alloc_request (struct request_queue *q,int flags,int priv,gfp_t gfp_mask) +blk_alloc_request (struct request_queue *q,struct bio *bio,int flags,int priv, +gfp_t gfp_mask) {struct request *rq =mempool_alloc (q->rq.rq_pool, gfp_mask);
@@ -655,7 +656,7 @@ blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask) rq->cmd_flags = flags |REQ_ALLOCED ;if (priv) { -if (unlikely (elv_set_request (q, rq, gfp_mask))) { +if (unlikely (elv_set_request (q, rq, bio, gfp_mask))) {mempool_free (rq, q->rq.rq_pool);return NULL ; }
@@ -796,7 +797,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags, rw_flags |=REQ_IO_STAT ;spin_unlock_irq (q->queue_lock); - rq =blk_alloc_request (q, rw_flags, priv, gfp_mask); + rq =blk_alloc_request (q, bio, rw_flags, priv, gfp_mask);if (unlikely (!rq)) {/* * Allocation failed presumably due to memory. Undo anything
block/cfq-iosched.c +120 -32
254 lines not on this page · raw message
block/deadline-iosched.c +1 -1
@@ -133,7 +133,7 @@ deadline_merge(struct request_queue *q, struct request **req, struct bio *bio)int ret;struct deadline_queue *dq; - dq =elv_get_sched_queue_current (q); + dq =elv_get_sched_queue_bio (q, bio);if (!dq)return ELEVATOR_NO_MERGE ;
block/elevator-fq.c +85 -16
184 lines not on this page · raw message
block/elevator-fq.h +19 -4
@@ -516,10 +516,12 @@ static inline bfq_weight_t iog_weight(struct io_group *iog) }extern int elv_fq_set_request_ioq (struct request_queue *q,struct request *rq, -gfp_t gfp_mask); +struct bio *bio,gfp_t gfp_mask);extern void elv_fq_unset_request_ioq (struct request_queue *q,struct request *rq);extern struct io_queue *elv_lookup_ioq_current (struct request_queue *q); +extern struct io_queue *elv_lookup_ioq_bio (struct request_queue *q, +struct bio *bio);/* Returns single ioq associated with the io group. */ static inline struct io_queue *io_group_ioq (struct io_group *iog)
@@ -574,7 +576,7 @@ static inline void io_group_set_ioq(struct io_group *iog, struct io_queue *ioq) }static inline int elv_fq_set_request_ioq (struct request_queue *q, -struct request *rq,gfp_t gfp_mask) +struct request *rq,struct bio *bio,gfp_t gfp_mask) {return 0 ; }
@@ -589,6 +591,12 @@ static inline struct io_queue *elv_lookup_ioq_current(struct request_queue *q) return NULL; } +static inline struct io_queue *elv_lookup_ioq_bio (struct request_queue *q, +struct bio *bio) +{ +return NULL ; +} +#endif /* GROUP_IOSCHED */ /* Functions used by blksysfs.c */
@@ -643,7 +651,8 @@ extern void *io_group_async_queue_prio(struct io_group *iog, int ioprio_class, int ioprio);extern void io_group_set_async_queue (struct io_group *iog,int ioprio_class,int ioprio,struct io_queue *ioq); -extern struct io_group *io_get_io_group (struct request_queue *q,int create); +extern struct io_group *io_get_io_group (struct request_queue *q, +struct bio *bio,int create);extern int elv_nr_busy_ioq (struct elevator_queue *e);extern int elv_nr_busy_rt_ioq (struct elevator_queue *e);extern struct io_queue *elv_alloc_ioq (struct request_queue *q,gfp_t gfp_mask);
@@ -703,7 +712,7 @@ static inline int io_group_allow_merge(struct request *rq, struct bio *bio)return 1 ; }static inline int elv_fq_set_request_ioq (struct request_queue *q, -struct request *rq,gfp_t gfp_mask) +struct request *rq,struct bio *bio,gfp_t gfp_mask) {return 0 ; }
@@ -718,5 +727,11 @@ static inline struct io_queue *elv_lookup_ioq_current(struct request_queue *q)return NULL ; } +static inline struct io_queue *elv_lookup_ioq_bio (struct request_queue *q, +struct bio *bio) +{ +return NULL ; +} +#endif /* CONFIG_ELV_FAIR_QUEUING */ #endif /* _BFQ_SCHED_H */
block/elevator.c +8 -7
@@ -967,7 +967,8 @@ struct request *elv_former_request(struct request_queue *q, struct request *rq)return NULL ; } -int elv_set_request (struct request_queue *q,struct request *rq,gfp_t gfp_mask) +int elv_set_request (struct request_queue *q,struct request *rq, +struct bio *bio,gfp_t gfp_mask) {struct elevator_queue *e = q->elevator;
@@ -976,10 +977,10 @@ int elv_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)* ioq per io group */ if (elv_iosched_single_ioq (e)) -return elv_fq_set_request_ioq (q, rq, gfp_mask); +return elv_fq_set_request_ioq (q, rq, bio, gfp_mask);if (e->ops->elevator_set_req_fn) -return e->ops->elevator_set_req_fn (q, rq, gfp_mask); +return e->ops->elevator_set_req_fn (q, rq, bio, gfp_mask); rq->elevator_private =NULL ;return 0 ;
@@ -1368,19 +1369,19 @@ void *elv_select_sched_queue(struct request_queue *q, int force)EXPORT_SYMBOL (elv_select_sched_queue);/* -* Get the io scheduler queue pointer for current task. +* Get the io scheduler queue pointer for the group bio belongs to. * * If fair queuing is enabled, determine the io group of task and retrieve * the ioq pointer from that. This is used by only single queue ioschedulers * for retrieving the queue associated with the group to decide whether the * new bio can do a front merge or not. */ -void *elv_get_sched_queue_current (struct request_queue *q) +void *elv_get_sched_queue_bio (struct request_queue *q,struct bio *bio) {/* Fair queuing is not enabled. There is only one queue. */ if (!elv_iosched_fair_queuing_enabled (q->elevator))return q->elevator->sched_queue; -return ioq_sched_queue (elv_lookup_ioq_current (q)); +return ioq_sched_queue (elv_lookup_ioq_bio (q, bio)); } -EXPORT_SYMBOL (elv_get_sched_queue_current); +EXPORT_SYMBOL (elv_get_sched_queue_bio);
include/linux/elevator.h +18 -3
@@ -23,7 +23,7 @@ typedef struct request *(elevator_request_list_fn) (struct request_queue *, strutypedef void (elevator_completed_req_fn ) (struct request_queue *,struct request *);typedef int (elevator_may_queue_fn ) (struct request_queue *,int ); -typedef int (elevator_set_req_fn ) (struct request_queue *,struct request *,gfp_t ); +typedef int (elevator_set_req_fn ) (struct request_queue *,struct request *,struct bio *bio,gfp_t );typedef void (elevator_put_req_fn ) (struct request *);typedef void (elevator_activate_req_fn ) (struct request_queue *,struct request *);typedef void (elevator_deactivate_req_fn ) (struct request_queue *,struct request *);
@@ -150,7 +150,8 @@ extern void elv_unregister_queue(struct request_queue *q);extern int elv_may_queue (struct request_queue *,int );extern void elv_abort_queue (struct request_queue *);extern void elv_completed_request (struct request_queue *,struct request *); -extern int elv_set_request (struct request_queue *,struct request *,gfp_t ); +extern int elv_set_request (struct request_queue *,struct request *, +struct bio *bio,gfp_t );extern void elv_put_request (struct request_queue *,struct request *);extern void elv_drain_elevator (struct request_queue *);
@@ -293,6 +294,20 @@ static inline int elv_gen_idling_enabled(struct elevator_queue *e)#endif /* ELV_IOSCHED_FAIR_QUEUING */ extern void *elv_get_sched_queue (struct request_queue *q,struct request *rq);extern void *elv_select_sched_queue (struct request_queue *q,int force); -extern void *elv_get_sched_queue_current (struct request_queue *q); +extern void *elv_get_sched_queue_bio (struct request_queue *q,struct bio *bio); + +/* +* This is equivalent of rq_is_sync()/cfq_bio_sync() function where we +* determine whether an rq/bio is sync or not. There are cases like during +* merging and during * request allocation, where we don't have rq but bio +* and needs to find out * if this bio will be considered as sync or async by +* elevator/iosched. This function is useful in such cases. +*/ +static inline int elv_bio_sync (struct bio *bio) +{ +if ((bio_data_dir (bio) ==READ ) ||bio_sync (bio)) +return 1 ; +return 0 ; +}#endif /* CONFIG_BLOCK */ #endif
Signature · 1 line
1.6.0.1
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Hi Vivek,
Quoted 9 lines
block/elevator-fq.c +9
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio);
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 10 lines
block/elevator-fq.c +10
-0+ +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else + cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}
There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use get_blkio_cgroup_id() to determine cgroup.
Thanks,
Ryo Tsuruta
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else
Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
Thanks
Vivek
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 46 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT + if (elv_bio_sync(bio)) { + /* sync io. Determine cgroup from submitting task context. */ + cgroup = task_cgroup(current, io_subsys_id); + return cgroup; + } + + /* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id = get_blkio_cgroup_id(bio); + + if (!bio_cgroup_id) + return NULL; + + cgroup = blkio_cgroup_lookup(bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
+ cgroup =
task_cgroup(current, io_subsys_id);block/elevator-fq.c +3
-0+#endif + return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
From what I remember, Fernando pointed out the same problem in LFS'09 (device mapper drivers submitting sync bio) for his motivation to track io_context in bio. Fernando, what is the state the patch you were working on?
Quoted 4 lines
Thanks
Vivek
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
In reply to Vivek GoyalHi Vivek,
Quoted 28 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.
Quoted 8 lines
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
In the case of READ, dm-raid1 submits read IOs in differenct context under some conditions. dm-ioband also does it.
Quoted 9 lines
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
There is a patchset which makes every bio points the iocontext of the process which is originally generated an IO request.
Date Tue, 22 Apr 2008 22:51:31 +0900 (JST) Subject [RFC][PATCH 1/10] I/O context inheritance From Hirokazu Takahashi <>
http://lkml.org/lkml/2008/4/22/195
Thanks,
Ryo Tsuruta
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 58 lines · Ryo Tsuruta
Hi Vivek,
Quoted 28 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.Quoted 8 lines
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
In the case of READ, dm-raid1 submits read IOs in differenct context under some conditions. dm-ioband also does it.
Quoted 9 lines
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
Quoted 7 lines
There is a patchset which makes every bio points the iocontext of the process which is originally generated an IO request.
Date Tue, 22 Apr 2008 22:51:31 +0900 (JST) Subject [RFC][PATCH 1/10] I/O context inheritance From Hirokazu Takahashi <>
http://lkml.org/lkml/2008/4/22/195
Ok, Thanks. This is good. So once above patches make to upstream, I will just forward port my patches to make use of this infrastructure.
Thanks
Vivek
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 62 lines · Vivek Goyal
Quoted 61 lines · Ryo Tsuruta
Hi Vivek,
Quoted 28 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.Quoted 8 lines
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
In the case of READ, dm-raid1 submits read IOs in differenct context under some conditions. dm-ioband also does it.
Quoted 9 lines
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
There is a patchset which makes every bio points the iocontext of the process which is originally generated an IO request.
Quoted 7 lines
Quoted 4 lines
Date Tue, 22 Apr 2008 22:51:31 +0900 (JST) Subject [RFC][PATCH 1/10] I/O context inheritance From Hirokazu Takahashi <>
http://lkml.org/lkml/2008/4/22/195Ok, Thanks. This is good. So once above patches make to upstream, I will just forward port my patches to make use of this infrastructure.
If there are other people who also need this patchset, I'll consider to port it to the latest kernel. I would like to hear opinions from other people. I think it is useful for CFQ users.
Thanks,
Ryo Tsuruta
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 62 lines · Vivek Goyal
Quoted 61 lines · Ryo Tsuruta
Hi Vivek,
Quoted 28 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.Quoted 8 lines
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
In the case of READ, dm-raid1 submits read IOs in differenct context under some conditions. dm-ioband also does it.
Quoted 9 lines
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
There is a patchset which makes every bio points the iocontext of the process which is originally generated an IO request.
Quoted 7 lines
Quoted 4 lines
Date Tue, 22 Apr 2008 22:51:31 +0900 (JST) Subject [RFC][PATCH 1/10] I/O context inheritance From Hirokazu Takahashi <>
http://lkml.org/lkml/2008/4/22/195Ok, Thanks. This is good. So once above patches make to upstream, I will just forward port my patches to make use of this infrastructure.
If there are other people who also need this patchset, I'll consider to port it to the latest kernel. I would like to hear opinions from other people. I think it is useful for CFQ users.
Thanks,
Ryo Tsuruta
A message that is not in the archive (sent off-list or not yet received): 20090529.121737.189708024.ryov-jCdQPDEk3idL9jVzuh4AOg@public.gmane.org
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 58 lines · Ryo Tsuruta
Hi Vivek,
Quoted 28 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.Quoted 8 lines
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
In the case of READ, dm-raid1 submits read IOs in differenct context under some conditions. dm-ioband also does it.
Quoted 9 lines
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
Quoted 7 lines
There is a patchset which makes every bio points the iocontext of the process which is originally generated an IO request.
Date Tue, 22 Apr 2008 22:51:31 +0900 (JST) Subject [RFC][PATCH 1/10] I/O context inheritance From Hirokazu Takahashi <>
http://lkml.org/lkml/2008/4/22/195
Ok, Thanks. This is good. So once above patches make to upstream, I will just forward port my patches to make use of this infrastructure.
Thanks
Vivek
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Hi Vivek,
Quoted 28 lines · Vivek Goyal
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.
Quoted 8 lines
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
In the case of READ, dm-raid1 submits read IOs in differenct context under some conditions. dm-ioband also does it.
Quoted 9 lines
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
Quoted 5 lines
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
There is a patchset which makes every bio points the iocontext of the process which is originally generated an IO request.
Date Tue, 22 Apr 2008 22:51:31 +0900 (JST) Subject [RFC][PATCH 1/10] I/O context inheritance From Hirokazu Takahashi <>
http://lkml.org/lkml/2008/4/22/195
Thanks,
Ryo Tsuruta
A message that is not in the archive (sent off-list or not yet received): 20090528.182740.193697101.ryov-jCdQPDEk3idL9jVzuh4AOg@public.gmane.org
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Quoted 17 lines · Ryo Tsuruta
Hi Vivek,
Quoted 15 lines
block/elevator-fq.c +15
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio); + +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else
Quoted 9 lines
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 4 lines
block/elevator-fq.c +4
-0+ cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use
get_blkio_cgroup_id()to determine cgroup.
Hi Ryo,
Ok. Can you give some examples of drivers which are submitting reads in different context al-together. You mentioned in the past that dm-crypt looks like the one. How does current CFQ takes care of that. So if a BE prio 7 or an RT prio 0, task is submitting a READ, CFQ will not know it and it will put that READ in the queue of the READ submitting device mapper thread (may be BE prio 3 or 4)?
Always determining the cgroup from bio, will make things slower at the same time complicated from the CFQ point of view. Right now cfq creates and caches the queue pointer in the io context of the bio submitting task and assumes sync requests are coming from that task/io context. Currently there can only be one sync queue associated with one context. So if a single thread is submitting reads (may be a worker thread) on behalf of other processes, then we loose the io context information. In fact currently we don't even carry ioprio and io class information in bio.
So looks like we need to carry task io context information also in bio to be able to associate the bio to right queue at CFQ level. This makes it bit more complicated. For the time being I will keep it in my TODO list and handle it once other more severe problems have been taken care of.
Thanks
Vivek
A message that is not in the archive (sent off-list or not yet received): 1243377729-2176-16-git-send-email-vgoyal-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org
Re: [PATCH 15/20] io-controller: map async requests to appropriate cgroup
Hi Vivek,
Quoted 9 lines
block/elevator-fq.c +9
-0+#ifdef CONFIG_TRACK_ASYNC_CONTEXT +if (elv_bio_sync (bio)) { +/* sync io. Determine cgroup from submitting task context. */ + cgroup =task_cgroup (current, io_subsys_id); +return cgroup; + } + +/* Async io. Determine cgroup from with cgroup id stored in page */ + bio_cgroup_id =get_blkio_cgroup_id (bio);
block/elevator-fq.c struct io_group *io_find_alloc_group(struct request_queue *q, · line 1342
Quoted 10 lines
block/elevator-fq.c +10
-0+ +if (!bio_cgroup_id) +return NULL ; + + cgroup =blkio_cgroup_lookup (bio_cgroup_id); +#else + cgroup =task_cgroup (current, io_subsys_id); +#endif +return cgroup; +}
There is a case where a kernel thread (such as device-mapper drivers) submits a sync IO instead of a task which originates the IO. I think you should always use get_blkio_cgroup_id() to determine cgroup.
Thanks,
Ryo Tsuruta
PATCH 16/20 io-controller: IO group refcounting support
In reply to Vivek Goyalo In the original BFQ patch once a cgroup is being deleted, it will clean up the associated io groups immediately and if there are any active io queues with that group, these will be moved to root group. This movement of queues is not good from fairness perspective as one can then create a cgroup, dump lots of IO and then delete the cgroup and then potentially get higher share. Apart from there are more issues hence it was felt that we need a io group refcounting mechanism also so that io group can be reclaimed asynchronously.
o This is V2 of the io group refcounting patch as implemented by Nauman and Divyesh. Currently keeping this patch as a separate one. Once it stablizes, will merge it with other patches instead of being a seprate one.
o Reference counting for io_group solves many problems, most of which occured when we tried to delete the cgroup. Earlier, ioqs were being moved out of cgroup to root cgroup. That is problematic in many ways, First, the pending requests in queues might get unfair service, and will also cause unfairness for other cgroups at the root level. This problem can become signficant if cgroups are created and destroyed relatively frequently. Second, moving queues to root cgroups was complicated and was causing many BUG_ON's to trigger. Third, there is a single io queue in AS, Deadline and Noop within a cgroup; and it does not make sense to move it to the root cgroup. The same is true of async queues.
o Requests already keep a reference on ioq, so queues keep a reference on cgroup. For async queues in CFQ, and single ioq in other schedulers, io_group also keeps are reference on io_queue. This reference on ioq is dropped when the queue is released (elv_release_ioq). So the queue can be freed.
When a queue is released, it puts the reference to io_group and the io_group is released after all the queues are released. Child groups also take reference on parent groups, and release it when they are destroyed.
o Also we no longer need to maintain a seprate linked list of idle entities, which was maintained only to help release the ioq references during elevator switch. The code for releasing io_groups is reused for elevator switch, resulting in simpler and tight code.
o Reads of iocg->group_data are not always iocg->lock; so all the operations on that list are still protected by RCU. All modifications to iocg->group_data should always done under iocg->lock.
Whenever iocg->lock and queue_lock can both be held, queue_lock should be held first. This avoids all deadlocks. In order to avoid race between cgroup deletion and elevator switch the following algorithm is used:
- Cgroup deletion path holds iocg->lock and removes iog entry to iocg->group_data list. Then it drops iocg->lock, holds queue_lock and destroys iog. So in this path, we never hold iocg->lock and queue_lock at the same time. Also, since we remove iog from iocg->group_data under iocg->lock, we can't race with elevator switch.
- Elevator switch path does not remove iog from iocg->group_data list directly. It first hold iocg->lock, scans iocg->group_data again to see if iog is still there; it removes iog only if it finds iog there. Otherwise, cgroup deletion must have removed it from the list, and cgroup deletion is responsible for removing iog.
So the path which removes iog from iocg->group_data list does the final removal of iog by calling __io_destroy_group() function.
--- block/cfq-iosched.c | 18 ++- block/elevator-fq.c | 405 +++++++++++++++++++++++++++++++++------------------ block/elevator-fq.h | 38 +++++- 3 files changed, 312 insertions(+), 149 deletions(-)
block/cfq-iosched.c +15 -3
@@ -1298,8 +1298,8 @@ static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic)if (async_cfqq !=NULL ) { __iog =cfqq_to_io_group (async_cfqq); -if (iog != __iog) { +/* cgroup changed, drop the reference to async queue */ cic_set_cfqq (cic,NULL ,0 );cfq_put_queue (async_cfqq); }
@@ -1307,8 +1307,18 @@ static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic)if (sync_cfqq !=NULL ) { __iog =cfqq_to_io_group (sync_cfqq); -if (iog != __iog) -io_ioq_move (q->elevator, sync_cfqq->ioq, iog); + +/* +* Drop reference to sync queue. A new sync queue will +* be assigned in new group upon arrival of a fresh request. +* If old queue has got requests, those reuests will be +* dispatched over a period of time and queue will be freed +* automatically. +*/ +if (iog != __iog) { +cic_set_cfqq (cic,NULL ,1 ); +cfq_put_queue (sync_cfqq); + } }spin_unlock_irqrestore (q->queue_lock, flags);
@@ -1425,6 +1435,8 @@ alloc_ioq:elv_mark_ioq_sync (cfqq->ioq); } cfqq->pid = current->pid; +/* ioq reference on iog */ +elv_get_iog (iog);cfq_log_cfqq (cfqd, cfqq,"alloced" ); }
block/elevator-fq.c +263 -142
652 lines not on this page · raw message
block/elevator-fq.h +34 -4
@@ -166,7 +166,6 @@ struct io_queue {/* Pointer to generic elevator data structure */ struct elv_fq_data *efqd; -struct list_head queue_list;pid_t pid;/* Number of requests queued on this io queue */
@@ -224,6 +223,7 @@ struct io_group {struct hlist_node elv_data_node;struct hlist_node group_node;struct io_sched_data sched_data; +atomic_t ref;struct io_entity *my_entity;
@@ -244,6 +244,9 @@ struct io_group {/* Single ioq per group, used for noop, deadline, anticipatory */ struct io_queue *ioq; +int deleting; +unsigned short iocg_id; + };/**
@@ -281,9 +284,6 @@ struct elv_fq_data {/* List of io groups hanging on this elevator */ struct hlist_head group_list; -/* List of io queues on idle tree. */ -struct list_head idle_list; -struct request_queue *queue;unsigned int busy_queues;/*
@@ -519,6 +519,8 @@ extern int elv_fq_set_request_ioq(struct request_queue *q, struct request *rq, struct bio *bio, gfp_t gfp_mask);extern void elv_fq_unset_request_ioq (struct request_queue *q,struct request *rq); +extern void elv_put_iog (struct io_group *iog); +extern struct io_queue *elv_lookup_ioq_current (struct request_queue *q);extern struct io_queue *elv_lookup_ioq_bio (struct request_queue *q,struct bio *bio);
@@ -539,6 +541,21 @@ static inline void io_group_set_ioq(struct io_group *iog, struct io_queue *ioq) iog->ioq = ioq; } +static inline void elv_get_iog (struct io_group *iog) +{ +atomic_inc (&iog->ref); +} + +static inline int update_requeue (struct io_queue *ioq,int requeue) +{ +struct io_group *iog =ioq_to_io_group (ioq); + +if (iog->deleting ==1 ) +return 0 ; + +return requeue; +} +#else /* !GROUP_IOSCHED */ /* * No ioq movement is needed in case of flat setup. root io group gets cleaned
@@ -597,6 +614,19 @@ static inline struct io_queue *elv_lookup_ioq_bio(struct request_queue *q, return NULL; } +static inline void elv_get_iog (struct io_group *iog) +{ +} + +static inline void elv_put_iog (struct io_group *iog) +{ +} + +static inline int update_requeue (struct io_queue *ioq,int requeue) +{ +return requeue; +} +#endif /* GROUP_IOSCHED */ /* Functions used by blksysfs.c */
Signature · 1 line
1.6.0.1
Re: [PATCH 16/20] io-controller: IO group refcounting support
Vivek Goyal wrote:
...
Quoted 63 lines
/**
block/elevator-fq.c +21
-6@@ -436,7 +443,6 @@ static void bfq_idle_insert(struct io_service_tree *st, {struct io_entity *first_idle = st->first_idle;struct io_entity *last_idle = st->last_idle; -struct io_queue *ioq =io_entity_to_ioq (entity);if (first_idle ==NULL ||bfq_gt (first_idle->finish, entity->finish)) st->first_idle = entity;@@ -444,10 +450,6 @@ static void bfq_idle_insert(struct io_service_tree *st, st->last_idle = entity;bfq_insert (&st->idle, entity); - -/* Add this queue to idle list */ -if (ioq) -list_add (&ioq->queue_list, &ioq->efqd->idle_list); }/** @@ -723,8 +725,26 @@ int __bfq_deactivate_entity(struct io_entity *entity, int requeue)void bfq_deactivate_entity (struct io_entity *entity,int requeue) {struct io_sched_data *sd; +struct io_group *iog;struct io_entity *parent; + iog = container_of(entity->sched_data, struct io_group, sched_data); + +/* +* Hold a reference to entity's iog until we are done. This function +* travels the hierarchy and we don't want to free up the group yet +* while we are traversing the hiearchy. It is possible that this +* group's cgroup has been removed hence cgroup reference is gone. +* If this entity was active entity, then its group will not be on +* any of the trees and it will be freed up the moment queue is +* freed up in __bfq_deactivate_entity(). +* +* Hence, hold a reference, deactivate the hierarhcy of entities and +* then drop the reference which should free up the whole chain of +* groups. +*/ +elv_get_iog (iog); +for_each_entity_safe (entity, parent) { sd = entity->sched_data;@@ -736,21 +756,28 @@ void bfq_deactivate_entity(struct io_entity *entity, int requeue)*/ break ; -if (sd->next_active !=NULL ) +if (sd->next_active !=NULL ) {/* * The parent entity is still backlogged and * the budgets on the path towards the root * need to be updated. */ +elv_put_iog (iog); goto update; + }
block/elevator-fq.c void bfq_deactivate_entity(struct io_entity *entity, int requeue) · line 777
Quoted 10 lines
block/elevator-fq.c +5
-1/* * If we reach there the parent is no more backlogged and * we want to propagate the dequeue upwards. +* +* If entity's group has been marked for deletion, don't +* requeue the group in idle tree so that it can be freed. */ - requeue =1 ; +if (!iog_deleting (iog)) + requeue =1 ;
Hi Vivek,
IIUC, if the iog is marked deleting, all iogs in the hierarchy don't have a chance to be requeued into idle trees. So, I wonder why do it like this? Why the upper iogs can't be requeued to the idle tree?
Signature · 2 lines
Regards Gui Jianfeng
Re: [PATCH 16/20] io-controller: IO group refcounting support
Quoted 78 lines · Gui Jianfeng
Vivek Goyal wrote:
...Quoted 63 lines
/**
(quoted diff) +21
-6@@ -436,7 +443,6 @@ static void bfq_idle_insert(struct io_service_tree *st, { struct io_entity *first_idle = st->first_idle; struct io_entity *last_idle = st->last_idle; - struct io_queue *ioq = io_entity_to_ioq(entity); if (first_idle == NULL || bfq_gt(first_idle->finish, entity->finish)) st->first_idle = entity;@@ -444,10 +450,6 @@ static void bfq_idle_insert(struct io_service_tree *st, st->last_idle = entity; bfq_insert(&st->idle, entity); - - /* Add this queue to idle list */ - if (ioq) - list_add(&ioq->queue_list, &ioq->efqd->idle_list); } /**@@ -723,8 +725,26 @@ int __bfq_deactivate_entity(struct io_entity *entity, int requeue) void bfq_deactivate_entity(struct io_entity *entity, int requeue) { struct io_sched_data *sd; + struct io_group *iog; struct io_entity *parent; + iog = container_of(entity->sched_data, struct io_group, sched_data); + + /* + * Hold a reference to entity's iog until we are done. This function + * travels the hierarchy and we don't want to free up the group yet + * while we are traversing the hiearchy. It is possible that this + * group's cgroup has been removed hence cgroup reference is gone. + * If this entity was active entity, then its group will not be on + * any of the trees and it will be freed up the moment queue is + * freed up in __bfq_deactivate_entity(). + * + * Hence, hold a reference, deactivate the hierarhcy of entities and + * then drop the reference which should free up the whole chain of + * groups. + */ + elv_get_iog(iog); + for_each_entity_safe(entity, parent) { sd = entity->sched_data;@@ -736,21 +756,28 @@ void bfq_deactivate_entity(struct io_entity *entity, int requeue) */ break; - if (sd->next_active != NULL) + if (sd->next_active != NULL) { /* * The parent entity is still backlogged and * the budgets on the path towards the root * need to be updated. */ + elv_put_iog(iog); goto update; + }Review void bfq_deactivate_entity(struct io_entity *entity, int requeue) · line 777
Quoted 10 lines
(quoted diff) +5
-1/* * If we reach there the parent is no more backlogged and * we want to propagate the dequeue upwards. + * + * If entity's group has been marked for deletion, don't + * requeue the group in idle tree so that it can be freed. */ - requeue = 1; + if (!iog_deleting(iog)) + requeue = 1;Hi Vivek,
Quoted 4 lines
IIUC, if the iog is marked deleting, all iogs in the hierarchy don't have a chance to be requeued into idle trees. So, I wonder why do it like this? Why the upper iogs can't be requeued to the idle tree?
I think this is a bug Gui. Good catch. I think following should fix it.
Thanks
Vivek
--- block/elevator-fq.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
block/elevator-fq.c +6 -3
@@ -863,7 +863,7 @@ int __bfq_deactivate_entity(struct io_envoid bfq_deactivate_entity (struct io_entity *entity,int requeue) {struct io_sched_data *sd; -struct io_group *iog; +struct io_group *iog, *__iog;struct io_entity *parent; iog = container_of(entity->sched_data, struct io_group, sched_data);
@@ -911,8 +911,11 @@ void bfq_deactivate_entity(struct io_ent* If entity's group has been marked for deletion, don't * requeue the group in idle tree so that it can be freed. */ -if (!iog_deleting (iog)) - requeue =1 ; +if (parent) { + __iog = container_of(parent, struct io_group, entity); +if (!iog_deleting (__iog)) + requeue =1 ; + } }elv_put_iog (iog);
A message that is not in the archive (sent off-list or not yet received): [email protected]
Re: [PATCH 16/20] io-controller: IO group refcounting support
Quoted 78 lines · Gui Jianfeng
Vivek Goyal wrote:
...Quoted 63 lines
/**
(quoted diff) +21
-6@@ -436,7 +443,6 @@ static void bfq_idle_insert(struct io_service_tree *st, { struct io_entity *first_idle = st->first_idle; struct io_entity *last_idle = st->last_idle; - struct io_queue *ioq = io_entity_to_ioq(entity); if (first_idle == NULL || bfq_gt(first_idle->finish, entity->finish)) st->first_idle = entity;@@ -444,10 +450,6 @@ static void bfq_idle_insert(struct io_service_tree *st, st->last_idle = entity; bfq_insert(&st->idle, entity); - - /* Add this queue to idle list */ - if (ioq) - list_add(&ioq->queue_list, &ioq->efqd->idle_list); } /**@@ -723,8 +725,26 @@ int __bfq_deactivate_entity(struct io_entity *entity, int requeue) void bfq_deactivate_entity(struct io_entity *entity, int requeue) { struct io_sched_data *sd; + struct io_group *iog; struct io_entity *parent; + iog = container_of(entity->sched_data, struct io_group, sched_data); + + /* + * Hold a reference to entity's iog until we are done. This function + * travels the hierarchy and we don't want to free up the group yet + * while we are traversing the hiearchy. It is possible that this + * group's cgroup has been removed hence cgroup reference is gone. + * If this entity was active entity, then its group will not be on + * any of the trees and it will be freed up the moment queue is + * freed up in __bfq_deactivate_entity(). + * + * Hence, hold a reference, deactivate the hierarhcy of entities and + * then drop the reference which should free up the whole chain of + * groups. + */ + elv_get_iog(iog); + for_each_entity_safe(entity, parent) { sd = entity->sched_data;@@ -736,21 +756,28 @@ void bfq_deactivate_entity(struct io_entity *entity, int requeue) */ break; - if (sd->next_active != NULL) + if (sd->next_active != NULL) { /* * The parent entity is still backlogged and * the budgets on the path towards the root * need to be updated. */ + elv_put_iog(iog); goto update; + }Review void bfq_deactivate_entity(struct io_entity *entity, int requeue) · line 777
Quoted 10 lines
(quoted diff) +5
-1/* * If we reach there the parent is no more backlogged and * we want to propagate the dequeue upwards. + * + * If entity's group has been marked for deletion, don't + * requeue the group in idle tree so that it can be freed. */ - requeue = 1; + if (!iog_deleting(iog)) + requeue = 1;Hi Vivek,
Quoted 4 lines
IIUC, if the iog is marked deleting, all iogs in the hierarchy don't have a chance to be requeued into idle trees. So, I wonder why do it like this? Why the upper iogs can't be requeued to the idle tree?
I think this is a bug Gui. Good catch. I think following should fix it.
Thanks
Vivek
--- block/elevator-fq.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
block/elevator-fq.c +6 -3
@@ -863,7 +863,7 @@ int __bfq_deactivate_entity(struct io_envoid bfq_deactivate_entity (struct io_entity *entity,int requeue) {struct io_sched_data *sd; -struct io_group *iog; +struct io_group *iog, *__iog;struct io_entity *parent; iog = container_of(entity->sched_data, struct io_group, sched_data);
@@ -911,8 +911,11 @@ void bfq_deactivate_entity(struct io_ent* If entity's group has been marked for deletion, don't * requeue the group in idle tree so that it can be freed. */ -if (!iog_deleting (iog)) - requeue =1 ; +if (parent) { + __iog = container_of(parent, struct io_group, entity); +if (!iog_deleting (__iog)) + requeue =1 ; + } }elv_put_iog (iog);
PATCH 17/20 io-controller: Per cgroup request descriptor support
In reply to Vivek Goyalo Currently a request queue has got fixed number of request descriptors for sync and async requests. Once the request descriptors are consumed, new processes are put to sleep and they effectively become serialized. Because sync and async queues are separate, async requests don't impact sync ones but if one is looking for fairness between async requests, that is not achievable if request queue descriptors become bottleneck.
o Make request descriptor's per io group so that if there is lots of IO going on in one cgroup, it does not impact the IO of other group.
o This is just one relatively simple way of doing things. This patch will probably change after the feedback. Folks have raised concerns that in hierchical setup, child's request descriptors should be capped by parent's request descriptors. May be we need to have per cgroup per device files in cgroups where one can specify the upper limit of request descriptors and whenever a cgroup is created one needs to assign request descritor limit making sure total sum of child's request descriptor is not more than of parent.
I guess something like memory controller. Anyway, that would be the next step. For the time being, we have implemented something simpler as follows.
o This patch implements the per cgroup request descriptors. request pool per queue is still common but every group will have its own wait list and its own count of request descriptors allocated to that group for sync and async queues. So effectively request_list becomes per io group property and not a global request queue feature.
o Currently one can define q->nr_requests to limit request descriptors allocated for the queue. Now there is another tunable q->nr_group_requests which controls the requests descriptr limit per group. q->nr_requests supercedes q->nr_group_requests to make sure if there are lots of groups present, we don't end up allocating too many request descriptors on the queue.
o Issues: Currently notion of congestion is per queue. With per group request descriptor it is possible that queue is not congested but the group bio will go into is congested.
--- block/blk-core.c | 305 +++++++++++++++++++++++++++++++++++++---------- block/blk-settings.c | 1 + block/blk-sysfs.c | 57 +++++++-- block/elevator-fq.c | 14 +++ block/elevator-fq.h | 4 + block/elevator.c | 6 +- include/linux/blkdev.h | 87 +++++++++++++- 7 files changed, 392 insertions(+), 82 deletions(-)
block/blk-core.c +241 -66
@@ -480,20 +480,30 @@ void blk_cleanup_queue(struct request_queue *q) }EXPORT_SYMBOL (blk_cleanup_queue); -static int blk_init_free_list (struct request_queue *q) +void blk_init_request_list (struct request_list *rl) { -struct request_list *rl = &q->rq; rl->count[BLK_RW_SYNC ] = rl->count[BLK_RW_ASYNC ] =0 ; - rl->starved[BLK_RW_SYNC ] = rl->starved[BLK_RW_ASYNC ] =0 ; - rl->elvpriv =0 ;init_waitqueue_head (&rl->wait[BLK_RW_SYNC ]);init_waitqueue_head (&rl->wait[BLK_RW_ASYNC ]); +} - rl->rq_pool =mempool_create_node (BLKDEV_MIN_RQ , mempool_alloc_slab, - mempool_free_slab, request_cachep, q->node); +static int blk_init_free_list (struct request_queue *q) +{ +/* +* Initialize the queue request list in case there are non-hiearchical +* io schedulers not making use of fair queuing infrastructure. +* +* For ioschedulers making use of fair queuing infrastructure, request +* list is inside the associated group and when that group is +* instanciated, it takes care of initializing the request list also. +*/ +blk_init_request_list (&q->rq); + q->rq_data.rq_pool =mempool_create_node (BLKDEV_MIN_RQ , + mempool_alloc_slab, mempool_free_slab, + request_cachep, q->node); -if (!rl->rq_pool) +if (!q->rq_data.rq_pool)return -ENOMEM ;return 0 ;
@@ -590,6 +600,9 @@ blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)return NULL ; } +/* init starved waiter wait queue */ +init_waitqueue_head (&q->rq_data.starved_wait); +/* * if caller didn't supply a lock, they get per-queue locking with * our embedded lock
@@ -639,14 +652,14 @@ static inline void blk_free_request(struct request_queue *q, struct request *rq) {if (rq->cmd_flags &REQ_ELVPRIV )elv_put_request (q, rq); -mempool_free (rq, q->rq.rq_pool); +mempool_free (rq, q->rq_data.rq_pool); }static struct request *blk_alloc_request (struct request_queue *q,struct bio *bio,int flags,int priv,gfp_t gfp_mask) { -struct request *rq =mempool_alloc (q->rq.rq_pool, gfp_mask); +struct request *rq =mempool_alloc (q->rq_data.rq_pool, gfp_mask);if (!rq)return NULL ;
@@ -657,7 +670,7 @@ blk_alloc_request(struct request_queue *q, struct bio *bio, int flags, int priv,if (priv) {if (unlikely (elv_set_request (q, rq, bio, gfp_mask))) { -mempool_free (rq, q->rq.rq_pool); +mempool_free (rq, q->rq_data.rq_pool);return NULL ; } rq->cmd_flags |=REQ_ELVPRIV ;
@@ -700,18 +713,18 @@ static void ioc_set_batching(struct request_queue *q, struct io_context *ioc) ioc->last_waited = jiffies; } -static void __freed_request (struct request_queue *q,int sync) +static void __freed_request (struct request_queue *q,int sync, +struct request_list *rl) { -struct request_list *rl = &q->rq; - -if (rl->count[sync] <queue_congestion_off_threshold (q)) +if (q->rq_data.count[sync] <queue_congestion_off_threshold (q))blk_clear_queue_congested (q, sync); -if (rl->count[sync] +1 <= q->nr_requests) { +if (q->rq_data.count[sync] +1 <= q->nr_requests) +blk_clear_queue_full (q, sync); + +if (rl->count[sync] +1 <= q->nr_group_requests) {if (waitqueue_active (&rl->wait[sync]))wake_up (&rl->wait[sync]); - -blk_clear_queue_full (q, sync); } }
@@ -719,63 +732,133 @@ static void __freed_request(struct request_queue *q, int sync)* A request has just been released. Account for it, update the full and * congestion status, wake up any waiters. Called under q->queue_lock. */ -static void freed_request (struct request_queue *q,int sync,int priv) -{ -struct request_list *rl = &q->rq; +static void freed_request (struct request_queue *q,int sync,int priv, +struct request_list *rl) +{ +/* There is a window during request allocation where request is +* mapped to one group but by the time a queue for the group is +* allocated, it is possible that original cgroup/io group has been +* deleted and now io queue is allocated in a different group (root) +* altogether. +* +* One solution to the problem is that rq should take io group +* reference. But it looks too much to do that to solve this issue. +* The only side affect to the hard to hit issue seems to be that +* we will try to decrement the rl->count for a request list which +* did not allocate that request. Chcek for rl->count going less than +* zero and do not decrement it if that's the case. +*/ + +if (priv && rl->count[sync] >0 ) + rl->count[sync]--; + +BUG_ON (!q->rq_data.count[sync]); + q->rq_data.count[sync]--; - rl->count[sync]--;if (priv) - rl->elvpriv--; + q->rq_data.elvpriv--; -__freed_request (q, sync); +__freed_request (q, sync, rl);if (unlikely (rl->starved[sync ^1 ])) -__freed_request (q, sync ^1 ); +__freed_request (q, sync ^1 , rl); + +/* Wake up the starved process on global list, if any */ +if (unlikely (q->rq_data.starved)) { +if (waitqueue_active (&q->rq_data.starved_wait)) +wake_up (&q->rq_data.starved_wait); + q->rq_data.starved--; + } +} + +/* +* Returns whether one can sleep on this request list or not. There are +* cases (elevator switch) where request list might not have allocated +* any request descriptor but we deny request allocation due to gloabl +* limits. In that case one should sleep on global list as on this request +* list no wakeup will take place. +* +* Also sets the request list starved flag if there are no requests pending +* in the direction of rq. +* +* Return 1 --> sleep on request list, 0 --> sleep on global list +*/ +static int can_sleep_on_request_list (struct request_list *rl,int is_sync) +{ +if (unlikely (rl->count[is_sync] ==0 )) { +/* +* If there is a request pending in other direction +* in same io group, then set the starved flag of +* the group request list. Otherwise, we need to +* make this process sleep in global starved list +* to make sure it will not sleep indefinitely. +*/ +if (rl->count[is_sync ^1 ] !=0 ) { + rl->starved[is_sync] =1 ; +return 1 ; + }else +return 0 ; + } + +return 1 ; }/* * Get a free request, queue_lock must be held. -* Returns NULL on failure, with queue_lock held. +* Returns NULL on failure, with queue_lock held. Also sets the "reason" field +* in case of failure. This reason field helps caller decide to whether sleep +* on per group list or global per queue list. +* reason = 0 sleep on per group list +* reason = 1 sleep on global list +* * Returns !NULL on success, with queue_lock *not held*. */ static struct request *get_request (struct request_queue *q,int rw_flags, -struct bio *bio,gfp_t gfp_mask) +struct bio *bio,gfp_t gfp_mask, +struct request_list *rl,int *reason) {struct request *rq =NULL ; -struct request_list *rl = &q->rq;struct io_context *ioc =NULL ;const bool is_sync =rw_is_sync (rw_flags) !=0 ;int may_queue, priv; +int sleep_on_global =0 ; may_queue =elv_may_queue (q, rw_flags);if (may_queue ==ELV_MQUEUE_NO ) goto rq_starved; -if (rl->count[is_sync]+1 >=queue_congestion_on_threshold (q)) { -if (rl->count[is_sync]+1 >= q->nr_requests) { - ioc =current_io_context (GFP_ATOMIC , q->node); -/* -* The queue will fill after this allocation, so set -* it as full, and mark this process as "batching". -* This process will be allowed to complete a batch of -* requests, others will be blocked. -*/ -if (!blk_queue_full (q, is_sync)) { -ioc_set_batching (q, ioc); -blk_set_queue_full (q, is_sync); - }else { -if (may_queue !=ELV_MQUEUE_MUST - && !ioc_batching (q, ioc)) { -/* -* The queue is full and the allocating -* process is not a "batcher", and not -* exempted by the IO scheduler -*/ - goto out; - } +if (q->rq_data.count[is_sync]+1 >=queue_congestion_on_threshold (q)) +blk_set_queue_congested (q, is_sync); + +/* +* Looks like there is no user of queue full now. +* Keeping it for time being. +*/ +if (q->rq_data.count[is_sync]+1 >= q->nr_requests) +blk_set_queue_full (q, is_sync); + +if (rl->count[is_sync]+1 >= q->nr_group_requests) { + ioc =current_io_context (GFP_ATOMIC , q->node); +/* +* The queue request descriptor group will fill after this +* allocation, so set +* it as full, and mark this process as "batching". +* This process will be allowed to complete a batch of +* requests, others will be blocked. +*/ +if (rl->count[is_sync] <= q->nr_group_requests) +ioc_set_batching (q, ioc); +else { +if (may_queue !=ELV_MQUEUE_MUST + && !ioc_batching (q, ioc)) { +/* +* The queue is full and the allocating +* process is not a "batcher", and not +* exempted by the IO scheduler +*/ + goto out; } } -blk_set_queue_congested (q, is_sync); }/*
@@ -783,21 +866,60 @@ static struct request *get_request(struct request_queue *q, int rw_flags,* limit of requests, otherwise we could have thousands of requests * allocated with any setting of ->nr_requests */ -if (rl->count[is_sync] >= (3 * q->nr_requests /2 )) + +if (q->rq_data.count[is_sync] >= (3 * q->nr_requests /2 )) { +/* +* Queue is too full for allocation. On which request queue +* the task should sleep? Generally it should sleep on its +* request list but if elevator switch is happening, in that +* window, request descriptors are allocated from global +* pool and are not accounted against any particular request +* list as group is going away. +* +* So it might happen that request list does not have any +* requests allocated at all and if process sleeps on per +* group request list, it will not be woken up. In such case, +* make it sleep on global starved list. +*/ +if (test_bit (QUEUE_FLAG_ELVSWITCH , &q->queue_flags) + || !can_sleep_on_request_list (rl, is_sync)) + sleep_on_global =1 ; + goto out; + } + +/* +* Allocation of request is allowed from queue perspective. Now check +* from per group request list +*/ + +if (rl->count[is_sync] >= (3 * q->nr_group_requests /2 )) goto out; - rl->count[is_sync]++; rl->starved[is_sync] =0 ; + q->rq_data.count[is_sync]++; + priv = !test_bit (QUEUE_FLAG_ELVSWITCH , &q->queue_flags); -if (priv) - rl->elvpriv++; +if (priv) { + q->rq_data.elvpriv++; +/* +* Account the request to request list only if request is +* going to elevator. During elevator switch, there will +* be small window where group is going away and new group +* will not be allocated till elevator switch is complete. +* So till then instead of slowing down the application, +* we will continue to allocate request from total common +* pool instead of per group limit +*/ + rl->count[is_sync]++; + }if (blk_queue_io_stat (q)) rw_flags |=REQ_IO_STAT ;spin_unlock_irq (q->queue_lock); rq =blk_alloc_request (q, bio, rw_flags, priv, gfp_mask); +if (unlikely (!rq)) {/* * Allocation failed presumably due to memory. Undo anything
@@ -807,7 +929,7 @@ static struct request *get_request(struct request_queue *q, int rw_flags,* wait queue, but this is pretty rare. */ spin_lock_irq (q->queue_lock); -freed_request (q, is_sync, priv); +freed_request (q, is_sync, priv, rl);/* * in the very unlikely event that allocation failed and no
@@ -817,9 +939,8 @@ static struct request *get_request(struct request_queue *q, int rw_flags,* rq mempool into READ and WRITE */ rq_starved: -if (unlikely (rl->count[is_sync] ==0 )) - rl->starved[is_sync] =1 ; - +if (!can_sleep_on_request_list (rl, is_sync)) + sleep_on_global =1 ; goto out; }
@@ -834,6 +955,8 @@ rq_starved:trace_block_getrq (q, bio, rw_flags &1 ); out: +if (reason && sleep_on_global) + *reason =1 ;return rq; }
@@ -847,16 +970,44 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags, struct bio *bio) {const bool is_sync =rw_is_sync (rw_flags) !=0 ; +int sleep_on_global =0 ;struct request *rq; +struct request_list *rl =blk_get_request_list (q, bio); +struct io_group *iog =NULL ; - rq =get_request (q, rw_flags, bio,GFP_NOIO ); + rq =get_request (q, rw_flags, bio,GFP_NOIO , rl, &sleep_on_global);while (!rq) {DEFINE_WAIT (wait);struct io_context *ioc; -struct request_list *rl = &q->rq; -prepare_to_wait_exclusive (&rl->wait[is_sync], &wait, -TASK_UNINTERRUPTIBLE ); +if (sleep_on_global) { +/* +* Task failed allocation and needs to wait and +* try again. There are no requests pending from +* the io group hence need to sleep on global +* wait queue. Most likely the allocation failed +* because of memory issues. +*/ + + q->rq_data.starved++; +prepare_to_wait_exclusive (&q->rq_data.starved_wait, + &wait,TASK_UNINTERRUPTIBLE ); + }else { +/* +* We are about to sleep on a request list and we +* drop queue lock. After waking up, we will do +* finish_wait() on request list and in the mean +* time group might be gone. Take a reference to +* the group now. +*/ +prepare_to_wait_exclusive (&rl->wait[is_sync], &wait, +TASK_UNINTERRUPTIBLE ); +#ifdef CONFIG_GROUP_IOSCHED + iog =rl_iog (rl); +if (iog) +elv_get_iog (iog); +#endif + }trace_block_sleeprq (q, bio, rw_flags &1 );
@@ -874,9 +1025,30 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,ioc_set_batching (q, ioc);spin_lock_irq (q->queue_lock); -finish_wait (&rl->wait[is_sync], &wait); - rq =get_request (q, rw_flags, bio,GFP_NOIO ); +if (sleep_on_global) { +finish_wait (&q->rq_data.starved_wait, &wait); + sleep_on_global =0 ; + }else { +finish_wait (&rl->wait[is_sync], &wait); +#ifdef CONFIG_GROUP_IOSCHED +/* +* We had taken a reference to the rl/iog. +* Put that now +*/ + iog =rl_iog (rl); +if (iog) +elv_put_iog (iog); +#endif + } + +/* +* After the sleep check the rl again in case cgrop bio +* belonged to is gone and it is mapped to root group now +*/ + rl =blk_get_request_list (q, bio); + rq =get_request (q, rw_flags, bio,GFP_NOIO , rl, + &sleep_on_global); };return rq;
@@ -885,14 +1057,16 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,struct request *blk_get_request (struct request_queue *q,int rw,gfp_t gfp_mask) {struct request *rq; +struct request_list *rl;BUG_ON (rw !=READ && rw !=WRITE );spin_lock_irq (q->queue_lock); + rl =blk_get_request_list (q,NULL );if (gfp_mask & __GFP_WAIT) { rq =get_request_wait (q, rw,NULL ); }else { - rq =get_request (q, rw,NULL , gfp_mask); + rq =get_request (q, rw,NULL , gfp_mask, rl,NULL );if (!rq)spin_unlock_irq (q->queue_lock); }
@@ -1075,12 +1249,13 @@ void __blk_put_request(struct request_queue *q, struct request *req)if (req->cmd_flags &REQ_ALLOCED ) {int is_sync =rq_is_sync (req) !=0 ;int priv = req->cmd_flags &REQ_ELVPRIV ; +struct request_list *rl =rq_rl (q, req);BUG_ON (!list_empty (&req->queuelist));BUG_ON (!hlist_unhashed (&req->hash));blk_free_request (q, req); -freed_request (q, is_sync, priv); +freed_request (q, is_sync, priv, rl); } }EXPORT_SYMBOL_GPL (__blk_put_request);
block/blk-settings.c +1 -0
@@ -123,6 +123,7 @@ void blk_queue_make_request(struct request_queue *q, make_request_fn *mfn)* set defaults */ q->nr_requests =BLKDEV_MAX_RQ ; + q->nr_group_requests =BLKDEV_MAX_GROUP_RQ ;blk_queue_max_phys_segments (q,MAX_PHYS_SEGMENTS );blk_queue_max_hw_segments (q,MAX_HW_SEGMENTS );blk_queue_segment_boundary (q,BLK_SEG_BOUNDARY_MASK );
block/blk-sysfs.c +45 -12
108 lines not on this page · raw message
block/elevator-fq.c +14 -0
@@ -996,6 +996,16 @@ struct io_cgroup *cgroup_to_io_cgroup(struct cgroup *cgroup) struct io_cgroup, css); } +struct request_list *io_group_get_request_list (struct request_queue *q, +struct bio *bio) +{ +struct io_group *iog; + + iog =io_get_io_group (q, bio,1 ); +BUG_ON (!iog); +return &iog->rl; +} +/* * Search the bfq_group for bfqd into the hash table (by now only a list) * of bgrp. Must be called under rcu_read_lock().
@@ -1224,6 +1234,8 @@ struct io_group *io_group_chain_alloc(struct request_queue *q, void *key, iog->iocg_id =css_id (&iocg->css); +blk_init_request_list (&iog->rl); +if (leaf ==NULL ) { leaf = iog; prev = leaf;
@@ -1483,6 +1495,8 @@ struct io_group *io_alloc_root_group(struct request_queue *q,for (i =0 ; i <IO_IOPRIO_CLASSES ; i++) iog->sched_data.service_tree[i] =IO_SERVICE_TREE_INIT ; +blk_init_request_list (&iog->rl); + iocg = &io_root_cgroup;spin_lock_irq (&iocg->lock);rcu_assign_pointer (iog->key, key);
block/elevator-fq.h +4 -0
@@ -247,6 +247,8 @@ struct io_group {int deleting;unsigned short iocg_id; +/* request list associated with the group */ +struct request_list rl; };/**
@@ -524,6 +526,8 @@ extern void elv_put_iog(struct io_group *iog);extern struct io_queue *elv_lookup_ioq_current (struct request_queue *q);extern struct io_queue *elv_lookup_ioq_bio (struct request_queue *q,struct bio *bio); +extern struct request_list *io_group_get_request_list (struct request_queue *q, +struct bio *bio);/* Returns single ioq associated with the io group. */ static inline struct io_queue *io_group_ioq (struct io_group *iog)
block/elevator.c +3 -3
@@ -668,7 +668,7 @@ void elv_quiesce_start(struct request_queue *q)* make sure we don't have any requests in flight */ elv_drain_elevator (q); -while (q->rq.elvpriv) { +while (q->rq_data.elvpriv) {blk_start_queueing (q);spin_unlock_irq (q->queue_lock);msleep (10 );
@@ -768,8 +768,8 @@ void elv_insert(struct request_queue *q, struct request *rq, int where) }if (unplug_it &&blk_queue_plugged (q)) { -int nrq = q->rq.count[BLK_RW_SYNC ] + q->rq.count[BLK_RW_ASYNC ] - - q->in_flight; +int nrq = q->rq_data.count[BLK_RW_SYNC ] + + q->rq_data.count[BLK_RW_ASYNC ] - q->in_flight;if (nrq >= q->unplug_thresh)__generic_unplug_device (q);
include/linux/blkdev.h +85 -2
124 lines not on this page · raw message
Signature · 1 line
1.6.0.1
PATCH 18/20 io-controller: Support per cgroup per device weights and io class
In reply to Vivek GoyalThis patch enables per-cgroup per-device weight and ioprio_class handling. A new cgroup interface "policy" is introduced. You can make use of this file to configure weight and ioprio_class for each device in a given cgroup. The original "weight" and "ioprio_class" files are still available. If you don't do special configuration for a particular device, "weight" and "ioprio_class" are used as default values in this device.
You can use the following format to play with the new interface. #echo DEV:weight:ioprio_class > /patch/to/cgroup/policy weight=0 means removing the policy for DEV.
Examples:
Configure weight=300 ioprio_class=2 on /dev/hdb in this cgroup
# echo /dev/hdb:300:2 > io.policy # cat io.policy dev weight class /dev/hdb 300 2
Configure weight=500 ioprio_class=1 on /dev/hda in this cgroup
# echo /dev/hda:500:1 > io.policy # cat io.policy dev weight class /dev/hda 500 1 /dev/hdb 300 2
Remove the policy for /dev/hda in this cgroup
# echo /dev/hda:0:1 > io.policy # cat io.policy dev weight class /dev/hdb 300 2
Changelog (v1 -> v2)
- Rename some structures
- Use spin_lock_irqsave() and spin_lock_irqrestore() version to prevent from enabling the interrupts unconditionally.
- Fix policy setup bug when switching to another io scheduler.
- If a policy is available for a specific device, don't update weight and io class when writing "weight" and "iprio_class".
- Fix a bug when parsing policy string.
--- block/elevator-fq.c | 245 +++++++++++++++++++++++++++++++++++++++++++++++++- block/elevator-fq.h | 11 +++ 2 files changed, 251 insertions(+), 5 deletions(-)
block/elevator-fq.c +240 -5
@@ -13,6 +13,7 @@#include <linux/blktrace_api.h> #include <linux/seq_file.h> #include <linux/biotrack.h> +#include <linux/genhd.h> /* Values taken from cfq */ const int elv_slice_sync =HZ /10 ;
@@ -1025,12 +1026,31 @@ struct io_group *io_cgroup_lookup_group(struct io_cgroup *iocg, void *key)return NULL ; } -void io_group_init_entity (struct io_cgroup *iocg,struct io_group *iog) +static struct io_policy_node *policy_search_node (const struct io_cgroup *iocg, +dev_t dev); + +void io_group_init_entity (struct io_cgroup *iocg,struct io_group *iog, +dev_t dev) {struct io_entity *entity = &iog->entity; +struct io_policy_node *pn; +unsigned long flags; + +spin_lock_irqsave (&iocg->lock, flags); + pn =policy_search_node (iocg, dev); +if (pn) { + entity->weight = pn->weight; + entity->new_weight = pn->weight; + entity->ioprio_class = pn->ioprio_class; + entity->new_ioprio_class = pn->ioprio_class; + }else { + entity->weight = iocg->weight; + entity->new_weight = iocg->weight; + entity->ioprio_class = iocg->ioprio_class; + entity->new_ioprio_class = iocg->ioprio_class; + } +spin_unlock_irqrestore (&iocg->lock, flags); - entity->weight = entity->new_weight = iocg->weight; - entity->ioprio_class = entity->new_ioprio_class = iocg->ioprio_class; entity->ioprio_changed =1 ; entity->my_sched_data = &iog->sched_data; }
@@ -1094,6 +1114,7 @@ static int io_cgroup_##__VAR##_write(struct cgroup *cgroup, \struct io_cgroup *iocg; \struct io_group *iog; \struct hlist_node *n; \ +struct io_policy_node *pn; \ \if (val < (__MIN) || val > (__MAX)) \return -EINVAL ; \
@@ -1106,6 +1127,9 @@ static int io_cgroup_##__VAR##_write(struct cgroup *cgroup, \spin_lock_irq (&iocg->lock); \ iocg->__VAR = (unsigned long )val; \ hlist_for_each_entry(iog, n, &iocg->group_data, group_node) { \ + pn =policy_search_node (iocg, iog->dev); \ +if (pn) \ +continue ; \ iog->entity.new_##__VAR = (unsigned long)val; \smp_wmb (); \ iog->entity.ioprio_changed =1 ; \
@@ -1214,8 +1238,6 @@ struct io_group *io_group_chain_alloc(struct request_queue *q, void *key,if (!iog) goto cleanup; -io_group_init_entity (iocg, iog); - iog->my_entity = &iog->entity;atomic_set (&iog->ref,0 ); iog->deleting =0 ;
@@ -1233,6 +1255,8 @@ struct io_group *io_group_chain_alloc(struct request_queue *q, void *key, } iog->iocg_id =css_id (&iocg->css); +io_group_init_entity (iocg, iog, iog->dev); + iog->my_entity = &iog->entity;blk_init_request_list (&iog->rl);
@@ -1507,8 +1531,212 @@ struct io_group *io_alloc_root_group(struct request_queue *q,return iog; } +static int io_cgroup_policy_read (struct cgroup *cgrp,struct cftype *cft, +struct seq_file *m) +{ +struct io_cgroup *iocg; +struct io_policy_node *pn; + + iocg =cgroup_to_io_cgroup (cgrp); + +if (list_empty (&iocg->policy_list)) + goto out; + +seq_printf (m,"dev weight class\n" ); + +spin_lock_irq (&iocg->lock); +list_for_each_entry (pn, &iocg->policy_list, node) { +seq_printf (m,"%s %lu %lu\n" , pn->dev_name, + pn->weight, pn->ioprio_class); + } +spin_unlock_irq (&iocg->lock); +out: +return 0 ; +} + +static inline void policy_insert_node (struct io_cgroup *iocg, +struct io_policy_node *pn) +{ +list_add (&pn->node, &iocg->policy_list); +} + +/* Must be called with iocg->lock held */ +static inline void policy_delete_node (struct io_policy_node *pn) +{ +list_del (&pn->node); +} + +/* Must be called with iocg->lock held */ +static struct io_policy_node *policy_search_node (const struct io_cgroup *iocg, +dev_t dev) +{ +struct io_policy_node *pn; + +if (list_empty (&iocg->policy_list)) +return NULL ; + +list_for_each_entry (pn, &iocg->policy_list, node) { +if (pn->dev == dev) +return pn; + } + +return NULL ; +} + +static int devname_to_devnum (const char *buf,dev_t *dev) +{ +struct block_device *bdev; +struct gendisk *disk; +int part; + + bdev =lookup_bdev (buf); +if (IS_ERR (bdev)) +return -ENODEV ; + + disk =get_gendisk (bdev->bd_dev, &part); +if (part) +return -EINVAL ; + + *dev =MKDEV (disk->major, disk->first_minor); +bdput (bdev); + +return 0 ; +} + +static int policy_parse_and_set (char *buf,struct io_policy_node *newpn) +{ +char *s[3 ], *p; +int ret; +int i =0 ; + +memset (s,0 ,sizeof (s)); +while ((p =strsep (&buf,":" )) !=NULL ) { +if (!*p) +continue ; + s[i++] = p; + } + + ret =devname_to_devnum (s[0 ], &newpn->dev); +if (ret) +return ret; + +strcpy (newpn->dev_name, s[0 ]); + +if (s[1 ] ==NULL ) +return -EINVAL ; + + ret =strict_strtoul (s[1 ],10 , &newpn->weight); +if (ret || newpn->weight >WEIGHT_MAX ) +return -EINVAL ; + +if (s[2 ] ==NULL ) +return -EINVAL ; + + ret =strict_strtoul (s[2 ],10 , &newpn->ioprio_class); +if (ret || newpn->ioprio_class <IOPRIO_CLASS_RT || + newpn->ioprio_class >IOPRIO_CLASS_IDLE ) +return -EINVAL ; + +return 0 ; +} + +static int io_cgroup_policy_write (struct cgroup *cgrp,struct cftype *cft, +const char *buffer) +{ +struct io_cgroup *iocg; +struct io_policy_node *newpn, *pn; +char *buf; +int ret =0 ; +int keep_newpn =0 ; +struct hlist_node *n; +struct io_group *iog; + + buf =kstrdup (buffer,GFP_KERNEL ); +if (!buf) +return -ENOMEM ; + + newpn =kzalloc (sizeof (*newpn),GFP_KERNEL ); +if (!newpn) { + ret = -ENOMEM ; + goto free_buf; + } + + ret =policy_parse_and_set (buf, newpn); +if (ret) + goto free_newpn; + +if (!cgroup_lock_live_group (cgrp)) { + ret = -ENODEV ; + goto free_newpn; + } + + iocg =cgroup_to_io_cgroup (cgrp); +spin_lock_irq (&iocg->lock); + + pn =policy_search_node (iocg, newpn->dev); +if (!pn) { +if (newpn->weight !=0 ) { +policy_insert_node (iocg, newpn); + keep_newpn =1 ; + } + goto update_io_group; + } + +if (newpn->weight ==0 ) { +/* weight == 0 means deleteing a policy */ +policy_delete_node (pn); + goto update_io_group; + } + + pn->weight = newpn->weight; + pn->ioprio_class = newpn->ioprio_class; + +update_io_group: +hlist_for_each_entry (iog, n, &iocg->group_data, group_node) { +if (iog->dev == newpn->dev) { +if (newpn->weight) { + iog->entity.new_weight = newpn->weight; + iog->entity.new_ioprio_class = + newpn->ioprio_class; +/* +* iog weight and ioprio_class updating +* actually happens if ioprio_changed is set. +* So ensure ioprio_changed is not set until +* new weight and new ioprio_class are updated. +*/ +smp_wmb (); + iog->entity.ioprio_changed =1 ; + }else { + iog->entity.new_weight = iocg->weight; + iog->entity.new_ioprio_class = + iocg->ioprio_class; + +/* The same as above */ +smp_wmb (); + iog->entity.ioprio_changed =1 ; + } + } + } +spin_unlock_irq (&iocg->lock); + +cgroup_unlock (); + +free_newpn: +if (!keep_newpn) +kfree (newpn); +free_buf: +kfree (buf); +return ret; +} +struct cftype bfqio_files[] = { { + .name ="policy" , + .read_seq_string = io_cgroup_policy_read, + .write_string = io_cgroup_policy_write, + .max_write_len =256 , + }, + { .name ="weight" , .read_u64 = io_cgroup_weight_read, .write_u64 = io_cgroup_weight_write,
@@ -1550,6 +1778,7 @@ struct cgroup_subsys_state *iocg_create(struct cgroup_subsys *subsys,INIT_HLIST_HEAD (&iocg->group_data); iocg->weight =IO_DEFAULT_GRP_WEIGHT ; iocg->ioprio_class =IO_DEFAULT_GRP_CLASS ; +INIT_LIST_HEAD (&iocg->policy_list);return &iocg->css; }
@@ -1739,6 +1968,7 @@ void iocg_destroy(struct cgroup_subsys *subsys, struct cgroup *cgroup)struct io_group *iog;struct elv_fq_data *efqd;unsigned long uninitialized_var (flags ); +struct io_policy_node *pn, *pntmp;/* * io groups are linked in two lists. One list is maintained
@@ -1777,6 +2007,11 @@ remove_entry: goto remove_entry; done: +list_for_each_entry_safe (pn, pntmp, &iocg->policy_list, node) { +policy_delete_node (pn); +kfree (pn); + } +free_css_id (&io_subsys, &iocg->css);rcu_read_unlock ();BUG_ON (!hlist_empty (&iocg->group_data));
block/elevator-fq.h +11 -0
@@ -251,6 +251,14 @@ struct io_group {struct request_list rl; }; +struct io_policy_node { +struct list_head node; +char dev_name[32 ]; +dev_t dev; +unsigned long weight; +unsigned long ioprio_class; +}; +/** * struct bfqio_cgroup - bfq cgroup data structure. * @css: subsystem state for bfq in the containing cgroup.
@@ -267,6 +275,9 @@ struct io_cgroup {unsigned long weight, ioprio_class; +/* list of io_policy_node */ +struct list_head policy_list; +spinlock_t lock;struct hlist_head group_data; };
Signature · 1 line
1.6.0.1
PATCH 19/20 io-controller: Debug hierarchical IO scheduling
In reply to Vivek Goyalo Littile debugging aid for hierarchical IO scheduling.
o Enabled under CONFIG_DEBUG_GROUP_IOSCHED
o Currently it outputs more debug messages in blktrace output which helps a great deal in debugging in hierarchical setup. It also creates additional cgroup interfaces io.disk_queue and io.disk_dequeue to output some more debugging data.
--- block/Kconfig.iosched | 10 ++- block/elevator-fq.c | 298 ++++++++++++++++++++++++++++++++++++++++++++++++- block/elevator-fq.h | 14 +++ 3 files changed, 318 insertions(+), 4 deletions(-)
block/Kconfig.iosched +9 -1
@@ -140,6 +140,14 @@ config TRACK_ASYNC_CONTEXT request, original owner of the bio is decided by using io tracking patches otherwise we continue to attribute the request to the submitting thread. -endmenu +config DEBUG_GROUP_IOSCHED + bool "Debug Hierarchical Scheduling support" + depends on CGROUPS && GROUP_IOSCHED + default n + ---help--- + Enable some debugging hooks for hierarchical scheduling support. + Currently it just outputs more information in blktrace output. + +endmenu endif
block/elevator-fq.c +295 -3
397 lines not on this page · raw message
block/elevator-fq.h +14 -0
@@ -249,6 +249,20 @@ struct io_group {/* request list associated with the group */ struct request_list rl; + +#ifdef CONFIG_DEBUG_GROUP_IOSCHED +/* How many times this group has been added to active tree */ +unsigned long queue; + +/* How long this group remained on active tree, in ms */ +unsigned long queue_duration; + +/* When was this group added to active tree */ +unsigned long queue_start; + +/* How many times this group has been removed from active tree */ +unsigned long dequeue; +#endif };struct io_policy_node {
Signature · 1 line
1.6.0.1
PATCH 20/20 io-controller: experimental debug patch for async queue wait before expiry
In reply to Vivek Goyalo A debug patch which does wait for next IO from async queue once it becomes empty.
o For async writes, traffic seen by IO scheduler is not in proportion to the weight of the cgroup task/page belongs to. So if there are two processes doing heavy writeouts in two cgroups with weights 1000 and 500 respectively, then IO scheduler does not see more traffic/IO from higher weight cgroup even if IO scheduler tries to give it higher disk time. Effectively, the async queue belonging to higher weight cgroup becomes empty, and gets out of contention for disk and lower weight cgroup gets to use disk giving an impression in user space that higher weight cgroup did not get higher time to disk.
o This is more of a problem at page cache level where a higher weight process might be writing out the pages of lower weight process etc and should be fixed there.
o While we fix those issues, introducing this debug patch which allows one
to idle on async queue (tunable via /sys/blolc/<disk>/queue/async_slice_idle) so that once a higher weight queue becomes empty, instead of expiring it we try to wait for next request to come from that queue hence giving it
higher disk time. A higher value of async_slice_idle, around 300ms, helps
me get some right numbers for my setup. Note: higher disk time would not
necessarily translate in more IO done as higher weight group is not pushing
enough IO to io scheduler. It is just a debugging aid to prove correctness
of IO controller by providing higher disk times to higher weight cgroup.
--- block/blk-sysfs.c | 7 +++++ block/elevator-fq.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++--- block/elevator-fq.h | 7 +++++ 3 files changed, 77 insertions(+), 4 deletions(-)
block/blk-sysfs.c +7 -0
@@ -314,6 +314,12 @@ static struct queue_sysfs_entry queue_slice_idle_entry = { .store = elv_slice_idle_store, }; +static struct queue_sysfs_entry queue_async_slice_idle_entry = { + .attr = {.name ="async_slice_idle" , .mode =S_IRUGO |S_IWUSR }, + .show = elv_async_slice_idle_show, + .store = elv_async_slice_idle_store, +}; +static struct queue_sysfs_entry queue_slice_sync_entry = { .attr = {.name ="slice_sync" , .mode =S_IRUGO |S_IWUSR }, .show = elv_slice_sync_show,
@@ -349,6 +355,7 @@ static struct attribute *default_attrs[] = { &queue_iostats_entry.attr, #ifdef CONFIG_ELV_FAIR_QUEUING &queue_slice_idle_entry.attr, + &queue_async_slice_idle_entry.attr, &queue_slice_sync_entry.attr, &queue_slice_async_entry.attr, &queue_fairness_entry.attr,
block/elevator-fq.c +63 -4
109 lines not on this page · raw message
block/elevator-fq.h +7 -0
@@ -339,6 +339,8 @@ struct elv_fq_data {* users of this functionality. */ unsigned int elv_slice_idle; +/* idle slice for async queue */ +unsigned int elv_async_slice_idle;struct timer_list idle_slice_timer;struct work_struct unplug_work;
@@ -665,6 +667,11 @@ extern ssize_t elv_slice_idle_store(struct request_queue *q, const char *name,extern ssize_t elv_slice_sync_show (struct request_queue *q,char *name);extern ssize_t elv_slice_sync_store (struct request_queue *q,const char *name,size_t count); + +extern ssize_t elv_async_slice_idle_show (struct request_queue *q,char *name); +extern ssize_t elv_async_slice_idle_store (struct request_queue *q, +const char *name,size_t count); +extern ssize_t elv_slice_async_show (struct request_queue *q,char *name);extern ssize_t elv_slice_async_store (struct request_queue *q,const char *name,size_t count);
Signature · 1 line
1.6.0.1