diff options
Diffstat (limited to 'block')
| -rw-r--r-- | block/as-iosched.c | 34 | ||||
| -rw-r--r-- | block/cfq-iosched.c | 425 | ||||
| -rw-r--r-- | block/ll_rw_blk.c | 407 |
3 files changed, 524 insertions, 342 deletions
diff --git a/block/as-iosched.c b/block/as-iosched.c index cb5e53b05c7c..b201d16a7102 100644 --- a/block/as-iosched.c +++ b/block/as-iosched.c @@ -170,9 +170,11 @@ static void free_as_io_context(struct as_io_context *aic) static void as_trim(struct io_context *ioc) { + spin_lock(&ioc->lock); if (ioc->aic) free_as_io_context(ioc->aic); ioc->aic = NULL; + spin_unlock(&ioc->lock); } /* Called when the task exits */ @@ -462,7 +464,9 @@ static void as_antic_timeout(unsigned long data) spin_lock_irqsave(q->queue_lock, flags); if (ad->antic_status == ANTIC_WAIT_REQ || ad->antic_status == ANTIC_WAIT_NEXT) { - struct as_io_context *aic = ad->io_context->aic; + struct as_io_context *aic; + spin_lock(&ad->io_context->lock); + aic = ad->io_context->aic; ad->antic_status = ANTIC_FINISHED; kblockd_schedule_work(&ad->antic_work); @@ -475,6 +479,7 @@ static void as_antic_timeout(unsigned long data) /* process not "saved" by a cooperating request */ ad->exit_no_coop = (7*ad->exit_no_coop + 256)/8; } + spin_unlock(&ad->io_context->lock); } spin_unlock_irqrestore(q->queue_lock, flags); } @@ -635,9 +640,11 @@ static int as_can_break_anticipation(struct as_data *ad, struct request *rq) ioc = ad->io_context; BUG_ON(!ioc); + spin_lock(&ioc->lock); if (rq && ioc == RQ_IOC(rq)) { /* request from same process */ + spin_unlock(&ioc->lock); return 1; } @@ -646,20 +653,25 @@ static int as_can_break_anticipation(struct as_data *ad, struct request *rq) * In this situation status should really be FINISHED, * however the timer hasn't had the chance to run yet. */ + spin_unlock(&ioc->lock); return 1; } aic = ioc->aic; - if (!aic) + if (!aic) { + spin_unlock(&ioc->lock); return 0; + } if (atomic_read(&aic->nr_queued) > 0) { /* process has more requests queued */ + spin_unlock(&ioc->lock); return 1; } if (atomic_read(&aic->nr_dispatched) > 0) { /* process has more requests dispatched */ + spin_unlock(&ioc->lock); return 1; } @@ -680,6 +692,7 @@ static int as_can_break_anticipation(struct as_data *ad, struct request *rq) } as_update_iohist(ad, aic, rq); + spin_unlock(&ioc->lock); return 1; } @@ -688,20 +701,27 @@ static int as_can_break_anticipation(struct as_data *ad, struct request *rq) if (aic->ttime_samples == 0) ad->exit_prob = (7*ad->exit_prob + 256)/8; - if (ad->exit_no_coop > 128) + if (ad->exit_no_coop > 128) { + spin_unlock(&ioc->lock); return 1; + } } if (aic->ttime_samples == 0) { - if (ad->new_ttime_mean > ad->antic_expire) + if (ad->new_ttime_mean > ad->antic_expire) { + spin_unlock(&ioc->lock); return 1; - if (ad->exit_prob * ad->exit_no_coop > 128*256) + } + if (ad->exit_prob * ad->exit_no_coop > 128*256) { + spin_unlock(&ioc->lock); return 1; + } } else if (aic->ttime_mean > ad->antic_expire) { /* the process thinks too much between requests */ + spin_unlock(&ioc->lock); return 1; } - + spin_unlock(&ioc->lock); return 0; } @@ -1255,7 +1275,9 @@ static void as_merged_requests(struct request_queue *q, struct request *req, * Don't copy here but swap, because when anext is * removed below, it must contain the unused context */ + double_spin_lock(&rioc->lock, &nioc->lock, rioc < nioc); swap_io_context(&rioc, &nioc); + double_spin_unlock(&rioc->lock, &nioc->lock, rioc < nioc); } } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 13553e015d72..f28d1fb30608 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -26,9 +26,9 @@ static const int cfq_slice_async_rq = 2; static int cfq_slice_idle = HZ / 125; /* - * grace period before allowing idle class to get disk access + * offset from end of service tree */ -#define CFQ_IDLE_GRACE (HZ / 10) +#define CFQ_IDLE_DELAY (HZ / 5) /* * below this threshold, we consider thinktime immediate @@ -98,8 +98,6 @@ struct cfq_data { struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR]; struct cfq_queue *async_idle_cfqq; - struct timer_list idle_class_timer; - sector_t last_position; unsigned long last_end_request; @@ -199,8 +197,8 @@ CFQ_CFQQ_FNS(sync); static void cfq_dispatch_insert(struct request_queue *, struct request *); static struct cfq_queue *cfq_get_queue(struct cfq_data *, int, - struct task_struct *, gfp_t); -static struct cfq_io_context *cfq_cic_rb_lookup(struct cfq_data *, + struct io_context *, gfp_t); +static struct cfq_io_context *cfq_cic_lookup(struct cfq_data *, struct io_context *); static inline struct cfq_queue *cic_to_cfqq(struct cfq_io_context *cic, @@ -384,12 +382,15 @@ cfq_choose_req(struct cfq_data *cfqd, struct request *rq1, struct request *rq2) /* * The below is leftmost cache rbtree addon */ -static struct rb_node *cfq_rb_first(struct cfq_rb_root *root) +static struct cfq_queue *cfq_rb_first(struct cfq_rb_root *root) { if (!root->left) root->left = rb_first(&root->rb); - return root->left; + if (root->left) + return rb_entry(root->left, struct cfq_queue, rb_node); + + return NULL; } static void cfq_rb_erase(struct rb_node *n, struct cfq_rb_root *root) @@ -446,12 +447,20 @@ static unsigned long cfq_slice_offset(struct cfq_data *cfqd, static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, int add_front) { - struct rb_node **p = &cfqd->service_tree.rb.rb_node; - struct rb_node *parent = NULL; + struct rb_node **p, *parent; + struct cfq_queue *__cfqq; unsigned long rb_key; int left; - if (!add_front) { + if (cfq_class_idle(cfqq)) { + rb_key = CFQ_IDLE_DELAY; + parent = rb_last(&cfqd->service_tree.rb); + if (parent && parent != &cfqq->rb_node) { + __cfqq = rb_entry(parent, struct cfq_queue, rb_node); + rb_key += __cfqq->rb_key; + } else + rb_key += jiffies; + } else if (!add_front) { rb_key = cfq_slice_offset(cfqd, cfqq) + jiffies; rb_key += cfqq->slice_resid; cfqq->slice_resid = 0; @@ -469,8 +478,9 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, } left = 1; + parent = NULL; + p = &cfqd->service_tree.rb.rb_node; while (*p) { - struct cfq_queue *__cfqq; struct rb_node **n; parent = *p; @@ -524,8 +534,7 @@ static void cfq_resort_rr_list(struct cfq_data *cfqd, struct cfq_queue *cfqq) * add to busy list of queues for service, trying to be fair in ordering * the pending list according to last request service */ -static inline void -cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) +static void cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) { BUG_ON(cfq_cfqq_on_rr(cfqq)); cfq_mark_cfqq_on_rr(cfqq); @@ -538,8 +547,7 @@ cfq_add_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) * Called when the cfqq no longer has requests pending, remove it from * the service tree. */ -static inline void -cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) +static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) { BUG_ON(!cfq_cfqq_on_rr(cfqq)); cfq_clear_cfqq_on_rr(cfqq); @@ -554,7 +562,7 @@ cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) /* * rb tree support functions */ -static inline void cfq_del_rq_rb(struct request *rq) +static void cfq_del_rq_rb(struct request *rq) { struct cfq_queue *cfqq = RQ_CFQQ(rq); struct cfq_data *cfqd = cfqq->cfqd; @@ -594,8 +602,7 @@ static void cfq_add_rq_rb(struct request *rq) BUG_ON(!cfqq->next_rq); } -static inline void -cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) +static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq) { elv_rb_del(&cfqq->sort_list, rq); cfqq->queued[rq_is_sync(rq)]--; @@ -609,7 +616,7 @@ cfq_find_rq_fmerge(struct cfq_data *cfqd, struct bio *bio) struct cfq_io_context *cic; struct cfq_queue *cfqq; - cic = cfq_cic_rb_lookup(cfqd, tsk->io_context); + cic = cfq_cic_lookup(cfqd, tsk->io_context); if (!cic) return NULL; @@ -721,7 +728,7 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq, * Lookup the cfqq that this bio will be queued with. Allow * merge only if rq is queued there. */ - cic = cfq_cic_rb_lookup(cfqd, current->io_context); + cic = cfq_cic_lookup(cfqd, current->io_context); if (!cic) return 0; @@ -732,15 +739,10 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq, return 0; } -static inline void -__cfq_set_active_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq) +static void __cfq_set_active_queue(struct cfq_data *cfqd, + struct cfq_queue *cfqq) { if (cfqq) { - /* - * stop potential idle class queues waiting service - */ - del_timer(&cfqd->idle_class_timer); - cfqq->slice_end = 0; cfq_clear_cfqq_must_alloc_slice(cfqq); cfq_clear_cfqq_fifo_expire(cfqq); @@ -789,47 +791,16 @@ static inline void cfq_slice_expired(struct cfq_data *cfqd, int timed_out) __cfq_slice_expired(cfqd, cfqq, timed_out); } -static int start_idle_class_timer(struct cfq_data *cfqd) -{ - unsigned long end = cfqd->last_end_request + CFQ_IDLE_GRACE; - unsigned long now = jiffies; - - if (time_before(now, end) && - time_after_eq(now, cfqd->last_end_request)) { - mod_timer(&cfqd->idle_class_timer, end); - return 1; - } - - return 0; -} - /* * Get next queue for service. Unless we have a queue preemption, * we'll simply select the first cfqq in the service tree. */ static struct cfq_queue *cfq_get_next_queue(struct cfq_data *cfqd) { - struct cfq_queue *cfqq; - struct rb_node *n; - if (RB_EMPTY_ROOT(&cfqd->service_tree.rb)) return NULL; - n = cfq_rb_first(&cfqd->service_tree); - cfqq = rb_entry(n, struct cfq_queue, rb_node); - - if (cfq_class_idle(cfqq)) { - /* - * if we have idle queues and no rt or be queues had - * pending requests, either allow immediate service if - * the grace period has passed or arm the idle grace - * timer - */ - if (start_idle_class_timer(cfqd)) - cfqq = NULL; - } - - return cfqq; + return cfq_rb_first(&cfqd->service_tree); } /* @@ -895,7 +866,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) * task has exited, don't wait */ cic = cfqd->active_cic; - if (!cic || !cic->ioc->task) + if (!cic || !atomic_read(&cic->ioc->nr_tasks)) return; /* @@ -939,7 +910,7 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq) /* * return expired entry, or NULL to just start from scratch in rbtree */ -static inline struct request *cfq_check_fifo(struct cfq_queue *cfqq) +static struct request *cfq_check_fifo(struct cfq_queue *cfqq) { struct cfq_data *cfqd = cfqq->cfqd; struct request *rq; @@ -1068,7 +1039,7 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq, return dispatched; } -static inline int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq) +static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq) { int dispatched = 0; @@ -1087,14 +1058,11 @@ static inline int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq) */ static int cfq_forced_dispatch(struct cfq_data *cfqd) { + struct cfq_queue *cfqq; int dispatched = 0; - struct rb_node *n; - - while ((n = cfq_rb_first(&cfqd->service_tree)) != NULL) { - struct cfq_queue *cfqq = rb_entry(n, struct cfq_queue, rb_node); + while ((cfqq = cfq_rb_first(&cfqd->service_tree)) != NULL) dispatched += __cfq_forced_dispatch_cfqq(cfqq); - } cfq_slice_expired(cfqd, 0); @@ -1170,20 +1138,69 @@ static void cfq_put_queue(struct cfq_queue *cfqq) kmem_cache_free(cfq_pool, cfqq); } -static void cfq_free_io_context(struct io_context *ioc) +/* + * Call func for each cic attached to this ioc. Returns number of cic's seen. + */ +#define CIC_GANG_NR 16 +static unsigned int +call_for_each_cic(struct io_context *ioc, + void (*func)(struct io_context *, struct cfq_io_context *)) { - struct cfq_io_context *__cic; - struct rb_node *n; - int freed = 0; + struct cfq_io_context *cics[CIC_GANG_NR]; + unsigned long index = 0; + unsigned int called = 0; + int nr; - ioc->ioc_data = NULL; + rcu_read_lock(); - while ((n = rb_first(&ioc->cic_root)) != NULL) { - __cic = rb_entry(n, struct cfq_io_context, rb_node); - rb_erase(&__cic->rb_node, &ioc->cic_root); - kmem_cache_free(cfq_ioc_pool, __cic); - freed++; - } + do { + int i; + + /* + * Perhaps there's a better way - this just gang lookups from + * 0 to the end, restarting after each CIC_GANG_NR from the + * last key + 1. + */ + nr = radix_tree_gang_lookup(&ioc->radix_root, (void **) cics, + index, CIC_GANG_NR); + if (!nr) + break; + + called += nr; + index = 1 + (unsigned long) cics[nr - 1]->key; + + for (i = 0; i < nr; i++) + func(ioc, cics[i]); + } while (nr == CIC_GANG_NR); + + rcu_read_unlock(); + + return called; +} + +static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic) +{ + unsigned long flags; + + BUG_ON(!cic->dead_key); + + spin_lock_irqsave(&ioc->lock, flags); + radix_tree_delete(&ioc->radix_root, cic->dead_key); + spin_unlock_irqrestore(&ioc->lock, flags); + + kmem_cache_free(cfq_ioc_pool, cic); +} + +static void cfq_free_io_context(struct io_context *ioc) +{ + int freed; + + /* + * ioc->refcount is zero here, so no more cic's are allowed to be + * linked into this ioc. So it should be ok to iterate over the known + * list, we will see all cic's since no new ones are added. + */ + freed = call_for_each_cic(ioc, cic_free_func); elv_ioc_count_mod(ioc_count, -freed); @@ -1205,7 +1222,12 @@ static void __cfq_exit_single_io_context(struct cfq_data *cfqd, struct cfq_io_context *cic) { list_del_init(&cic->queue_list); + + /* + * Make sure key == NULL is seen for dead queues + */ smp_wmb(); + cic->dead_key = (unsigned long) cic->key; cic->key = NULL; if (cic->cfqq[ASYNC]) { @@ -1219,16 +1241,18 @@ static void __cfq_exit_single_io_context(struct cfq_data *cfqd, } } -static void cfq_exit_single_io_context(struct cfq_io_context *cic) +static void cfq_exit_single_io_context(struct io_context *ioc, + struct cfq_io_context *cic) { struct cfq_data *cfqd = cic->key; if (cfqd) { struct request_queue *q = cfqd->queue; + unsigned long flags; - spin_lock_irq(q->queue_lock); + spin_lock_irqsave(q->queue_lock, flags); __cfq_exit_single_io_context(cfqd, cic); - spin_unlock_irq(q->queue_lock); + spin_unlock_irqrestore(q->queue_lock, flags); } } @@ -1238,21 +1262,8 @@ static void cfq_exit_single_io_context(struct cfq_io_context *cic) */ static void cfq_exit_io_context(struct io_context *ioc) { - struct cfq_io_context *__cic; - struct rb_node *n; - - ioc->ioc_data = NULL; - - /* - * put the reference this task is holding to the various queues - */ - n = rb_first(&ioc->cic_root); - while (n != NULL) { - __cic = rb_entry(n, struct cfq_io_context, rb_node); - - cfq_exit_single_io_context(__cic); - n = rb_next(n); - } + rcu_assign_pointer(ioc->ioc_data, NULL); + call_for_each_cic(ioc, cfq_exit_single_io_context); } static struct cfq_io_context * @@ -1273,7 +1284,7 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) return cic; } -static void cfq_init_prio_data(struct cfq_queue *cfqq) +static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc) { struct task_struct *tsk = current; int ioprio_class; @@ -1281,7 +1292,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq) if (!cfq_cfqq_prio_changed(cfqq)) return; - ioprio_class = IOPRIO_PRIO_CLASS(tsk->ioprio); + ioprio_class = IOPRIO_PRIO_CLASS(ioc->ioprio); switch (ioprio_class) { default: printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class); @@ -1293,11 +1304,11 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq) cfqq->ioprio_class = IOPRIO_CLASS_BE; break; case IOPRIO_CLASS_RT: - cfqq->ioprio = task_ioprio(tsk); + cfqq->ioprio = task_ioprio(ioc); cfqq->ioprio_class = IOPRIO_CLASS_RT; break; case IOPRIO_CLASS_BE: - cfqq->ioprio = task_ioprio(tsk); + cfqq->ioprio = task_ioprio(ioc); cfqq->ioprio_class = IOPRIO_CLASS_BE; break; case IOPRIO_CLASS_IDLE: @@ -1316,7 +1327,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq) cfq_clear_cfqq_prio_changed(cfqq); } -static inline void changed_ioprio(struct cfq_io_context *cic) +static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic) { struct cfq_data *cfqd = cic->key; struct cfq_queue *cfqq; @@ -1330,8 +1341,7 @@ static inline void changed_ioprio(struct cfq_io_context *cic) cfqq = cic->cfqq[ASYNC]; if (cfqq) { struct cfq_queue *new_cfqq; - new_cfqq = cfq_get_queue(cfqd, ASYNC, cic->ioc->task, - GFP_ATOMIC); + new_cfqq = cfq_get_queue(cfqd, ASYNC, cic->ioc, GFP_ATOMIC); if (new_cfqq) { cic->cfqq[ASYNC] = new_cfqq; cfq_put_queue(cfqq); @@ -1347,29 +1357,19 @@ static inline void changed_ioprio(struct cfq_io_context *cic) static void cfq_ioc_set_ioprio(struct io_context *ioc) { - struct cfq_io_context *cic; - struct rb_node *n; - + call_for_each_cic(ioc, changed_ioprio); ioc->ioprio_changed = 0; - - n = rb_first(&ioc->cic_root); - while (n != NULL) { - cic = rb_entry(n, struct cfq_io_context, rb_node); - - changed_ioprio(cic); - n = rb_next(n); - } } static struct cfq_queue * cfq_find_alloc_queue(struct cfq_data *cfqd, int is_sync, - struct task_struct *tsk, gfp_t gfp_mask) + struct io_context *ioc, gfp_t gfp_mask) { struct cfq_queue *cfqq, *new_cfqq = NULL; struct cfq_io_context *cic; retry: - cic = cfq_cic_rb_lookup(cfqd, tsk->io_context); + cic = cfq_cic_lookup(cfqd, ioc); /* cic always exists here */ cfqq = cic_to_cfqq(cic, is_sync); @@ -1404,15 +1404,16 @@ retry: atomic_set(&cfqq->ref, 0); cfqq->cfqd = cfqd; - if (is_sync) { - cfq_mark_cfqq_idle_window(cfqq); - cfq_mark_cfqq_sync(cfqq); - } - cfq_mark_cfqq_prio_changed(cfqq); cfq_mark_cfqq_queue_new(cfqq); - cfq_init_prio_data(cfqq); + cfq_init_prio_data(cfqq, ioc); + + if (is_sync) { + if (!cfq_class_idle(cfqq)) + cfq_mark_cfqq_idle_window(cfqq); + cfq_mark_cfqq_sync(cfqq); + } } if (new_cfqq) @@ -1439,11 +1440,11 @@ cfq_async_queue_prio(struct cfq_data *cfqd, int ioprio_class, int ioprio) } static struct cfq_queue * -cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct task_struct *tsk, +cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct io_context *ioc, gfp_t gfp_mask) { - const int ioprio = task_ioprio(tsk); - const int ioprio_class = task_ioprio_class(tsk); + const int ioprio = task_ioprio(ioc); + const int ioprio_class = task_ioprio_class(ioc); struct cfq_queue **async_cfqq = NULL; struct cfq_queue *cfqq = NULL; @@ -1453,7 +1454,7 @@ cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct task_struct *tsk, } if (!cfqq) { - cfqq = cfq_find_alloc_queue(cfqd, is_sync, tsk, gfp_mask); + cfqq = cfq_find_alloc_queue(cfqd, is_sync, ioc, gfp_mask); if (!cfqq) return NULL; } @@ -1470,28 +1471,42 @@ cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct task_struct *tsk, return cfqq; } +static void cfq_cic_free(struct cfq_io_context *cic) +{ + kmem_cache_free(cfq_ioc_pool, cic); + elv_ioc_count_dec(ioc_count); + + if (ioc_gone && !elv_ioc_count_read(ioc_count)) + complete(ioc_gone); +} + /* * We drop cfq io contexts lazily, so we may find a dead one. */ static void -cfq_drop_dead_cic(struct io_context *ioc, struct cfq_io_context *cic) +cfq_drop_dead_cic(struct cfq_data *cfqd, struct io_context *ioc, + struct cfq_io_context *cic) { + unsigned long flags; + WARN_ON(!list_empty(&cic->queue_list)); + spin_lock_irqsave(&ioc->lock, flags); + if (ioc->ioc_data == cic) - ioc->ioc_data = NULL; + rcu_assign_pointer(ioc->ioc_data, NULL); - rb_erase(&cic->rb_node, &ioc->cic_root); - kmem_cache_free(cfq_ioc_pool, cic); - elv_ioc_count_dec(ioc_count); + radix_tree_delete(&ioc->radix_root, (unsigned long) cfqd); + spin_unlock_irqrestore(&ioc->lock, flags); + + cfq_cic_free(cic); } static struct cfq_io_context * -cfq_cic_rb_lookup(struct cfq_data *cfqd, struct io_context *ioc) +cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc) { - struct rb_node *n; struct cfq_io_context *cic; - void *k, *key = cfqd; + void *k; if (unlikely(!ioc)) return NULL; @@ -1499,74 +1514,64 @@ cfq_cic_rb_lookup(struct cfq_data *cfqd, struct io_context *ioc) /* * we maintain a last-hit cache, to avoid browsing over the tree */ - cic = ioc->ioc_data; + cic = rcu_dereference(ioc->ioc_data); if (cic && cic->key == cfqd) return cic; -restart: - n = ioc->cic_root.rb_node; - while (n) { - cic = rb_entry(n, struct cfq_io_context, rb_node); + do { + rcu_read_lock(); + cic = radix_tree_lookup(&ioc->radix_root, (unsigned long) cfqd); + rcu_read_unlock(); + if (!cic) + break; /* ->key must be copied to avoid race with cfq_exit_queue() */ k = cic->key; if (unlikely(!k)) { - cfq_drop_dead_cic(ioc, cic); - goto restart; + cfq_drop_dead_cic(cfqd, ioc, cic); + continue; } - if (key < k) - n = n->rb_left; - else if (key > k) - n = n->rb_right; - else { - ioc->ioc_data = cic; - return cic; - } - } + rcu_assign_pointer(ioc->ioc_data, cic); + break; + } while (1); - return NULL; + return cic; } -static inline void -cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc, - struct cfq_io_context *cic) +/* + * Add cic into ioc, using cfqd as the search key. This enables us to lookup + * the process specific cfq io context when entered from the block layer. + * Also adds the cic to a per-cfqd list, used when this queue is removed. + */ +static int cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc, + struct cfq_io_context *cic, gfp_t gfp_mask) { - struct rb_node **p; - struct rb_node *parent; - struct cfq_io_context *__cic; unsigned long flags; - void *k; + int ret; - cic->ioc = ioc; - cic->key = cfqd; + ret = radix_tree_preload(gfp_mask); + if (!ret) { + cic->ioc = ioc; + cic->key = cfqd; -restart: - parent = NULL; - p = &ioc->cic_root.rb_node; - while (*p) { - parent = *p; - __cic = rb_entry(parent, struct cfq_io_context, rb_node); - /* ->key must be copied to avoid race with cfq_exit_queue() */ - k = __cic->key; - if (unlikely(!k)) { - cfq_drop_dead_cic(ioc, __cic); - goto restart; - } + spin_lock_irqsave(&ioc->lock, flags); + ret = radix_tree_insert(&ioc->radix_root, + (unsigned long) cfqd, cic); + spin_unlock_irqrestore(&ioc->lock, flags); - if (cic->key < k) - p = &(*p)->rb_left; - else if (cic->key > k) - p = &(*p)->rb_right; - else - BUG(); + radix_tree_preload_end(); + + if (!ret) { + spin_lock_irqsave(cfqd->queue->queue_lock, flags); + list_add(&cic->queue_list, &cfqd->cic_list); + spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); + } } - rb_link_node(&cic->rb_node, parent, p); - rb_insert_color(&cic->rb_node, &ioc->cic_root); + if (ret) + printk(KERN_ERR "cfq: cic link failed!\n"); - spin_lock_irqsave(cfqd->queue->queue_lock, flags); - list_add(&cic->queue_list, &cfqd->cic_list); - spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); + return ret; } /* @@ -1586,7 +1591,7 @@ cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) if (!ioc) return NULL; - cic = cfq_cic_rb_lookup(cfqd, ioc); + cic = cfq_cic_lookup(cfqd, ioc); if (cic) goto out; @@ -1594,13 +1599,17 @@ cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) if (cic == NULL) goto err; - cfq_cic_link(cfqd, ioc, cic); + if (cfq_cic_link(cfqd, ioc, cic, gfp_mask)) + goto err_free; + out: smp_read_barrier_depends(); if (unlikely(ioc->ioprio_changed)) cfq_ioc_set_ioprio(ioc); return cic; +err_free: + cfq_cic_free(cic); err: put_io_context(ioc); return NULL; @@ -1655,12 +1664,15 @@ cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq, { int enable_idle; - if (!cfq_cfqq_sync(cfqq)) + /* + * Don't idle for async or idle io prio class + */ + if (!cfq_cfqq_sync(cfqq) || cfq_class_idle(cfqq)) return; enable_idle = cfq_cfqq_idle_window(cfqq); - if (!cic->ioc->task || !cfqd->cfq_slice_idle || + if (!atomic_read(&cic->ioc->nr_tasks) || !cfqd->cfq_slice_idle || (cfqd->hw_tag && CIC_SEEKY(cic))) enable_idle = 0; else if (sample_valid(cic->ttime_samples)) { @@ -1793,7 +1805,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq) struct cfq_data *cfqd = q->elevator->elevator_data; struct cfq_queue *cfqq = RQ_CFQQ(rq); - cfq_init_prio_data(cfqq); + cfq_init_prio_data(cfqq, RQ_CIC(rq)->ioc); cfq_add_rq_rb(rq); @@ -1834,7 +1846,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) cfq_set_prio_slice(cfqd, cfqq); cfq_clear_cfqq_slice_new(cfqq); } - if (cfq_slice_used(cfqq)) + if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq)) cfq_slice_expired(cfqd, 1); else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list)) cfq_arm_slice_timer(cfqd); @@ -1894,13 +1906,13 @@ static int cfq_may_queue(struct request_queue *q, int rw) * so just lookup a possibly existing queue, or return 'may queue' * if that fails */ - cic = cfq_cic_rb_lookup(cfqd, tsk->io_context); + cic = cfq_cic_lookup(cfqd, tsk->io_context); if (!cic) return ELV_MQUEUE_MAY; cfqq = cic_to_cfqq(cic, rw & REQ_RW_SYNC); if (cfqq) { - cfq_init_prio_data(cfqq); + cfq_init_prio_data(cfqq, cic->ioc); cfq_prio_boost(cfqq); return __cfq_may_queue(cfqq); @@ -1938,7 +1950,6 @@ static int cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) { struct cfq_data *cfqd = q->elevator->elevator_data; - struct task_struct *tsk = current; struct cfq_io_context *cic; const int rw = rq_data_dir(rq); const int is_sync = rq_is_sync(rq); @@ -1956,7 +1967,7 @@ cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) cfqq = cic_to_cfqq(cic, is_sync); if (!cfqq) { - cfqq = cfq_get_queue(cfqd, is_sync, tsk, gfp_mask); + cfqq = cfq_get_queue(cfqd, is_sync, cic->ioc, gfp_mask); if (!cfqq) goto queue_fail; @@ -2039,29 +2050,9 @@ out_cont: spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); } -/* - * Timer running if an idle class queue is waiting for service - */ -static void cfq_idle_class_timer(unsigned long data) -{ - struct cfq_data *cfqd = (struct cfq_data *) data; - unsigned long flags; - - spin_lock_irqsave(cfqd->queue->queue_lock, flags); - - /* - * race with a non-idle queue, reset timer - */ - if (!start_idle_class_timer(cfqd)) - cfq_schedule_dispatch(cfqd); - - spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); -} - static void cfq_shutdown_timer_wq(struct cfq_data *cfqd) { del_timer_sync(&cfqd->idle_slice_timer); - del_timer_sync(&cfqd->idle_class_timer); kblockd_flush_work(&cfqd->unplug_work); } @@ -2126,10 +2117,6 @@ static void *cfq_init_queue(struct request_queue *q) cfqd->idle_slice_timer.function = cfq_idle_slice_timer; cfqd->idle_slice_timer.data = (unsigned long) cfqd; - init_timer(&cfqd->idle_class_timer); - cfqd->idle_class_timer.function = cfq_idle_class_timer; - cfqd->idle_class_timer.data = (unsigned long) cfqd; - INIT_WORK(&cfqd->unplug_work, cfq_kick_queue); cfqd->last_end_request = jiffies; @@ -2160,7 +2147,7 @@ static int __init cfq_slab_setup(void) if (!cfq_pool) goto fail; - cfq_ioc_pool = KMEM_CACHE(cfq_io_context, 0); + cfq_ioc_pool = KMEM_CACHE(cfq_io_context, SLAB_DESTROY_BY_RCU); if (!cfq_ioc_pool) goto fail; diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 768987dc2697..1932a56f5e4b 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -347,7 +347,6 @@ unsigned blk_ordered_req_seq(struct request *rq) void blk_ordered_complete_seq(struct request_queue *q, unsigned seq, int error) { struct request *rq; - int uptodate; if (error && !q->orderr) q->orderr = error; @@ -361,15 +360,11 @@ void blk_ordered_complete_seq(struct request_queue *q, unsigned seq, int error) /* * Okay, sequence complete. */ - uptodate = 1; - if (q->orderr) - uptodate = q->orderr; - q->ordseq = 0; rq = q->orig_bar_rq; - end_that_request_first(rq, uptodate, rq->hard_nr_sectors); - end_that_request_last(rq, uptodate); + if (__blk_end_request(rq, q->orderr, blk_rq_bytes(rq))) + BUG(); } static void pre_flush_end_io(struct request *rq, int error) @@ -486,9 +481,9 @@ int blk_do_ordered(struct request_queue *q, struct request **rqp) * ORDERED_NONE while this request is on it. */ blkdev_dequeue_request(rq); - end_that_request_first(rq, -EOPNOTSUPP, - rq->hard_nr_sectors); - end_that_request_last(rq, -EOPNOTSUPP); + if (__blk_end_request(rq, -EOPNOTSUPP, + blk_rq_bytes(rq))) + BUG(); *rqp = NULL; return 0; } @@ -3486,29 +3481,36 @@ static void blk_recalc_rq_sectors(struct request *rq, int nsect) } } -static int __end_that_request_first(struct request *req, int uptodate, +/** + * __end_that_request_first - end I/O on a request + * @req: the request being processed + * @error: 0 for success, < 0 for error + * @nr_bytes: number of bytes to complete + * + * Description: + * Ends I/O on a number of bytes attached to @req, and sets it up + * for the next range of segments (if any) in the cluster. + * + * Return: + * 0 - we are done with this request, call end_that_request_last() + * 1 - still buffers pending for this request + **/ +static int __end_that_request_first(struct request *req, int error, int nr_bytes) { - int total_bytes, bio_nbytes, error, next_idx = 0; + int total_bytes, bio_nbytes, next_idx = 0; struct bio *bio; blk_add_trace_rq(req->q, req, BLK_TA_COMPLETE); /* - * extend uptodate bool to allow < 0 value to be direct io error - */ - error = 0; - if (end_io_error(uptodate)) - error = !uptodate ? -EIO : uptodate; - - /* * for a REQ_BLOCK_PC request, we want to carry any eventual * sense key with us all the way through */ if (!blk_pc_request(req)) req->errors = 0; - if (!uptodate) { + if (error) { if (blk_fs_request(req) && !(req->cmd_flags & REQ_QUIET)) printk("end_request: I/O error, dev %s, sector %llu\n", req->rq_disk ? req->rq_disk->disk_name : "?", @@ -3602,49 +3604,6 @@ static int __end_that_request_first(struct request *req, int uptodate, return 1; } -/** - * end_that_request_first - end I/O on a request - * @req: the request being processed - * @uptodate: 1 for success, 0 for I/O error, < 0 for specific error - * @nr_sectors: number of sectors to end I/O on - * - * Description: - * Ends I/O on a number of sectors attached to @req, and sets it up - * for the next range of segments (if any) in the cluster. - * - * Return: - * 0 - we are done with this request, call end_that_request_last() - * 1 - still buffers pending for this request - **/ -int end_that_request_first(struct request *req, int uptodate, int nr_sectors) -{ - return __end_that_request_first(req, uptodate, nr_sectors << 9); -} - -EXPORT_SYMBOL(end_that_request_first); - -/** - * end_that_request_chunk - end I/O on a request - * @req: the request being processed - * @uptodate: 1 for success, 0 for I/O error, < 0 for specific error - * @nr_bytes: number of bytes to complete - * - * Description: - * Ends I/O on a number of bytes attached to @req, and sets it up - * for the next range of segments (if any). Like end_that_request_first(), - * but deals with bytes instead of sectors. - * - * Return: - * 0 - we are done with this request, call end_that_request_last() - * 1 - still buffers pending for this request - **/ -int end_that_request_chunk(struct request *req, int uptodate, int nr_bytes) -{ - return __end_that_request_first(req, uptodate, nr_bytes); -} - -EXPORT_SYMBOL(end_that_request_chunk); - /* * splice the completion data to a local structure and hand off to * process_completion_queue() to complete the requests @@ -3724,17 +3683,15 @@ EXPORT_SYMBOL(blk_complete_request); /* * queue lock must be held */ -void end_that_request_last(struct request *req, int uptodate) +static void end_that_request_last(struct request *req, int error) { struct gendisk *disk = req->rq_disk; - int error; - /* - * extend uptodate bool to allow < 0 value to be direct io error - */ - error = 0; - if (end_io_error(uptodate)) - error = !uptodate ? -EIO : uptodate; + if (blk_rq_tagged(req)) + blk_queue_end_tag(req->q, req); + + if (blk_queued_rq(req)) + blkdev_dequeue_request(req); if (unlikely(laptop_mode) && blk_fs_request(req)) laptop_io_completion(); @@ -3753,32 +3710,54 @@ void end_that_request_last(struct request *req, int uptodate) disk_round_stats(disk); disk->in_flight--; } + if (req->end_io) req->end_io(req, error); - else + else { + if (blk_bidi_rq(req)) + __blk_put_request(req->next_rq->q, req->next_rq); + __blk_put_request(req->q, req); + } } -EXPORT_SYMBOL(end_that_request_last); - static inline void __end_request(struct request *rq, int uptodate, - unsigned int nr_bytes, int dequeue) + unsigned int nr_bytes) { - if (!end_that_request_chunk(rq, uptodate, nr_bytes)) { - if (dequeue) - blkdev_dequeue_request(rq); - add_disk_randomness(rq->rq_disk); - end_that_request_last(rq, uptodate); - } + int error = 0; + + if (uptodate <= 0) + error = uptodate ? uptodate : -EIO; + + __blk_end_request(rq, error, nr_bytes); } -static unsigned int rq_byte_size(struct request *rq) +/** + * blk_rq_bytes - Returns bytes left to complete in the entire request + **/ +unsigned int blk_rq_bytes(struct request *rq) { if (blk_fs_request(rq)) return rq->hard_nr_sectors << 9; return rq->data_len; } +EXPORT_SYMBOL_GPL(blk_rq_bytes); + +/** + * blk_rq_cur_bytes - Returns bytes left to complete in the current segment + **/ +unsigned int blk_rq_cur_bytes(struct request *rq) +{ + if (blk_fs_request(rq)) + return rq->current_nr_sectors << 9; + + if (rq->bio) + return rq->bio->bi_size; + + return rq->data_len; +} +EXPORT_SYMBOL_GPL(blk_rq_cur_bytes); /** * end_queued_request - end all I/O on a queued request @@ -3793,7 +3772,7 @@ static unsigned int rq_byte_size(struct request *rq) **/ void end_queued_request(struct request *rq, int uptodate) { - __end_request(rq, uptodate, rq_byte_size(rq), 1); + __end_request(rq, uptodate, blk_rq_bytes(rq)); } EXPORT_SYMBOL(end_queued_request); @@ -3810,7 +3789,7 @@ EXPORT_SYMBOL(end_queued_request); **/ void end_dequeued_request(struct request *rq, int uptodate) { - __end_request(rq, uptodate, rq_byte_size(rq), 0); + __end_request(rq, uptodate, blk_rq_bytes(rq)); } EXPORT_SYMBOL(end_dequeued_request); @@ -3836,10 +3815,159 @@ EXPORT_SYMBOL(end_dequeued_request); **/ void end_request(struct request *req, int uptodate) { - __end_request(req, uptodate, req->hard_cur_sectors << 9, 1); + __end_request(req, uptodate, req->hard_cur_sectors << 9); } EXPORT_SYMBOL(end_request); +/** + * blk_end_io - Generic end_io function to complete a request. + * @rq: the request being processed + * @error: 0 for success, < 0 for error + * @nr_bytes: number of bytes to complete @rq + * @bidi_bytes: number of bytes to complete @rq->next_rq + * @drv_callback: function called between completion of bios in the request + * and completion of the request. + * If the callback returns non 0, this helper returns without + * completion of the request. + * + * Description: + * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. + * If @rq has leftover, sets it up for the next range of segments. + * + * Return: + * 0 - we are done with this request + * 1 - this request is not freed yet, it still has pending buffers. + **/ +static int blk_end_io(struct request *rq, int error, int nr_bytes, + int bidi_bytes, int (drv_callback)(struct request *)) +{ + struct request_queue *q = rq->q; + unsigned long flags = 0UL; + + if (blk_fs_request(rq) || blk_pc_request(rq)) { + if (__end_that_request_first(rq, error, nr_bytes)) + return 1; + + /* Bidi request must be completed as a whole */ + if (blk_bidi_rq(rq) && + __end_that_request_first(rq->next_rq, error, bidi_bytes)) + return 1; + } + + /* Special feature for tricky drivers */ + if (drv_callback && drv_callback(rq)) + return 1; + + add_disk_randomness(rq->rq_disk); + + spin_lock_irqsave(q->queue_lock, flags); + end_that_request_last(rq, error); + spin_unlock_irqrestore(q->queue_lock, flags); + + return 0; +} + +/** + * blk_end_request - Helper function for drivers to complete the request. + * @rq: the request being processed + * @error: 0 for success, < 0 for error + * @nr_bytes: number of bytes to complete + * + * Description: + * Ends I/O on a number of bytes attached to @rq. + * If @rq has leftover, sets it up for the next range of segments. + * + * Return: + * 0 - we are done with this request + * 1 - still buffers pending for this request + **/ +int blk_end_request(struct request *rq, int error, int nr_bytes) +{ + return blk_end_io(rq, error, nr_bytes, 0, NULL); +} +EXPORT_SYMBOL_GPL(blk_end_request); + +/** + * __blk_end_request - Helper function for drivers to complete the request. + * @rq: the request being processed + * @error: 0 for success, < 0 for error + * @nr_bytes: number of bytes to complete + * + * Description: + * Must be called with queue lock held unlike blk_end_request(). + * + * Return: + * 0 - we are done with this request + * 1 - still buffers pending for this request + **/ +int __blk_end_request(struct request *rq, int error, int nr_bytes) +{ + if (blk_fs_request(rq) || blk_pc_request(rq)) { + if (__end_that_request_first(rq, error, nr_bytes)) + return 1; + } + + add_disk_randomness(rq->rq_disk); + + end_that_request_last(rq, error); + + return 0; +} +EXPORT_SYMBOL_GPL(__blk_end_request); + +/** + * blk_end_bidi_request - Helper function for drivers to complete bidi request. + * @rq: the bidi request being processed + * @error: 0 for success, < 0 for error + * @nr_bytes: number of bytes to complete @rq + * @bidi_bytes: number of bytes to complete @rq->next_rq + * + * Description: + * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. + * + * Return: + * 0 - we are done with this request + * 1 - still buffers pending for this request + **/ +int blk_end_bidi_request(struct request *rq, int error, int nr_bytes, + int bidi_bytes) +{ + return blk_end_io(rq, error, nr_bytes, bidi_bytes, NULL); +} +EXPORT_SYMBOL_GPL(blk_end_bidi_request); + +/** + * blk_end_request_callback - Special helper function for tricky drivers + * @rq: the request being processed + * @error: 0 for success, < 0 for error + * @nr_bytes: number of bytes to complete + * @drv_callback: function called between completion of bios in the request + * and completion of the request. + * If the callback returns non 0, this helper returns without + * completion of the request. + * + * Description: + * Ends I/O on a number of bytes attached to @rq. + * If @rq has leftover, sets it up for the next range of segments. + * + * This special helper function is used only for existing tricky drivers. + * (e.g. cdrom_newpc_intr() of ide-cd) + * This interface will be removed when such drivers are rewritten. + * Don't use this interface in other places anymore. + * + * Return: + * 0 - we are done with this request + * 1 - this request is not freed yet. + * this request still has pending buffers or + * the driver doesn't want to finish this request yet. + **/ +int blk_end_request_callback(struct request *rq, int error, int nr_bytes, + int (drv_callback)(struct request *)) +{ + return blk_end_io(rq, error, nr_bytes, 0, drv_callback); +} +EXPORT_SYMBOL_GPL(blk_end_request_callback); + static void blk_rq_bio_prep(struct request_queue *q, struct request *rq, struct bio *bio) { @@ -3902,55 +4030,100 @@ int __init blk_dev_init(void) return 0; } +static void cfq_dtor(struct io_context *ioc) +{ + struct cfq_io_context *cic[1]; + int r; + + /* + * We don't have a specific key to lookup with, so use the gang + * lookup to just retrieve the first item stored. The cfq exit + * function will iterate the full tree, so any member will do. + */ + r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); + if (r > 0) + cic[0]->dtor(ioc); +} + /* - * IO Context helper functions + * IO Context helper functions. put_io_context() returns 1 if there are no + * more users of this io context, 0 otherwise. */ -void put_io_context(struct io_context *ioc) +int put_io_context(struct io_context *ioc) { if (ioc == NULL) - return; + return 1; BUG_ON(atomic_read(&ioc->refcount) == 0); if (atomic_dec_and_test(&ioc->refcount)) { - struct cfq_io_context *cic; - rcu_read_lock(); if (ioc->aic && ioc->aic->dtor) ioc->aic->dtor(ioc->aic); - if (ioc->cic_root.rb_node != NULL) { - struct rb_node *n = rb_first(&ioc->cic_root); - - cic = rb_entry(n, struct cfq_io_context, rb_node); - cic->dtor(ioc); - } rcu_read_unlock(); + cfq_dtor(ioc); kmem_cache_free(iocontext_cachep, ioc); + return 1; } + return 0; } EXPORT_SYMBOL(put_io_context); +static void cfq_exit(struct io_context *ioc) +{ + struct cfq_io_context *cic[1]; + int r; + + rcu_read_lock(); + /* + * See comment for cfq_dtor() + */ + r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); + rcu_read_unlock(); + + if (r > 0) + cic[0]->exit(ioc); +} + /* Called by the exitting task */ void exit_io_context(void) { struct io_context *ioc; - struct cfq_io_context *cic; task_lock(current); ioc = current->io_context; current->io_context = NULL; task_unlock(current); - ioc->task = NULL; - if (ioc->aic && ioc->aic->exit) - ioc->aic->exit(ioc->aic); - if (ioc->cic_root.rb_node != NULL) { - cic = rb_entry(rb_first(&ioc->cic_root), struct cfq_io_context, rb_node); - cic->exit(ioc); + if (atomic_dec_and_test(&ioc->nr_tasks)) { + if (ioc->aic && ioc->aic->exit) + ioc->aic->exit(ioc->aic); + cfq_exit(ioc); + + put_io_context(ioc); + } +} + +struct io_context *alloc_io_context(gfp_t gfp_flags, int node) +{ + struct io_context *ret; + + ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); + if (ret) { + atomic_set(&ret->refcount, 1); + atomic_set(&ret->nr_tasks, 1); + spin_lock_init(&ret->lock); + ret->ioprio_changed = 0; + ret->ioprio = 0; + ret->last_waited = jiffies; /* doesn't matter... */ + ret->nr_batch_requests = 0; /* because this is 0 */ + ret->aic = NULL; + INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH); + ret->ioc_data = NULL; } - put_io_context(ioc); + return ret; } /* @@ -3970,16 +4143,8 @@ static struct io_context *current_io_context(gfp_t gfp_flags, int node) if (likely(ret)) return ret; - ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node); + ret = alloc_io_context(gfp_flags, node); if (ret) { - atomic_set(&ret->refcount, 1); - ret->task = current; - ret->ioprio_changed = 0; - ret->last_waited = jiffies; /* doesn't matter... */ - ret->nr_batch_requests = 0; /* because this is 0 */ - ret->aic = NULL; - ret->cic_root.rb_node = NULL; - ret->ioc_data = NULL; /* make sure set_task_ioprio() sees the settings above */ smp_wmb(); tsk->io_context = ret; @@ -3996,10 +4161,18 @@ static struct io_context *current_io_context(gfp_t gfp_flags, int node) */ struct io_context *get_io_context(gfp_t gfp_flags, int node) { - struct io_context *ret; - ret = current_io_context(gfp_flags, node); - if (likely(ret)) - atomic_inc(&ret->refcount); + struct io_context *ret = NULL; + + /* + * Check for unlikely race with exiting task. ioc ref count is + * zero when ioc is being detached. + */ + do { + ret = current_io_context(gfp_flags, node); + if (unlikely(!ret)) + break; + } while (!atomic_inc_not_zero(&ret->refcount)); + return ret; } EXPORT_SYMBOL(get_io_context); |