diff options
| -rw-r--r-- | block/blk-mq-tag.c | 41 |
1 files changed, 32 insertions, 9 deletions
diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 05d8bc76d025..b37125e2c7dc 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -340,6 +340,7 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag) { const int index = TAG_TO_INDEX(bt, tag); struct bt_wait_state *bs; + unsigned int wake_batch; int wait_cnt; clear_bit(TAG_TO_BIT(bt, tag), &bt->map[index].word); @@ -352,10 +353,22 @@ static void bt_clear_tag(struct blk_mq_bitmap_tags *bt, unsigned int tag) return; wait_cnt = atomic_dec_return(&bs->wait_cnt); - if (unlikely(wait_cnt < 0)) - wait_cnt = atomic_inc_return(&bs->wait_cnt); - if (wait_cnt == 0) { - atomic_add(bt->wake_cnt, &bs->wait_cnt); + if (wait_cnt <= 0) { + wake_batch = ACCESS_ONCE(bt->wake_cnt); + /* + * Pairs with the memory barrier in bt_update_count() to + * ensure that we see the batch size update before the wait + * count is reset. + */ + smp_mb__before_atomic(); + /* + * If there are concurrent callers to bt_clear_tag(), the last + * one to decrement the wait count below zero will bump it back + * up. If there is a concurrent resize, the count reset will + * either cause the cmpxchg to fail or overwrite after the + * cmpxchg. + */ + atomic_cmpxchg(&bs->wait_cnt, wait_cnt, wait_cnt + wake_batch); bt_index_atomic_inc(&bt->wake_index); wake_up(&bs->wait); } @@ -450,20 +463,30 @@ static void bt_update_count(struct blk_mq_bitmap_tags *bt, { unsigned int tags_per_word = 1U << bt->bits_per_word; unsigned int map_depth = depth; + unsigned int wake_batch; + int i; if (depth) { - int i; - for (i = 0; i < bt->map_nr; i++) { bt->map[i].depth = min(map_depth, tags_per_word); map_depth -= bt->map[i].depth; } } - bt->wake_cnt = BT_WAIT_BATCH; - if (bt->wake_cnt > depth / BT_WAIT_QUEUES) - bt->wake_cnt = max(1U, depth / BT_WAIT_QUEUES); + wake_batch = BT_WAIT_BATCH; + if (wake_batch > depth / BT_WAIT_QUEUES) + wake_batch = max(1U, depth / BT_WAIT_QUEUES); + if (bt->wake_cnt != wake_batch) { + ACCESS_ONCE(bt->wake_cnt) = wake_batch; + /* + * Pairs with the memory barrier in bt_clear_tag() to ensure + * that the batch size is updated before the wait counts. + */ + smp_mb__before_atomic(); + for (i = 0; i < BT_WAIT_QUEUES; i++) + atomic_set(&bt->bs[i].wait_cnt, 1); + } bt->depth = depth; } |