diff options
| author | Pavel Begunkov <asml.silence@gmail.com> | 2021-11-10 15:49:33 +0000 |
|---|---|---|
| committer | Jens Axboe <axboe@kernel.dk> | 2021-11-24 11:17:53 -0700 |
| commit | 3d4aeb9f98058c3bdfef5286e240cf18c50fee89 (patch) | |
| tree | 247e0136427a62faff8447d9b477c20a0839e391 /fs/io_uring.c | |
| parent | 04c76b41ca974b508522831441dd7e5b1b59cbb0 (diff) | |
| download | linux-stable-3d4aeb9f98058c3bdfef5286e240cf18c50fee89.tar.gz linux-stable-3d4aeb9f98058c3bdfef5286e240cf18c50fee89.tar.bz2 linux-stable-3d4aeb9f98058c3bdfef5286e240cf18c50fee89.zip | |
io_uring: don't spinlock when not posting CQEs
When no of queued for the batch completion requests need to post an CQE,
see IOSQE_CQE_SKIP_SUCCESS, avoid grabbing ->completion_lock and other
commit/post.
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/8d4b4a08bca022cbe19af00266407116775b3e4d.1636559119.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'fs/io_uring.c')
| -rw-r--r-- | fs/io_uring.c | 26 |
1 files changed, 17 insertions, 9 deletions
diff --git a/fs/io_uring.c b/fs/io_uring.c index 7d3589e3a277..f01263a31ea4 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -321,6 +321,7 @@ struct io_submit_state { bool plug_started; bool need_plug; + bool flush_cqes; unsigned short submit_nr; struct blk_plug plug; }; @@ -1525,8 +1526,11 @@ static void io_prep_async_link(struct io_kiocb *req) static inline void io_req_add_compl_list(struct io_kiocb *req) { + struct io_ring_ctx *ctx = req->ctx; struct io_submit_state *state = &req->ctx->submit_state; + if (!(req->flags & REQ_F_CQE_SKIP)) + ctx->submit_state.flush_cqes = true; wq_list_add_tail(&req->comp_list, &state->compl_reqs); } @@ -2386,18 +2390,22 @@ static void __io_submit_flush_completions(struct io_ring_ctx *ctx) struct io_wq_work_node *node, *prev; struct io_submit_state *state = &ctx->submit_state; - spin_lock(&ctx->completion_lock); - wq_list_for_each(node, prev, &state->compl_reqs) { - struct io_kiocb *req = container_of(node, struct io_kiocb, + if (state->flush_cqes) { + spin_lock(&ctx->completion_lock); + wq_list_for_each(node, prev, &state->compl_reqs) { + struct io_kiocb *req = container_of(node, struct io_kiocb, comp_list); - if (!(req->flags & REQ_F_CQE_SKIP)) - __io_fill_cqe(ctx, req->user_data, req->result, - req->cflags); + if (!(req->flags & REQ_F_CQE_SKIP)) + __io_fill_cqe(ctx, req->user_data, req->result, + req->cflags); + } + + io_commit_cqring(ctx); + spin_unlock(&ctx->completion_lock); + io_cqring_ev_posted(ctx); + state->flush_cqes = false; } - io_commit_cqring(ctx); - spin_unlock(&ctx->completion_lock); - io_cqring_ev_posted(ctx); io_free_batch_list(ctx, state->compl_reqs.first); INIT_WQ_LIST(&state->compl_reqs); |