diff options
| -rw-r--r-- | io_uring/io_uring.c | 21 | ||||
| -rw-r--r-- | io_uring/io_uring.h | 10 | ||||
| -rw-r--r-- | io_uring/kbuf.c | 4 | ||||
| -rw-r--r-- | io_uring/poll.c | 2 | ||||
| -rw-r--r-- | io_uring/uring_cmd.c | 2 |
5 files changed, 27 insertions, 12 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index a0c71a2dce19..cc27413129fc 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -814,7 +814,7 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, return filled; } -void io_req_complete_post(struct io_kiocb *req) +static void __io_req_complete_post(struct io_kiocb *req) { struct io_ring_ctx *ctx = req->ctx; @@ -850,9 +850,18 @@ void io_req_complete_post(struct io_kiocb *req) io_cq_unlock_post(ctx); } -inline void __io_req_complete(struct io_kiocb *req, unsigned issue_flags) +void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags) { - io_req_complete_post(req); + if (!(issue_flags & IO_URING_F_UNLOCKED) || + !(req->ctx->flags & IORING_SETUP_IOPOLL)) { + __io_req_complete_post(req); + } else { + struct io_ring_ctx *ctx = req->ctx; + + mutex_lock(&ctx->uring_lock); + __io_req_complete_post(req); + mutex_unlock(&ctx->uring_lock); + } } void io_req_complete_failed(struct io_kiocb *req, s32 res) @@ -866,7 +875,7 @@ void io_req_complete_failed(struct io_kiocb *req, s32 res) io_req_set_res(req, res, io_put_kbuf(req, IO_URING_F_UNLOCKED)); if (def->fail) def->fail(req); - io_req_complete_post(req); + io_req_complete_post(req, 0); } /* @@ -1450,7 +1459,7 @@ void io_req_task_complete(struct io_kiocb *req, bool *locked) if (*locked) io_req_complete_defer(req); else - io_req_complete_post(req); + io_req_complete_post_tw(req, locked); } /* @@ -1718,7 +1727,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) if (issue_flags & IO_URING_F_COMPLETE_DEFER) io_req_complete_defer(req); else - io_req_complete_post(req); + io_req_complete_post(req, issue_flags); } else if (ret != IOU_ISSUE_SKIP_COMPLETE) return ret; diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 222af88df10f..b5b80bf03385 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -31,14 +31,20 @@ int io_run_task_work_sig(struct io_ring_ctx *ctx); int __io_run_local_work(struct io_ring_ctx *ctx, bool *locked); int io_run_local_work(struct io_ring_ctx *ctx); void io_req_complete_failed(struct io_kiocb *req, s32 res); -void __io_req_complete(struct io_kiocb *req, unsigned issue_flags); -void io_req_complete_post(struct io_kiocb *req); +void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags); bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags, bool allow_overflow); bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags, bool allow_overflow); void __io_commit_cqring_flush(struct io_ring_ctx *ctx); +static inline void io_req_complete_post_tw(struct io_kiocb *req, bool *locked) +{ + unsigned flags = *locked ? 0 : IO_URING_F_UNLOCKED; + + io_req_complete_post(req, flags); +} + struct page **io_pin_pages(unsigned long ubuf, unsigned long len, int *npages); struct file *io_file_get_normal(struct io_kiocb *req, int fd); diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index e2c46889d5fa..e8150ed637d8 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -311,7 +311,7 @@ int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) /* complete before unlock, IOPOLL may need the lock */ io_req_set_res(req, ret, 0); - __io_req_complete(req, issue_flags); + io_req_complete_post(req, 0); io_ring_submit_unlock(ctx, issue_flags); return IOU_ISSUE_SKIP_COMPLETE; } @@ -462,7 +462,7 @@ err: req_set_fail(req); /* complete before unlock, IOPOLL may need the lock */ io_req_set_res(req, ret, 0); - __io_req_complete(req, issue_flags); + io_req_complete_post(req, 0); io_ring_submit_unlock(ctx, issue_flags); return IOU_ISSUE_SKIP_COMPLETE; } diff --git a/io_uring/poll.c b/io_uring/poll.c index cd4d98d622d2..4624e5eba63e 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -312,7 +312,7 @@ static void io_apoll_task_func(struct io_kiocb *req, bool *locked) io_poll_tw_hash_eject(req, locked); if (ret == IOU_POLL_REMOVE_POLL_USE_RES) - io_req_complete_post(req); + io_req_complete_post_tw(req, locked); else if (ret == IOU_POLL_DONE) io_req_task_submit(req, locked); else diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index e50de0b6b9f8..446a189b78b0 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -56,7 +56,7 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) /* order with io_iopoll_req_issued() checking ->iopoll_complete */ smp_store_release(&req->iopoll_completed, 1); else - __io_req_complete(req, 0); + io_req_complete_post(req, 0); } EXPORT_SYMBOL_GPL(io_uring_cmd_done); |