From 3e1a8bdd05d6b1734a8ccf7af28042d72c447780 Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:02 +0100 Subject: dm crypt: tidy inc pending Move io pending to one place. No functional change, usefull to simplify debugging. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 13956437bc81..6b9be99eff5b 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -517,6 +517,11 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) } } +static void crypt_inc_pending(struct dm_crypt_io *io) +{ + atomic_inc(&io->pending); +} + /* * One of the bios was finished. Check for completion of * the whole request and correctly clean up the buffer. @@ -591,7 +596,7 @@ static void kcryptd_io_read(struct dm_crypt_io *io) struct bio *base_bio = io->base_bio; struct bio *clone; - atomic_inc(&io->pending); + crypt_inc_pending(io); /* * The block layer might modify the bvec array, so always @@ -665,7 +670,7 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, if (async) kcryptd_queue_io(io); else { - atomic_inc(&io->pending); + crypt_inc_pending(io); generic_make_request(clone); } } @@ -701,7 +706,7 @@ static void kcryptd_crypt_write_convert_loop(struct dm_crypt_io *io) if (unlikely(r < 0)) return; } else - atomic_inc(&io->pending); + crypt_inc_pending(io); /* out of memory -> run queues */ if (unlikely(remaining)) { @@ -720,7 +725,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) /* * Prevent io from disappearing until this function completes. */ - atomic_inc(&io->pending); + crypt_inc_pending(io); crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, io->sector); kcryptd_crypt_write_convert_loop(io); @@ -741,7 +746,7 @@ static void kcryptd_crypt_read_convert(struct dm_crypt_io *io) struct crypt_config *cc = io->target->private; int r = 0; - atomic_inc(&io->pending); + crypt_inc_pending(io); crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio, io->sector); -- cgit v1.2.3 From dc440d1e56c481f80d5350daadc7d078a04ca729 Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:03 +0100 Subject: dm crypt: tidy crypt alloc Factor out crypt io allocation code. Later patches will call it from another place. No functional change. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 6b9be99eff5b..cac064d4aa72 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -517,6 +517,22 @@ static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone) } } +static struct dm_crypt_io *crypt_io_alloc(struct dm_target *ti, + struct bio *bio, sector_t sector) +{ + struct crypt_config *cc = ti->private; + struct dm_crypt_io *io; + + io = mempool_alloc(cc->io_pool, GFP_NOIO); + io->target = ti; + io->base_bio = bio; + io->sector = sector; + io->error = 0; + atomic_set(&io->pending, 0); + + return io; +} + static void crypt_inc_pending(struct dm_crypt_io *io) { atomic_inc(&io->pending); @@ -1113,15 +1129,9 @@ static void crypt_dtr(struct dm_target *ti) static int crypt_map(struct dm_target *ti, struct bio *bio, union map_info *map_context) { - struct crypt_config *cc = ti->private; struct dm_crypt_io *io; - io = mempool_alloc(cc->io_pool, GFP_NOIO); - io->target = ti; - io->base_bio = bio; - io->sector = bio->bi_sector - ti->begin; - io->error = 0; - atomic_set(&io->pending, 0); + io = crypt_io_alloc(ti, bio, bio->bi_sector - ti->begin); if (bio_data_dir(io->base_bio) == READ) kcryptd_queue_io(io); -- cgit v1.2.3 From fc5a5e9aa878f86642c962b309f793fb2db0727e Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:04 +0100 Subject: dm crypt: tidy write loop pending Move kcryptd_crypt_write_convert_loop inside kcryptd_crypt_write_convert. This change is needed for a later patch. No functional change. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index cac064d4aa72..97b407582c03 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -691,13 +691,19 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, } } -static void kcryptd_crypt_write_convert_loop(struct dm_crypt_io *io) +static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) { struct crypt_config *cc = io->target->private; struct bio *clone; unsigned remaining = io->base_bio->bi_size; int r; + /* + * Prevent io from disappearing until this function completes. + */ + crypt_inc_pending(io); + crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, io->sector); + /* * The allocated buffers can be smaller than the whole bio, * so repeat the whole process until all the data can be handled. @@ -706,7 +712,7 @@ static void kcryptd_crypt_write_convert_loop(struct dm_crypt_io *io) clone = crypt_alloc_buffer(io, remaining); if (unlikely(!clone)) { io->error = -ENOMEM; - return; + break; } io->ctx.bio_out = clone; @@ -720,7 +726,7 @@ static void kcryptd_crypt_write_convert_loop(struct dm_crypt_io *io) /* processed, no running async crypto */ kcryptd_crypt_write_io_submit(io, r, 0); if (unlikely(r < 0)) - return; + break; } else crypt_inc_pending(io); @@ -732,19 +738,6 @@ static void kcryptd_crypt_write_convert_loop(struct dm_crypt_io *io) congestion_wait(WRITE, HZ/100); } } -} - -static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) -{ - struct crypt_config *cc = io->target->private; - - /* - * Prevent io from disappearing until this function completes. - */ - crypt_inc_pending(io); - - crypt_convert_init(cc, &io->ctx, NULL, io->base_bio, io->sector); - kcryptd_crypt_write_convert_loop(io); crypt_dec_pending(io); } -- cgit v1.2.3 From 1e37bb8e557a186d327eb4d1387953880ffc2cdd Mon Sep 17 00:00:00 2001 From: Alasdair G Kergon Date: Fri, 10 Oct 2008 13:37:05 +0100 Subject: dm crypt: remove inc_pending from write_io_submit Make the caller reponsible for incrementing the pending count before calling kcryptd_crypt_write_io_submit() in the non-async case to bring it into line with the async case. Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 97b407582c03..0042636ad375 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -685,10 +685,8 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, if (async) kcryptd_queue_io(io); - else { - crypt_inc_pending(io); + else generic_make_request(clone); - } } static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) @@ -724,9 +722,12 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) if (atomic_dec_and_test(&io->ctx.pending)) { /* processed, no running async crypto */ + crypt_inc_pending(io); kcryptd_crypt_write_io_submit(io, r, 0); - if (unlikely(r < 0)) + if (unlikely(r < 0)) { + crypt_dec_pending(io); break; + } } else crypt_inc_pending(io); -- cgit v1.2.3 From 6c031f41db15b6cb0cd33545cec28ca706cd3c7e Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:06 +0100 Subject: dm crypt: move dec_pending on error into write_io_submit Make kcryptd_crypt_write_io_submit() responsible for decrementing the pending count after an error. Also fixes a bug in the async path that forgot to decrement it. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 0042636ad375..d8126ac82960 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -674,6 +674,7 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, crypt_free_buffer_pages(cc, clone); bio_put(clone); io->error = -EIO; + crypt_dec_pending(io); return; } @@ -724,10 +725,8 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) /* processed, no running async crypto */ crypt_inc_pending(io); kcryptd_crypt_write_io_submit(io, r, 0); - if (unlikely(r < 0)) { - crypt_dec_pending(io); + if (unlikely(r < 0)) break; - } } else crypt_inc_pending(io); -- cgit v1.2.3 From 4e59409891c9cc30cb4d5d73250b0c968af8e39b Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:07 +0100 Subject: dm crypt: fix async inc_pending The pending reference count must be incremented *before* the async work is queued to another thread, not after. Otherwise there's a race if the work completes and decrements the reference count before it gets incremented. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index d8126ac82960..262ed1816695 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -719,16 +719,15 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) remaining -= clone->bi_size; + crypt_inc_pending(io); r = crypt_convert(cc, &io->ctx); if (atomic_dec_and_test(&io->ctx.pending)) { /* processed, no running async crypto */ - crypt_inc_pending(io); kcryptd_crypt_write_io_submit(io, r, 0); if (unlikely(r < 0)) break; - } else - crypt_inc_pending(io); + } /* out of memory -> run queues */ if (unlikely(remaining)) { -- cgit v1.2.3 From c8081618a9f832fdf7ca81eb087f9f61f2bf07d5 Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:08 +0100 Subject: dm crypt: tidy ctx pending Move the initialisation of ctx->pending into one place, at the start of crypt_convert(). Introduce crypt_finished to indicate whether or not the encryption is finished, for use in a later patch. No functional change. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 262ed1816695..f6018f5961de 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -333,7 +333,6 @@ static void crypt_convert_init(struct crypt_config *cc, ctx->idx_out = bio_out ? bio_out->bi_idx : 0; ctx->sector = sector + cc->iv_offset; init_completion(&ctx->restart); - atomic_set(&ctx->pending, 1); } static int crypt_convert_block(struct crypt_config *cc, @@ -408,6 +407,8 @@ static int crypt_convert(struct crypt_config *cc, { int r; + atomic_set(&ctx->pending, 1); + while(ctx->idx_in < ctx->bio_in->bi_vcnt && ctx->idx_out < ctx->bio_out->bi_vcnt) { @@ -694,6 +695,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) { struct crypt_config *cc = io->target->private; struct bio *clone; + int crypt_finished; unsigned remaining = io->base_bio->bi_size; int r; @@ -721,19 +723,23 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) crypt_inc_pending(io); r = crypt_convert(cc, &io->ctx); + crypt_finished = atomic_dec_and_test(&io->ctx.pending); - if (atomic_dec_and_test(&io->ctx.pending)) { - /* processed, no running async crypto */ + /* Encryption was already finished, submit io now */ + if (crypt_finished) { kcryptd_crypt_write_io_submit(io, r, 0); + + /* + * If there was an error, do not try next fragments. + * For async, error is processed in async handler. + */ if (unlikely(r < 0)) break; } /* out of memory -> run queues */ if (unlikely(remaining)) { - /* wait for async crypto then reinitialize pending */ wait_event(cc->writeq, !atomic_read(&io->ctx.pending)); - atomic_set(&io->ctx.pending, 1); congestion_wait(WRITE, HZ/100); } } -- cgit v1.2.3 From 933f01d43326fb12a978a8e0bb062c28a2de4d5a Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Fri, 10 Oct 2008 13:37:08 +0100 Subject: dm crypt: avoid unnecessary wait when splitting bio Don't wait between submitting crypt requests for a bio unless we are short of memory. There are two situations when we must split an encrypted bio: 1) there are no free pages; 2) the new bio would violate underlying device restrictions (e.g. max hw segments). In case (2) we do not need to wait. Add output variable to crypt_alloc_buffer() to distinguish between these cases. Signed-off-by: Milan Broz Signed-off-by: Alasdair G Kergon --- drivers/md/dm-crypt.c | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'drivers/md/dm-crypt.c') diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index f6018f5961de..682ef9e6acd3 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -457,9 +457,11 @@ static void dm_crypt_bio_destructor(struct bio *bio) /* * Generate a new unfragmented bio with the given size * This should never violate the device limitations - * May return a smaller bio when running out of pages + * May return a smaller bio when running out of pages, indicated by + * *out_of_pages set to 1. */ -static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) +static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, + unsigned *out_of_pages) { struct crypt_config *cc = io->target->private; struct bio *clone; @@ -473,11 +475,14 @@ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size) return NULL; clone_init(io, clone); + *out_of_pages = 0; for (i = 0; i < nr_iovecs; i++) { page = mempool_alloc(cc->page_pool, gfp_mask); - if (!page) + if (!page) { + *out_of_pages = 1; break; + } /* * if additional pages cannot be allocated without waiting, @@ -696,6 +701,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) struct crypt_config *cc = io->target->private; struct bio *clone; int crypt_finished; + unsigned out_of_pages = 0; unsigned remaining = io->base_bio->bi_size; int r; @@ -710,7 +716,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) * so repeat the whole process until all the data can be handled. */ while (remaining) { - clone = crypt_alloc_buffer(io, remaining); + clone = crypt_alloc_buffer(io, remaining, &out_of_pages); if (unlikely(!clone)) { io->error = -ENOMEM; break; @@ -737,11 +743,15 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) break; } - /* out of memory -> run queues */ - if (unlikely(remaining)) { - wait_event(cc->writeq, !atomic_read(&io->ctx.pending)); + /* + * Out of memory -> run queues + * But don't wait if split was due to the io size restriction + */ + if (unlikely(out_of_pages)) congestion_wait(WRITE, HZ/100); - } + + if (unlikely(remaining)) + wait_event(cc->writeq, !atomic_read(&io->ctx.pending)); } crypt_dec_pending(io); -- cgit v1.2.3