From 29bcff787a2593b2126cfaff612c0b4e560022e9 Mon Sep 17 00:00:00 2001 From: Yufen Yu Date: Thu, 20 Aug 2020 09:22:08 -0400 Subject: md/raid5: add new xor function to support different page offset raid5 will call async_xor() and async_xor_val() to compute xor. For now, both of them require the common src/dst page offset. But, we want them to support different src/dst page offset for following shared page. Here, adding two new function async_xor_offs() and async_xor_val_offs() respectively for async_xor() and async_xor_val(). Signed-off-by: Yufen Yu Signed-off-by: Song Liu --- crypto/async_tx/async_xor.c | 120 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 17 deletions(-) (limited to 'crypto') diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index 4e5eebe52e6a..a057ecb1288d 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c @@ -97,7 +97,8 @@ do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap, } static void -do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, +do_sync_xor_offs(struct page *dest, unsigned int offset, + struct page **src_list, unsigned int *src_offs, int src_cnt, size_t len, struct async_submit_ctl *submit) { int i; @@ -114,7 +115,8 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, /* convert to buffer pointers */ for (i = 0; i < src_cnt; i++) if (src_list[i]) - srcs[xor_src_cnt++] = page_address(src_list[i]) + offset; + srcs[xor_src_cnt++] = page_address(src_list[i]) + + (src_offs ? src_offs[i] : offset); src_cnt = xor_src_cnt; /* set destination address */ dest_buf = page_address(dest) + offset; @@ -135,11 +137,31 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, async_tx_sync_epilog(submit); } +static inline bool +dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset, + unsigned int *src_offs, int src_cnt, int len) +{ + int i; + + if (!is_dma_xor_aligned(device, offset, 0, len)) + return false; + + if (!src_offs) + return true; + + for (i = 0; i < src_cnt; i++) { + if (!is_dma_xor_aligned(device, src_offs[i], 0, len)) + return false; + } + return true; +} + /** - * async_xor - attempt to xor a set of blocks with a dma engine. + * async_xor_offs - attempt to xor a set of blocks with a dma engine. * @dest: destination page + * @offset: dst offset to start transaction * @src_list: array of source pages - * @offset: common src/dst offset to start transaction + * @src_offs: array of source pages offset, NULL means common src/dst offset * @src_cnt: number of source pages * @len: length in bytes * @submit: submission / completion modifiers @@ -157,8 +179,9 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, * is not specified. */ struct dma_async_tx_descriptor * -async_xor(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, struct async_submit_ctl *submit) +async_xor_offs(struct page *dest, unsigned int offset, + struct page **src_list, unsigned int *src_offs, + int src_cnt, size_t len, struct async_submit_ctl *submit) { struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, &dest, 1, src_list, @@ -171,7 +194,8 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, if (device) unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOWAIT); - if (unmap && is_dma_xor_aligned(device, offset, 0, len)) { + if (unmap && dma_xor_aligned_offsets(device, offset, + src_offs, src_cnt, len)) { struct dma_async_tx_descriptor *tx; int i, j; @@ -184,7 +208,8 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, continue; unmap->to_cnt++; unmap->addr[j++] = dma_map_page(device->dev, src_list[i], - offset, len, DMA_TO_DEVICE); + src_offs ? src_offs[i] : offset, + len, DMA_TO_DEVICE); } /* map it bidirectional as it may be re-used as a source */ @@ -213,11 +238,42 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); - do_sync_xor(dest, src_list, offset, src_cnt, len, submit); + do_sync_xor_offs(dest, offset, src_list, src_offs, + src_cnt, len, submit); return NULL; } } +EXPORT_SYMBOL_GPL(async_xor_offs); + +/** + * async_xor - attempt to xor a set of blocks with a dma engine. + * @dest: destination page + * @src_list: array of source pages + * @offset: common src/dst offset to start transaction + * @src_cnt: number of source pages + * @len: length in bytes + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST + * + * xor_blocks always uses the dest as a source so the + * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in + * the calculation. The assumption with dma eninges is that they only + * use the destination buffer as a source when it is explicity specified + * in the source list. + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. + */ +struct dma_async_tx_descriptor * +async_xor(struct page *dest, struct page **src_list, unsigned int offset, + int src_cnt, size_t len, struct async_submit_ctl *submit) +{ + return async_xor_offs(dest, offset, src_list, NULL, + src_cnt, len, submit); +} EXPORT_SYMBOL_GPL(async_xor); static int page_is_zero(struct page *p, unsigned int offset, size_t len) @@ -237,10 +293,11 @@ xor_val_chan(struct async_submit_ctl *submit, struct page *dest, } /** - * async_xor_val - attempt a xor parity check with a dma engine. + * async_xor_val_offs - attempt a xor parity check with a dma engine. * @dest: destination page used if the xor is performed synchronously + * @offset: des offset in pages to start transaction * @src_list: array of source pages - * @offset: offset in pages to start transaction + * @src_offs: array of source pages offset, NULL means common src/det offset * @src_cnt: number of source pages * @len: length in bytes * @result: 0 if sum == 0 else non-zero @@ -253,9 +310,10 @@ xor_val_chan(struct async_submit_ctl *submit, struct page *dest, * is not specified. */ struct dma_async_tx_descriptor * -async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, enum sum_check_flags *result, - struct async_submit_ctl *submit) +async_xor_val_offs(struct page *dest, unsigned int offset, + struct page **src_list, unsigned int *src_offs, + int src_cnt, size_t len, enum sum_check_flags *result, + struct async_submit_ctl *submit) { struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; @@ -268,7 +326,7 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOWAIT); if (unmap && src_cnt <= device->max_xor && - is_dma_xor_aligned(device, offset, 0, len)) { + dma_xor_aligned_offsets(device, offset, src_offs, src_cnt, len)) { unsigned long dma_prep_flags = 0; int i; @@ -281,7 +339,8 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, for (i = 0; i < src_cnt; i++) { unmap->addr[i] = dma_map_page(device->dev, src_list[i], - offset, len, DMA_TO_DEVICE); + src_offs ? src_offs[i] : offset, + len, DMA_TO_DEVICE); unmap->to_cnt++; } unmap->len = len; @@ -312,7 +371,8 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, submit->flags |= ASYNC_TX_XOR_DROP_DST; submit->flags &= ~ASYNC_TX_ACK; - tx = async_xor(dest, src_list, offset, src_cnt, len, submit); + tx = async_xor_offs(dest, offset, src_list, src_offs, + src_cnt, len, submit); async_tx_quiesce(&tx); @@ -325,6 +385,32 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, return tx; } +EXPORT_SYMBOL_GPL(async_xor_val_offs); + +/** + * async_xor_val - attempt a xor parity check with a dma engine. + * @dest: destination page used if the xor is performed synchronously + * @src_list: array of source pages + * @offset: offset in pages to start transaction + * @src_cnt: number of source pages + * @len: length in bytes + * @result: 0 if sum == 0 else non-zero + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. + */ +struct dma_async_tx_descriptor * +async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, + int src_cnt, size_t len, enum sum_check_flags *result, + struct async_submit_ctl *submit) +{ + return async_xor_val_offs(dest, offset, src_list, NULL, src_cnt, + len, result, submit); +} EXPORT_SYMBOL_GPL(async_xor_val); MODULE_AUTHOR("Intel Corporation"); -- cgit v1.2.3 From d69454bc9fdfb63e84152e0533454079ea3e38a3 Mon Sep 17 00:00:00 2001 From: Yufen Yu Date: Thu, 20 Aug 2020 09:22:10 -0400 Subject: md/raid6: let syndrome computor support different page offset For now, syndrome compute functions require common offset in the pages array. However, we expect them to support different offset when try to use shared page in the following. Simplily covert them by adding page offset where each page address are referred. Since the only caller of async_gen_syndrome() and async_syndrome_val() are in raid6, we don't want to reserve the old interface but modify the interface directly. After that, replacing old interfaces with new ones for raid6 and raid6test. Signed-off-by: Yufen Yu Signed-off-by: Song Liu --- crypto/async_tx/async_pq.c | 72 ++++++++++++++++++++++++++++++--------------- crypto/async_tx/raid6test.c | 24 ++++++++++----- 2 files changed, 64 insertions(+), 32 deletions(-) (limited to 'crypto') diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c index 341ece61cf9b..f9cdc5e91664 100644 --- a/crypto/async_tx/async_pq.c +++ b/crypto/async_tx/async_pq.c @@ -104,7 +104,7 @@ do_async_gen_syndrome(struct dma_chan *chan, * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome */ static void -do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, +do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, size_t len, struct async_submit_ctl *submit) { void **srcs; @@ -121,7 +121,8 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, BUG_ON(i > disks - 3); /* P or Q can't be zero */ srcs[i] = (void*)raid6_empty_zero_page; } else { - srcs[i] = page_address(blocks[i]) + offset; + srcs[i] = page_address(blocks[i]) + offsets[i]; + if (i < disks - 2) { stop = i; if (start == -1) @@ -138,10 +139,23 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, async_tx_sync_epilog(submit); } +static inline bool +is_dma_pq_aligned_offs(struct dma_device *dev, unsigned int *offs, + int src_cnt, size_t len) +{ + int i; + + for (i = 0; i < src_cnt; i++) { + if (!is_dma_pq_aligned(dev, offs[i], 0, len)) + return false; + } + return true; +} + /** * async_gen_syndrome - asynchronously calculate a raid6 syndrome * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 - * @offset: common offset into each block (src and dest) to start transaction + * @offsets: offset array into each block (src and dest) to start transaction * @disks: number of blocks (including missing P or Q, see below) * @len: length of operation in bytes * @submit: submission/completion modifiers @@ -160,7 +174,7 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, * path. */ struct dma_async_tx_descriptor * -async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, +async_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, size_t len, struct async_submit_ctl *submit) { int src_cnt = disks - 2; @@ -179,7 +193,7 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, if (unmap && !(submit->flags & ASYNC_TX_PQ_XOR_DST) && (src_cnt <= dma_maxpq(device, 0) || dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && - is_dma_pq_aligned(device, offset, 0, len)) { + is_dma_pq_aligned_offs(device, offsets, disks, len)) { struct dma_async_tx_descriptor *tx; enum dma_ctrl_flags dma_flags = 0; unsigned char coefs[MAX_DISKS]; @@ -196,8 +210,8 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, for (i = 0, j = 0; i < src_cnt; i++) { if (blocks[i] == NULL) continue; - unmap->addr[j] = dma_map_page(device->dev, blocks[i], offset, - len, DMA_TO_DEVICE); + unmap->addr[j] = dma_map_page(device->dev, blocks[i], + offsets[i], len, DMA_TO_DEVICE); coefs[j] = raid6_gfexp[i]; unmap->to_cnt++; j++; @@ -210,7 +224,8 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, unmap->bidi_cnt++; if (P(blocks, disks)) unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks), - offset, len, DMA_BIDIRECTIONAL); + P(offsets, disks), + len, DMA_BIDIRECTIONAL); else { unmap->addr[j++] = 0; dma_flags |= DMA_PREP_PQ_DISABLE_P; @@ -219,7 +234,8 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, unmap->bidi_cnt++; if (Q(blocks, disks)) unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks), - offset, len, DMA_BIDIRECTIONAL); + Q(offsets, disks), + len, DMA_BIDIRECTIONAL); else { unmap->addr[j++] = 0; dma_flags |= DMA_PREP_PQ_DISABLE_Q; @@ -240,13 +256,13 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, if (!P(blocks, disks)) { P(blocks, disks) = pq_scribble_page; - BUG_ON(len + offset > PAGE_SIZE); + P(offsets, disks) = 0; } if (!Q(blocks, disks)) { Q(blocks, disks) = pq_scribble_page; - BUG_ON(len + offset > PAGE_SIZE); + Q(offsets, disks) = 0; } - do_sync_gen_syndrome(blocks, offset, disks, len, submit); + do_sync_gen_syndrome(blocks, offsets, disks, len, submit); return NULL; } @@ -270,6 +286,7 @@ pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, si * @len: length of operation in bytes * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set * @spare: temporary result buffer for the synchronous case + * @s_off: spare buffer page offset * @submit: submission / completion modifiers * * The same notes from async_gen_syndrome apply to the 'blocks', @@ -278,9 +295,9 @@ pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, si * specified. */ struct dma_async_tx_descriptor * -async_syndrome_val(struct page **blocks, unsigned int offset, int disks, +async_syndrome_val(struct page **blocks, unsigned int *offsets, int disks, size_t len, enum sum_check_flags *pqres, struct page *spare, - struct async_submit_ctl *submit) + unsigned int s_off, struct async_submit_ctl *submit) { struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); struct dma_device *device = chan ? chan->device : NULL; @@ -295,7 +312,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOWAIT); if (unmap && disks <= dma_maxpq(device, 0) && - is_dma_pq_aligned(device, offset, 0, len)) { + is_dma_pq_aligned_offs(device, offsets, disks, len)) { struct device *dev = device->dev; dma_addr_t pq[2]; int i, j = 0, src_cnt = 0; @@ -307,7 +324,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, for (i = 0; i < disks-2; i++) if (likely(blocks[i])) { unmap->addr[j] = dma_map_page(dev, blocks[i], - offset, len, + offsets[i], len, DMA_TO_DEVICE); coefs[j] = raid6_gfexp[i]; unmap->to_cnt++; @@ -320,7 +337,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, dma_flags |= DMA_PREP_PQ_DISABLE_P; } else { pq[0] = dma_map_page(dev, P(blocks, disks), - offset, len, + P(offsets, disks), len, DMA_TO_DEVICE); unmap->addr[j++] = pq[0]; unmap->to_cnt++; @@ -330,7 +347,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, dma_flags |= DMA_PREP_PQ_DISABLE_Q; } else { pq[1] = dma_map_page(dev, Q(blocks, disks), - offset, len, + Q(offsets, disks), len, DMA_TO_DEVICE); unmap->addr[j++] = pq[1]; unmap->to_cnt++; @@ -355,7 +372,9 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, async_tx_submit(chan, tx, submit); } else { struct page *p_src = P(blocks, disks); + unsigned int p_off = P(offsets, disks); struct page *q_src = Q(blocks, disks); + unsigned int q_off = Q(offsets, disks); enum async_tx_flags flags_orig = submit->flags; dma_async_tx_callback cb_fn_orig = submit->cb_fn; void *scribble = submit->scribble; @@ -381,27 +400,32 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, if (p_src) { init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, NULL, NULL, scribble); - tx = async_xor(spare, blocks, offset, disks-2, len, submit); + tx = async_xor_offs(spare, s_off, + blocks, offsets, disks-2, len, submit); async_tx_quiesce(&tx); - p = page_address(p_src) + offset; - s = page_address(spare) + offset; + p = page_address(p_src) + p_off; + s = page_address(spare) + s_off; *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; } if (q_src) { P(blocks, disks) = NULL; Q(blocks, disks) = spare; + Q(offsets, disks) = s_off; init_async_submit(submit, 0, NULL, NULL, NULL, scribble); - tx = async_gen_syndrome(blocks, offset, disks, len, submit); + tx = async_gen_syndrome(blocks, offsets, disks, + len, submit); async_tx_quiesce(&tx); - q = page_address(q_src) + offset; - s = page_address(spare) + offset; + q = page_address(q_src) + q_off; + s = page_address(spare) + s_off; *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; } /* restore P, Q and submit */ P(blocks, disks) = p_src; + P(offsets, disks) = p_off; Q(blocks, disks) = q_src; + Q(offsets, disks) = q_off; submit->cb_fn = cb_fn_orig; submit->cb_param = cb_param_orig; diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c index 14e73dcd7475..ef4e6e68f759 100644 --- a/crypto/async_tx/raid6test.c +++ b/crypto/async_tx/raid6test.c @@ -18,6 +18,7 @@ #define NDISKS 64 /* Including P and Q */ static struct page *dataptrs[NDISKS]; +unsigned int dataoffs[NDISKS]; static addr_conv_t addr_conv[NDISKS]; static struct page *data[NDISKS+3]; static struct page *spare; @@ -38,6 +39,7 @@ static void makedata(int disks) for (i = 0; i < disks; i++) { prandom_bytes(page_address(data[i]), PAGE_SIZE); dataptrs[i] = data[i]; + dataoffs[i] = 0; } } @@ -52,7 +54,8 @@ static char disk_type(int d, int disks) } /* Recover two failed blocks. */ -static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, struct page **ptrs) +static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, + struct page **ptrs, unsigned int *offs) { struct async_submit_ctl submit; struct completion cmp; @@ -66,7 +69,8 @@ static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, stru if (faila == disks-2) { /* P+Q failure. Just rebuild the syndrome. */ init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); - tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit); + tx = async_gen_syndrome(ptrs, offs, + disks, bytes, &submit); } else { struct page *blocks[NDISKS]; struct page *dest; @@ -89,22 +93,26 @@ static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, stru tx = async_xor(dest, blocks, 0, count, bytes, &submit); init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv); - tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit); + tx = async_gen_syndrome(ptrs, offs, + disks, bytes, &submit); } } else { if (failb == disks-2) { /* data+P failure. */ init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); - tx = async_raid6_datap_recov(disks, bytes, faila, ptrs, &submit); + tx = async_raid6_datap_recov(disks, bytes, + faila, ptrs, &submit); } else { /* data+data failure. */ init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); - tx = async_raid6_2data_recov(disks, bytes, faila, failb, ptrs, &submit); + tx = async_raid6_2data_recov(disks, bytes, + faila, failb, ptrs, &submit); } } init_completion(&cmp); init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv); - tx = async_syndrome_val(ptrs, 0, disks, bytes, &result, spare, &submit); + tx = async_syndrome_val(ptrs, offs, + disks, bytes, &result, spare, 0, &submit); async_tx_issue_pending(tx); if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) @@ -126,7 +134,7 @@ static int test_disks(int i, int j, int disks) dataptrs[i] = recovi; dataptrs[j] = recovj; - raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs); + raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs, dataoffs); erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE); errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE); @@ -162,7 +170,7 @@ static int test(int disks, int *tests) /* Generate assumed good syndrome */ init_completion(&cmp); init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv); - tx = async_gen_syndrome(dataptrs, 0, disks, PAGE_SIZE, &submit); + tx = async_gen_syndrome(dataptrs, dataoffs, disks, PAGE_SIZE, &submit); async_tx_issue_pending(tx); if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) { -- cgit v1.2.3 From 4f86ff5580fa692f850f37f948c73814a24a722b Mon Sep 17 00:00:00 2001 From: Yufen Yu Date: Thu, 20 Aug 2020 09:22:11 -0400 Subject: md/raid6: let async recovery function support different page offset For now, asynchronous raid6 recovery calculate functions are require common offset for pages. But, we expect them to support different page offset after introducing stripe shared page. Do that by simplily adding page offset where each page address are referred. Then, replace the old interface with the new ones in raid6 and raid6test. Signed-off-by: Yufen Yu Signed-off-by: Song Liu --- crypto/async_tx/async_raid6_recov.c | 163 ++++++++++++++++++++++++++---------- crypto/async_tx/raid6test.c | 4 +- 2 files changed, 122 insertions(+), 45 deletions(-) (limited to 'crypto') diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c index f249142ceac4..354b8cd5537f 100644 --- a/crypto/async_tx/async_raid6_recov.c +++ b/crypto/async_tx/async_raid6_recov.c @@ -15,8 +15,9 @@ #include static struct dma_async_tx_descriptor * -async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, - size_t len, struct async_submit_ctl *submit) +async_sum_product(struct page *dest, unsigned int d_off, + struct page **srcs, unsigned int *src_offs, unsigned char *coef, + size_t len, struct async_submit_ctl *submit) { struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, &dest, 1, srcs, 2, len); @@ -37,11 +38,14 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; - unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE); - unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE); + unmap->addr[0] = dma_map_page(dev, srcs[0], src_offs[0], + len, DMA_TO_DEVICE); + unmap->addr[1] = dma_map_page(dev, srcs[1], src_offs[1], + len, DMA_TO_DEVICE); unmap->to_cnt = 2; - unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); + unmap->addr[2] = dma_map_page(dev, dest, d_off, + len, DMA_BIDIRECTIONAL); unmap->bidi_cnt = 1; /* engine only looks at Q, but expects it to follow P */ pq[1] = unmap->addr[2]; @@ -66,9 +70,9 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, async_tx_quiesce(&submit->depend_tx); amul = raid6_gfmul[coef[0]]; bmul = raid6_gfmul[coef[1]]; - a = page_address(srcs[0]); - b = page_address(srcs[1]); - c = page_address(dest); + a = page_address(srcs[0]) + src_offs[0]; + b = page_address(srcs[1]) + src_offs[1]; + c = page_address(dest) + d_off; while (len--) { ax = amul[*a++]; @@ -80,8 +84,9 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, } static struct dma_async_tx_descriptor * -async_mult(struct page *dest, struct page *src, u8 coef, size_t len, - struct async_submit_ctl *submit) +async_mult(struct page *dest, unsigned int d_off, struct page *src, + unsigned int s_off, u8 coef, size_t len, + struct async_submit_ctl *submit) { struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, &dest, 1, &src, 1, len); @@ -101,9 +106,11 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len, if (submit->flags & ASYNC_TX_FENCE) dma_flags |= DMA_PREP_FENCE; - unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE); + unmap->addr[0] = dma_map_page(dev, src, s_off, + len, DMA_TO_DEVICE); unmap->to_cnt++; - unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); + unmap->addr[1] = dma_map_page(dev, dest, d_off, + len, DMA_BIDIRECTIONAL); dma_dest[1] = unmap->addr[1]; unmap->bidi_cnt++; unmap->len = len; @@ -133,8 +140,8 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len, */ async_tx_quiesce(&submit->depend_tx); qmul = raid6_gfmul[coef]; - d = page_address(dest); - s = page_address(src); + d = page_address(dest) + d_off; + s = page_address(src) + s_off; while (len--) *d++ = qmul[*s++]; @@ -144,11 +151,14 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len, static struct dma_async_tx_descriptor * __2data_recov_4(int disks, size_t bytes, int faila, int failb, - struct page **blocks, struct async_submit_ctl *submit) + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct page *p, *q, *a, *b; + unsigned int p_off, q_off, a_off, b_off; struct page *srcs[2]; + unsigned int src_offs[2]; unsigned char coef[2]; enum async_tx_flags flags = submit->flags; dma_async_tx_callback cb_fn = submit->cb_fn; @@ -156,26 +166,34 @@ __2data_recov_4(int disks, size_t bytes, int faila, int failb, void *scribble = submit->scribble; p = blocks[disks-2]; + p_off = offs[disks-2]; q = blocks[disks-1]; + q_off = offs[disks-1]; a = blocks[faila]; + a_off = offs[faila]; b = blocks[failb]; + b_off = offs[failb]; /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */ /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ srcs[0] = p; + src_offs[0] = p_off; srcs[1] = q; + src_offs[1] = q_off; coef[0] = raid6_gfexi[failb-faila]; coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_sum_product(b, srcs, coef, bytes, submit); + tx = async_sum_product(b, b_off, srcs, src_offs, coef, bytes, submit); /* Dy = P+Pxy+Dx */ srcs[0] = p; + src_offs[0] = p_off; srcs[1] = b; + src_offs[1] = b_off; init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn, cb_param, scribble); - tx = async_xor(a, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(a, a_off, srcs, src_offs, 2, bytes, submit); return tx; @@ -183,11 +201,14 @@ __2data_recov_4(int disks, size_t bytes, int faila, int failb, static struct dma_async_tx_descriptor * __2data_recov_5(int disks, size_t bytes, int faila, int failb, - struct page **blocks, struct async_submit_ctl *submit) + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct page *p, *q, *g, *dp, *dq; + unsigned int p_off, q_off, g_off, dp_off, dq_off; struct page *srcs[2]; + unsigned int src_offs[2]; unsigned char coef[2]; enum async_tx_flags flags = submit->flags; dma_async_tx_callback cb_fn = submit->cb_fn; @@ -208,60 +229,77 @@ __2data_recov_5(int disks, size_t bytes, int faila, int failb, BUG_ON(good_srcs > 1); p = blocks[disks-2]; + p_off = offs[disks-2]; q = blocks[disks-1]; + q_off = offs[disks-1]; g = blocks[good]; + g_off = offs[good]; /* Compute syndrome with zero for the missing data pages * Use the dead data pages as temporary storage for delta p and * delta q */ dp = blocks[faila]; + dp_off = offs[faila]; dq = blocks[failb]; + dq_off = offs[failb]; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_memcpy(dp, g, 0, 0, bytes, submit); + tx = async_memcpy(dp, g, dp_off, g_off, bytes, submit); init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); + tx = async_mult(dq, dq_off, g, g_off, + raid6_gfexp[good], bytes, submit); /* compute P + Pxy */ srcs[0] = dp; + src_offs[0] = dp_off; srcs[1] = p; + src_offs[1] = p_off; init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, NULL, NULL, scribble); - tx = async_xor(dp, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); /* compute Q + Qxy */ srcs[0] = dq; + src_offs[0] = dq_off; srcs[1] = q; + src_offs[1] = q_off; init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, NULL, NULL, scribble); - tx = async_xor(dq, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit); /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ srcs[0] = dp; + src_offs[0] = dp_off; srcs[1] = dq; + src_offs[1] = dq_off; coef[0] = raid6_gfexi[failb-faila]; coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_sum_product(dq, srcs, coef, bytes, submit); + tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit); /* Dy = P+Pxy+Dx */ srcs[0] = dp; + src_offs[0] = dp_off; srcs[1] = dq; + src_offs[1] = dq_off; init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, cb_param, scribble); - tx = async_xor(dp, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); return tx; } static struct dma_async_tx_descriptor * __2data_recov_n(int disks, size_t bytes, int faila, int failb, - struct page **blocks, struct async_submit_ctl *submit) + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct page *p, *q, *dp, *dq; + unsigned int p_off, q_off, dp_off, dq_off; struct page *srcs[2]; + unsigned int src_offs[2]; unsigned char coef[2]; enum async_tx_flags flags = submit->flags; dma_async_tx_callback cb_fn = submit->cb_fn; @@ -269,56 +307,74 @@ __2data_recov_n(int disks, size_t bytes, int faila, int failb, void *scribble = submit->scribble; p = blocks[disks-2]; + p_off = offs[disks-2]; q = blocks[disks-1]; + q_off = offs[disks-1]; /* Compute syndrome with zero for the missing data pages * Use the dead data pages as temporary storage for * delta p and delta q */ dp = blocks[faila]; + dp_off = offs[faila]; blocks[faila] = NULL; blocks[disks-2] = dp; + offs[disks-2] = dp_off; dq = blocks[failb]; + dq_off = offs[failb]; blocks[failb] = NULL; blocks[disks-1] = dq; + offs[disks-1] = dq_off; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); + tx = async_gen_syndrome(blocks, offs, disks, bytes, submit); /* Restore pointer table */ blocks[faila] = dp; + offs[faila] = dp_off; blocks[failb] = dq; + offs[failb] = dq_off; blocks[disks-2] = p; + offs[disks-2] = p_off; blocks[disks-1] = q; + offs[disks-1] = q_off; /* compute P + Pxy */ srcs[0] = dp; + src_offs[0] = dp_off; srcs[1] = p; + src_offs[1] = p_off; init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, NULL, NULL, scribble); - tx = async_xor(dp, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); /* compute Q + Qxy */ srcs[0] = dq; + src_offs[0] = dq_off; srcs[1] = q; + src_offs[1] = q_off; init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, NULL, NULL, scribble); - tx = async_xor(dq, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit); /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ srcs[0] = dp; + src_offs[0] = dp_off; srcs[1] = dq; + src_offs[1] = dq_off; coef[0] = raid6_gfexi[failb-faila]; coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_sum_product(dq, srcs, coef, bytes, submit); + tx = async_sum_product(dq, dq_off, srcs, src_offs, coef, bytes, submit); /* Dy = P+Pxy+Dx */ srcs[0] = dp; + src_offs[0] = dp_off; srcs[1] = dq; + src_offs[1] = dq_off; init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, cb_param, scribble); - tx = async_xor(dp, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dp, dp_off, srcs, src_offs, 2, bytes, submit); return tx; } @@ -330,11 +386,13 @@ __2data_recov_n(int disks, size_t bytes, int faila, int failb, * @faila: first failed drive index * @failb: second failed drive index * @blocks: array of source pointers where the last two entries are p and q + * @offs: array of offset for pages in blocks * @submit: submission/completion modifiers */ struct dma_async_tx_descriptor * async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, - struct page **blocks, struct async_submit_ctl *submit) + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) { void *scribble = submit->scribble; int non_zero_srcs, i; @@ -358,7 +416,7 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, if (blocks[i] == NULL) ptrs[i] = (void *) raid6_empty_zero_page; else - ptrs[i] = page_address(blocks[i]); + ptrs[i] = page_address(blocks[i]) + offs[i]; raid6_2data_recov(disks, bytes, faila, failb, ptrs); @@ -383,16 +441,19 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, * explicitly handle the special case of a 4 disk array with * both data disks missing. */ - return __2data_recov_4(disks, bytes, faila, failb, blocks, submit); + return __2data_recov_4(disks, bytes, faila, failb, + blocks, offs, submit); case 3: /* dma devices do not uniformly understand a single * source pq operation (in contrast to the synchronous * case), so explicitly handle the special case of a 5 disk * array with 2 of 3 data disks missing. */ - return __2data_recov_5(disks, bytes, faila, failb, blocks, submit); + return __2data_recov_5(disks, bytes, faila, failb, + blocks, offs, submit); default: - return __2data_recov_n(disks, bytes, faila, failb, blocks, submit); + return __2data_recov_n(disks, bytes, faila, failb, + blocks, offs, submit); } } EXPORT_SYMBOL_GPL(async_raid6_2data_recov); @@ -403,14 +464,17 @@ EXPORT_SYMBOL_GPL(async_raid6_2data_recov); * @bytes: block size * @faila: failed drive index * @blocks: array of source pointers where the last two entries are p and q + * @offs: array of offset for pages in blocks * @submit: submission/completion modifiers */ struct dma_async_tx_descriptor * async_raid6_datap_recov(int disks, size_t bytes, int faila, - struct page **blocks, struct async_submit_ctl *submit) + struct page **blocks, unsigned int *offs, + struct async_submit_ctl *submit) { struct dma_async_tx_descriptor *tx = NULL; struct page *p, *q, *dq; + unsigned int p_off, q_off, dq_off; u8 coef; enum async_tx_flags flags = submit->flags; dma_async_tx_callback cb_fn = submit->cb_fn; @@ -418,6 +482,7 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila, void *scribble = submit->scribble; int good_srcs, good, i; struct page *srcs[2]; + unsigned int src_offs[2]; pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); @@ -434,7 +499,7 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila, if (blocks[i] == NULL) ptrs[i] = (void*)raid6_empty_zero_page; else - ptrs[i] = page_address(blocks[i]); + ptrs[i] = page_address(blocks[i]) + offs[i]; raid6_datap_recov(disks, bytes, faila, ptrs); @@ -458,55 +523,67 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila, BUG_ON(good_srcs == 0); p = blocks[disks-2]; + p_off = offs[disks-2]; q = blocks[disks-1]; + q_off = offs[disks-1]; /* Compute syndrome with zero for the missing data page * Use the dead data page as temporary storage for delta q */ dq = blocks[faila]; + dq_off = offs[faila]; blocks[faila] = NULL; blocks[disks-1] = dq; + offs[disks-1] = dq_off; /* in the 4-disk case we only need to perform a single source * multiplication with the one good data block. */ if (good_srcs == 1) { struct page *g = blocks[good]; + unsigned int g_off = offs[good]; init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_memcpy(p, g, 0, 0, bytes, submit); + tx = async_memcpy(p, g, p_off, g_off, bytes, submit); init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); + tx = async_mult(dq, dq_off, g, g_off, + raid6_gfexp[good], bytes, submit); } else { init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); + tx = async_gen_syndrome(blocks, offs, disks, bytes, submit); } /* Restore pointer table */ blocks[faila] = dq; + offs[faila] = dq_off; blocks[disks-1] = q; + offs[disks-1] = q_off; /* calculate g^{-faila} */ coef = raid6_gfinv[raid6_gfexp[faila]]; srcs[0] = dq; + src_offs[0] = dq_off; srcs[1] = q; + src_offs[1] = q_off; init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, NULL, NULL, scribble); - tx = async_xor(dq, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(dq, dq_off, srcs, src_offs, 2, bytes, submit); init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); - tx = async_mult(dq, dq, coef, bytes, submit); + tx = async_mult(dq, dq_off, dq, dq_off, coef, bytes, submit); srcs[0] = p; + src_offs[0] = p_off; srcs[1] = dq; + src_offs[1] = dq_off; init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, cb_param, scribble); - tx = async_xor(p, srcs, 0, 2, bytes, submit); + tx = async_xor_offs(p, p_off, srcs, src_offs, 2, bytes, submit); return tx; } diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c index ef4e6e68f759..66db82e5a3b1 100644 --- a/crypto/async_tx/raid6test.c +++ b/crypto/async_tx/raid6test.c @@ -101,12 +101,12 @@ static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, /* data+P failure. */ init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); tx = async_raid6_datap_recov(disks, bytes, - faila, ptrs, &submit); + faila, ptrs, offs, &submit); } else { /* data+data failure. */ init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); tx = async_raid6_2data_recov(disks, bytes, - faila, failb, ptrs, &submit); + faila, failb, ptrs, offs, &submit); } } init_completion(&cmp); -- cgit v1.2.3