diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2019-07-08 20:57:08 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2019-07-08 20:57:08 -0700 |
commit | 4d2fa8b44b891f0da5ceda3e5a1402ccf0ab6f26 (patch) | |
tree | cbb763ec5e74cfbaac6ce53df277883cb78a8a1a /drivers/crypto/talitos.c | |
parent | 8b68150883ca466a23e90902dd4113b22e692f04 (diff) | |
parent | f3880a23564e3172437285ebcb5b8a124539fdae (diff) | |
download | linux-4d2fa8b44b891f0da5ceda3e5a1402ccf0ab6f26.tar.gz linux-4d2fa8b44b891f0da5ceda3e5a1402ccf0ab6f26.tar.bz2 linux-4d2fa8b44b891f0da5ceda3e5a1402ccf0ab6f26.zip |
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto updates from Herbert Xu:
"Here is the crypto update for 5.3:
API:
- Test shash interface directly in testmgr
- cra_driver_name is now mandatory
Algorithms:
- Replace arc4 crypto_cipher with library helper
- Implement 5 way interleave for ECB, CBC and CTR on arm64
- Add xxhash
- Add continuous self-test on noise source to drbg
- Update jitter RNG
Drivers:
- Add support for SHA204A random number generator
- Add support for 7211 in iproc-rng200
- Fix fuzz test failures in inside-secure
- Fix fuzz test failures in talitos
- Fix fuzz test failures in qat"
* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (143 commits)
crypto: stm32/hash - remove interruptible condition for dma
crypto: stm32/hash - Fix hmac issue more than 256 bytes
crypto: stm32/crc32 - rename driver file
crypto: amcc - remove memset after dma_alloc_coherent
crypto: ccp - Switch to SPDX license identifiers
crypto: ccp - Validate the the error value used to index error messages
crypto: doc - Fix formatting of new crypto engine content
crypto: doc - Add parameter documentation
crypto: arm64/aes-ce - implement 5 way interleave for ECB, CBC and CTR
crypto: arm64/aes-ce - add 5 way interleave routines
crypto: talitos - drop icv_ool
crypto: talitos - fix hash on SEC1.
crypto: talitos - move struct talitos_edesc into talitos.h
lib/scatterlist: Fix mapping iterator when sg->offset is greater than PAGE_SIZE
crypto/NX: Set receive window credits to max number of CRBs in RxFIFO
crypto: asymmetric_keys - select CRYPTO_HASH where needed
crypto: serpent - mark __serpent_setkey_sbox noinline
crypto: testmgr - dynamically allocate crypto_shash
crypto: testmgr - dynamically allocate testvec_config
crypto: talitos - eliminate unneeded 'done' functions at build time
...
Diffstat (limited to 'drivers/crypto/talitos.c')
-rw-r--r-- | drivers/crypto/talitos.c | 368 |
1 files changed, 172 insertions, 196 deletions
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index fbc7bf9d7380..c9d686a0e805 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -265,11 +265,11 @@ static int init_device(struct device *dev) * callback must check err and feedback in descriptor header * for device processing status. */ -int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, - void (*callback)(struct device *dev, - struct talitos_desc *desc, - void *context, int error), - void *context) +static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error), + void *context) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; @@ -319,7 +319,21 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } -EXPORT_SYMBOL(talitos_submit); + +static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (!is_sec1) + return request->desc->hdr; + + if (!request->desc->next_desc) + return request->desc->hdr1; + + edesc = container_of(request->desc, struct talitos_edesc, desc); + + return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; +} /* * process what was done, notify callback of error if not @@ -342,12 +356,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - if (!is_sec1) - hdr = request->desc->hdr; - else if (request->desc->next_desc) - hdr = (request->desc + 1)->hdr1; - else - hdr = request->desc->hdr1; + hdr = get_request_hdr(request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -477,8 +486,14 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } - if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) - return (priv->chan[ch].fifo[iter].desc + 1)->hdr; + if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { + struct talitos_edesc *edesc; + + edesc = container_of(priv->chan[ch].fifo[iter].desc, + struct talitos_edesc, desc); + return ((struct talitos_desc *) + (edesc->buf + edesc->dma_len))->hdr; + } return priv->chan[ch].fifo[iter].desc->hdr; } @@ -824,7 +839,11 @@ static void talitos_unregister_rng(struct device *dev) * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP */ #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) +#ifdef CONFIG_CRYPTO_DEV_TALITOS2 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) +#else +#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) +#endif #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ struct talitos_ctx { @@ -948,36 +967,6 @@ badkey: goto out; } -/* - * talitos_edesc - s/w-extended descriptor - * @src_nents: number of segments in input scatterlist - * @dst_nents: number of segments in output scatterlist - * @icv_ool: whether ICV is out-of-line - * @iv_dma: dma address of iv for checking continuity and link table - * @dma_len: length of dma mapped link_tbl space - * @dma_link_tbl: bus physical address of link_tbl/buf - * @desc: h/w descriptor - * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) - * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) - * - * if decrypting (with authcheck), or either one of src_nents or dst_nents - * is greater than 1, an integrity check value is concatenated to the end - * of link_tbl data - */ -struct talitos_edesc { - int src_nents; - int dst_nents; - bool icv_ool; - dma_addr_t iv_dma; - int dma_len; - dma_addr_t dma_link_tbl; - struct talitos_desc desc; - union { - struct talitos_ptr link_tbl[0]; - u8 buf[0]; - }; -}; - static void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc, struct scatterlist *src, @@ -1008,11 +997,13 @@ static void talitos_sg_unmap(struct device *dev, static void ipsec_esp_unmap(struct device *dev, struct talitos_edesc *edesc, - struct aead_request *areq) + struct aead_request *areq, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; @@ -1021,8 +1012,8 @@ static void ipsec_esp_unmap(struct device *dev, DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, - areq->assoclen); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, + cryptlen + authsize, areq->assoclen); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, @@ -1032,7 +1023,7 @@ static void ipsec_esp_unmap(struct device *dev, unsigned int dst_nents = edesc->dst_nents ? : 1; sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, - areq->assoclen + areq->cryptlen - ivsize); + areq->assoclen + cryptlen - ivsize); } } @@ -1043,31 +1034,14 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - unsigned int authsize = crypto_aead_authsize(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); struct talitos_edesc *edesc; - struct scatterlist *sg; - void *icvdata; edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, areq); - - /* copy the generated ICV to dst */ - if (edesc->icv_ool) { - if (is_sec1) - icvdata = edesc->buf + areq->assoclen + areq->cryptlen; - else - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - sg = sg_last(areq->dst, edesc->dst_nents); - memcpy((char *)sg_virt(sg) + sg->length - authsize, - icvdata, authsize); - } + ipsec_esp_unmap(dev, edesc, areq, true); dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); @@ -1084,32 +1058,16 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, struct crypto_aead *authenc = crypto_aead_reqtfm(req); unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; - struct scatterlist *sg; char *oicv, *icv; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, req); + ipsec_esp_unmap(dev, edesc, req, false); if (!err) { /* auth check */ - sg = sg_last(req->dst, edesc->dst_nents ? : 1); - icv = (char *)sg_virt(sg) + sg->length - authsize; - - if (edesc->dma_len) { - if (is_sec1) - oicv = (char *)&edesc->dma_link_tbl + - req->assoclen + req->cryptlen; - else - oicv = (char *) - &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - if (edesc->icv_ool) - icv = oicv + authsize; - } else - oicv = (char *)&edesc->link_tbl[0]; + oicv = edesc->buf + edesc->dma_len; + icv = oicv - authsize; err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; } @@ -1128,7 +1086,7 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, req); + ipsec_esp_unmap(dev, edesc, req, false); /* check ICV auth status */ if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != @@ -1145,11 +1103,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, * stop at cryptlen bytes */ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, - unsigned int offset, int cryptlen, + unsigned int offset, int datalen, int elen, struct talitos_ptr *link_tbl_ptr) { - int n_sg = sg_count; + int n_sg = elen ? sg_count + 1 : sg_count; int count = 0; + int cryptlen = datalen + elen; while (cryptlen && sg && n_sg--) { unsigned int len = sg_dma_len(sg); @@ -1164,11 +1123,20 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, if (len > cryptlen) len = cryptlen; + if (datalen > 0 && len > datalen) { + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, datalen, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); + count++; + len -= datalen; + offset += datalen; + } to_talitos_ptr(link_tbl_ptr + count, sg_dma_address(sg) + offset, len, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; + datalen -= len; offset = 0; next: @@ -1178,7 +1146,7 @@ next: /* tag end of link table */ if (count > 0) to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, - DESC_PTR_LNKTBL_RETURN, 0); + DESC_PTR_LNKTBL_RET, 0); return count; } @@ -1186,7 +1154,8 @@ next: static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, - unsigned int offset, int tbl_off, int elen) + unsigned int offset, int tbl_off, int elen, + bool force) { struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1196,7 +1165,7 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, return 1; } to_talitos_ptr_ext_set(ptr, elen, is_sec1); - if (sg_count == 1) { + if (sg_count == 1 && !force) { to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); return sg_count; } @@ -1204,9 +1173,9 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); return sg_count; } - sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen, + sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, &edesc->link_tbl[tbl_off]); - if (sg_count == 1) { + if (sg_count == 1 && !force) { /* Only one segment now, so no link tbl needed*/ copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); return sg_count; @@ -1224,13 +1193,14 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int offset, int tbl_off) { return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, - tbl_off, 0); + tbl_off, 0, false); } /* * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, + bool encrypt, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1240,7 +1210,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, struct talitos_ctx *ctx = crypto_aead_ctx(aead); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; - unsigned int cryptlen = areq->cryptlen; + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); unsigned int ivsize = crypto_aead_ivsize(aead); int tbl_off = 0; int sg_count, ret; @@ -1251,6 +1221,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; + dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; /* hmac key */ to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); @@ -1290,7 +1261,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, elen = authsize; ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], - sg_count, areq->assoclen, tbl_off, elen); + sg_count, areq->assoclen, tbl_off, elen, + false); if (ret > 1) { tbl_off += ret; @@ -1304,55 +1276,32 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); } - ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], - sg_count, areq->assoclen, tbl_off); - - if (is_ipsec_esp) - to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); - - /* ICV data */ - if (ret > 1) { - tbl_off += ret; - edesc->icv_ool = true; - sync_needed = true; - - if (is_ipsec_esp) { - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - int offset = (edesc->src_nents + edesc->dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize; - - /* Add an entry to the link table for ICV data */ - to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); - to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, - is_sec1); + if (is_ipsec_esp && encrypt) + elen = authsize; + else + elen = 0; + ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], + sg_count, areq->assoclen, tbl_off, elen, + is_ipsec_esp && !encrypt); + tbl_off += ret; - /* icv data follows link tables */ - to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, - authsize, is_sec1); - } else { - dma_addr_t addr = edesc->dma_link_tbl; + if (!encrypt && is_ipsec_esp) { + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - if (is_sec1) - addr += areq->assoclen + cryptlen; - else - addr += sizeof(struct talitos_ptr) * tbl_off; + /* Add an entry to the link table for ICV data */ + to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); + to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); - to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1); - } + /* icv data follows link tables */ + to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); + to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + sync_needed = true; + } else if (!encrypt) { + to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); + sync_needed = true; } else if (!is_ipsec_esp) { - ret = talitos_sg_map(dev, areq->dst, authsize, edesc, - &desc->ptr[6], sg_count, areq->assoclen + - cryptlen, - tbl_off); - if (ret > 1) { - tbl_off += ret; - edesc->icv_ool = true; - sync_needed = true; - } else { - edesc->icv_ool = false; - } - } else { - edesc->icv_ool = false; + talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], + sg_count, areq->assoclen + cryptlen, tbl_off); } /* iv out */ @@ -1367,7 +1316,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { - ipsec_esp_unmap(dev, edesc, areq); + ipsec_esp_unmap(dev, edesc, areq, encrypt); kfree(edesc); } return ret; @@ -1435,18 +1384,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, * and space for two sets of ICVs (stashed and generated) */ alloc_len = sizeof(struct talitos_edesc); - if (src_nents || dst_nents) { + if (src_nents || dst_nents || !encrypt) { if (is_sec1) dma_len = (src_nents ? src_len : 0) + - (dst_nents ? dst_len : 0); + (dst_nents ? dst_len : 0) + authsize; else dma_len = (src_nents + dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize * 2; + sizeof(struct talitos_ptr) + authsize; alloc_len += dma_len; } else { dma_len = 0; - alloc_len += icv_stashing ? authsize : 0; } + alloc_len += icv_stashing ? authsize : 0; /* if its a ahash, add space for a second desc next to the first one */ if (is_sec1 && !dst) @@ -1466,15 +1415,11 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; - if (dma_len) { - void *addr = &edesc->link_tbl[0]; - - if (is_sec1 && !dst) - addr += sizeof(struct talitos_desc); - edesc->dma_link_tbl = dma_map_single(dev, addr, + if (dma_len) + edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], edesc->dma_len, DMA_BIDIRECTIONAL); - } + return edesc; } @@ -1485,9 +1430,10 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, - iv, areq->assoclen, areq->cryptlen, + iv, areq->assoclen, cryptlen, authsize, ivsize, icv_stashing, areq->base.flags, encrypt); } @@ -1506,7 +1452,7 @@ static int aead_encrypt(struct aead_request *req) /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return ipsec_esp(edesc, req, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); } static int aead_decrypt(struct aead_request *req) @@ -1516,17 +1462,15 @@ static int aead_decrypt(struct aead_request *req) struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct talitos_private *priv = dev_get_drvdata(ctx->dev); struct talitos_edesc *edesc; - struct scatterlist *sg; void *icvdata; - req->cryptlen -= authsize; - /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, req->iv, 1, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); - if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) && + if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && + (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && ((!edesc->src_nents && !edesc->dst_nents) || priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { @@ -1537,24 +1481,20 @@ static int aead_decrypt(struct aead_request *req) /* reset integrity check result bits */ - return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); + return ipsec_esp(edesc, req, false, + ipsec_esp_decrypt_hwauth_done); } /* Have to check the ICV with software */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; /* stash incoming ICV for later cmp with ICV generated by the h/w */ - if (edesc->dma_len) - icvdata = (char *)&edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - else - icvdata = &edesc->link_tbl[0]; + icvdata = edesc->buf + edesc->dma_len; - sg = sg_last(req->src, edesc->src_nents ? : 1); + sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, + req->assoclen + req->cryptlen - authsize); - memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); - - return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done); + return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); } static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, @@ -1605,6 +1545,18 @@ static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher, return ablkcipher_setkey(cipher, key, keylen); } +static int ablkcipher_aes_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || + keylen == AES_KEYSIZE_256) + return ablkcipher_setkey(cipher, key, keylen); + + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + + return -EINVAL; +} + static void common_nonsnoop_unmap(struct device *dev, struct talitos_edesc *edesc, struct ablkcipher_request *areq) @@ -1624,11 +1576,15 @@ static void ablkcipher_done(struct device *dev, int err) { struct ablkcipher_request *areq = context; + struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); struct talitos_edesc *edesc; edesc = container_of(desc, struct talitos_edesc, desc); common_nonsnoop_unmap(dev, edesc, areq); + memcpy(areq->info, ctx->iv, ivsize); kfree(edesc); @@ -1723,6 +1679,14 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq) struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher)); + + if (!areq->nbytes) + return 0; + + if (areq->nbytes % blocksize) + return -EINVAL; /* allocate extended descriptor */ edesc = ablkcipher_edesc_alloc(areq, true); @@ -1740,6 +1704,14 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq) struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher)); + + if (!areq->nbytes) + return 0; + + if (areq->nbytes % blocksize) + return -EINVAL; /* allocate extended descriptor */ edesc = ablkcipher_edesc_alloc(areq, false); @@ -1759,14 +1731,16 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); struct talitos_desc *desc = &edesc->desc; - struct talitos_desc *desc2 = desc + 1; + struct talitos_desc *desc2 = (struct talitos_desc *) + (edesc->buf + edesc->dma_len); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); if (desc->next_desc && desc->ptr[5].ptr != desc2->ptr[5].ptr) unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); - talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); + if (req_ctx->psrc) + talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) @@ -1833,7 +1807,6 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct ahash_request *areq, unsigned int length, - unsigned int offset, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1872,9 +1845,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_pcopy_to_buffer(req_ctx->psrc, sg_count, - edesc->buf + sizeof(struct talitos_desc), - length, req_ctx->nbuf); + sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); else if (length) sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, DMA_TO_DEVICE); @@ -1887,7 +1858,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, DMA_TO_DEVICE); } else { sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, offset, 0); + &desc->ptr[3], sg_count, 0, 0); if (sg_count > 1) sync_needed = true; } @@ -1911,7 +1882,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); if (is_sec1 && req_ctx->nbuf && length) { - struct talitos_desc *desc2 = desc + 1; + struct talitos_desc *desc2 = (struct talitos_desc *) + (edesc->buf + edesc->dma_len); dma_addr_t next_desc; memset(desc2, 0, sizeof(*desc2)); @@ -1932,7 +1904,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, DMA_TO_DEVICE); copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc2->ptr[3], sg_count, offset, 0); + &desc2->ptr[3], sg_count, 0, 0); if (sg_count > 1) sync_needed = true; copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); @@ -2043,7 +2015,6 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) struct device *dev = ctx->dev; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - int offset = 0; u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { @@ -2083,6 +2054,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { + int offset; + if (nbytes_to_hash > blocksize) offset = blocksize - req_ctx->nbuf; else @@ -2095,7 +2068,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_copy_to_buffer(areq->src, nents, ctx_buf + req_ctx->nbuf, offset); req_ctx->nbuf += offset; - req_ctx->psrc = areq->src; + req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src, + offset); } else req_ctx->psrc = areq->src; @@ -2135,8 +2109,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (ctx->keylen && (req_ctx->first || req_ctx->last)) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset, - ahash_done); + return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done); } static int ahash_update(struct ahash_request *areq) @@ -2339,7 +2312,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "authenc-hmac-sha1-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2384,7 +2357,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha1)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha1-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2427,7 +2400,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha224),cbc(aes))", .cra_driver_name = "authenc-hmac-sha224-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2472,7 +2445,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha224)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha224-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2515,7 +2488,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha256),cbc(aes))", .cra_driver_name = "authenc-hmac-sha256-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2560,7 +2533,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha256)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha256-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2689,7 +2662,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(md5),cbc(aes))", .cra_driver_name = "authenc-hmac-md5-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2732,7 +2705,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(md5),cbc(des3_ede))", .cra_driver_name = "authenc-hmac-md5-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2760,7 +2733,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2777,6 +2750,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2787,13 +2761,14 @@ static struct talitos_alg_template driver_algs[] = { .alg.crypto = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | @@ -2810,7 +2785,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, .setkey = ablkcipher_des_setkey, } }, @@ -2845,7 +2819,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, .setkey = ablkcipher_des3_setkey, } }, @@ -3270,7 +3243,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_priority = t_alg->algt.priority; else alg->cra_priority = TALITOS_CRA_PRIORITY; - alg->cra_alignmask = 0; + if (has_ftr_sec1(priv)) + alg->cra_alignmask = 3; + else + alg->cra_alignmask = 0; alg->cra_ctxsize = sizeof(struct talitos_ctx); alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; @@ -3418,7 +3394,7 @@ static int talitos_probe(struct platform_device *ofdev) if (err) goto err_out; - if (of_device_is_compatible(np, "fsl,sec1.0")) { + if (has_ftr_sec1(priv)) { if (priv->num_channels == 1) tasklet_init(&priv->done_task[0], talitos1_done_ch0, (unsigned long)dev); |