summaryrefslogtreecommitdiffstats
path: root/drivers/crypto/hisilicon
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/crypto/hisilicon')
-rw-r--r--drivers/crypto/hisilicon/Kconfig2
-rw-r--r--drivers/crypto/hisilicon/hpre/hpre.h18
-rw-r--r--drivers/crypto/hisilicon/hpre/hpre_crypto.c921
-rw-r--r--drivers/crypto/hisilicon/hpre/hpre_main.c158
-rw-r--r--drivers/crypto/hisilicon/qm.c396
-rw-r--r--drivers/crypto/hisilicon/qm.h29
-rw-r--r--drivers/crypto/hisilicon/sec/sec_algs.c2
-rw-r--r--drivers/crypto/hisilicon/sec/sec_drv.c13
-rw-r--r--drivers/crypto/hisilicon/sec/sec_drv.h2
-rw-r--r--drivers/crypto/hisilicon/sec2/sec.h10
-rw-r--r--drivers/crypto/hisilicon/sec2/sec_crypto.c137
-rw-r--r--drivers/crypto/hisilicon/sec2/sec_crypto.h6
-rw-r--r--drivers/crypto/hisilicon/sec2/sec_main.c267
-rw-r--r--drivers/crypto/hisilicon/sgl.c37
-rw-r--r--drivers/crypto/hisilicon/trng/trng.c13
-rw-r--r--drivers/crypto/hisilicon/zip/zip.h50
-rw-r--r--drivers/crypto/hisilicon/zip/zip_crypto.c698
-rw-r--r--drivers/crypto/hisilicon/zip/zip_main.c99
18 files changed, 2009 insertions, 849 deletions
diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig
index 843192666dc3..e572f9982d4e 100644
--- a/drivers/crypto/hisilicon/Kconfig
+++ b/drivers/crypto/hisilicon/Kconfig
@@ -68,6 +68,8 @@ config CRYPTO_DEV_HISI_HPRE
select CRYPTO_DEV_HISI_QM
select CRYPTO_DH
select CRYPTO_RSA
+ select CRYPTO_CURVE25519
+ select CRYPTO_ECDH
help
Support for HiSilicon HPRE(High Performance RSA Engine)
accelerator, which can accelerate RSA and DH algorithms.
diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h
index 181c109b19f7..e0b4a1982ee9 100644
--- a/drivers/crypto/hisilicon/hpre/hpre.h
+++ b/drivers/crypto/hisilicon/hpre/hpre.h
@@ -10,6 +10,14 @@
#define HPRE_PF_DEF_Q_NUM 64
#define HPRE_PF_DEF_Q_BASE 0
+/*
+ * type used in qm sqc DW6.
+ * 0 - Algorithm which has been supported in V2, like RSA, DH and so on;
+ * 1 - ECC algorithm in V3.
+ */
+#define HPRE_V2_ALG_TYPE 0
+#define HPRE_V3_ECC_ALG_TYPE 1
+
enum {
HPRE_CLUSTER0,
HPRE_CLUSTER1,
@@ -18,7 +26,6 @@ enum {
};
enum hpre_ctrl_dbgfs_file {
- HPRE_CURRENT_QM,
HPRE_CLEAR_ENABLE,
HPRE_CLUSTER_CTRL,
HPRE_DEBUG_FILE_NUM,
@@ -75,6 +82,9 @@ enum hpre_alg_type {
HPRE_ALG_KG_CRT = 0x3,
HPRE_ALG_DH_G2 = 0x4,
HPRE_ALG_DH = 0x5,
+ HPRE_ALG_ECC_MUL = 0xD,
+ /* shared by x25519 and x448, but x448 is not supported now */
+ HPRE_ALG_CURVE25519_MUL = 0x10,
};
struct hpre_sqe {
@@ -92,8 +102,8 @@ struct hpre_sqe {
__le32 rsvd1[_HPRE_SQE_ALIGN_EXT];
};
-struct hisi_qp *hpre_create_qp(void);
-int hpre_algs_register(void);
-void hpre_algs_unregister(void);
+struct hisi_qp *hpre_create_qp(u8 type);
+int hpre_algs_register(struct hisi_qm *qm);
+void hpre_algs_unregister(struct hisi_qm *qm);
#endif
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
index a87f9904087a..a380087c83f7 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c
@@ -1,7 +1,10 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 HiSilicon Limited. */
#include <crypto/akcipher.h>
+#include <crypto/curve25519.h>
#include <crypto/dh.h>
+#include <crypto/ecc_curve.h>
+#include <crypto/ecdh.h>
#include <crypto/internal/akcipher.h>
#include <crypto/internal/kpp.h>
#include <crypto/internal/rsa.h>
@@ -36,6 +39,13 @@ struct hpre_ctx;
#define HPRE_DFX_SEC_TO_US 1000000
#define HPRE_DFX_US_TO_NS 1000
+/* size in bytes of the n prime */
+#define HPRE_ECC_NIST_P192_N_SIZE 24
+#define HPRE_ECC_NIST_P256_N_SIZE 32
+
+/* size in bytes */
+#define HPRE_ECC_HW256_KSZ_B 32
+
typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
struct hpre_rsa_ctx {
@@ -61,14 +71,35 @@ struct hpre_dh_ctx {
* else if base if the counterpart public key we
* compute the shared secret
* ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
+ * low address: d--->n, please refer to Hisilicon HPRE UM
*/
- char *xa_p; /* low address: d--->n, please refer to Hisilicon HPRE UM */
+ char *xa_p;
dma_addr_t dma_xa_p;
char *g; /* m */
dma_addr_t dma_g;
};
+struct hpre_ecdh_ctx {
+ /* low address: p->a->k->b */
+ unsigned char *p;
+ dma_addr_t dma_p;
+
+ /* low address: x->y */
+ unsigned char *g;
+ dma_addr_t dma_g;
+};
+
+struct hpre_curve25519_ctx {
+ /* low address: p->a->k */
+ unsigned char *p;
+ dma_addr_t dma_p;
+
+ /* gx coordinate */
+ unsigned char *g;
+ dma_addr_t dma_g;
+};
+
struct hpre_ctx {
struct hisi_qp *qp;
struct hpre_asym_request **req_list;
@@ -80,7 +111,11 @@ struct hpre_ctx {
union {
struct hpre_rsa_ctx rsa;
struct hpre_dh_ctx dh;
+ struct hpre_ecdh_ctx ecdh;
+ struct hpre_curve25519_ctx curve25519;
};
+ /* for ecc algorithms */
+ unsigned int curve_id;
};
struct hpre_asym_request {
@@ -91,6 +126,8 @@ struct hpre_asym_request {
union {
struct akcipher_request *rsa;
struct kpp_request *dh;
+ struct kpp_request *ecdh;
+ struct kpp_request *curve25519;
} areq;
int err;
int req_id;
@@ -152,12 +189,12 @@ static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
}
}
-static struct hisi_qp *hpre_get_qp_and_start(void)
+static struct hisi_qp *hpre_get_qp_and_start(u8 type)
{
struct hisi_qp *qp;
int ret;
- qp = hpre_create_qp();
+ qp = hpre_create_qp(type);
if (!qp) {
pr_err("Can not create hpre qp!\n");
return ERR_PTR(-ENODEV);
@@ -261,8 +298,6 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
dma_addr_t tmp;
tmp = le64_to_cpu(sqe->in);
- if (unlikely(!tmp))
- return;
if (src) {
if (req->src)
@@ -272,8 +307,6 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
}
tmp = le64_to_cpu(sqe->out);
- if (unlikely(!tmp))
- return;
if (req->dst) {
if (dst)
@@ -288,13 +321,16 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx,
static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
void **kreq)
{
+ struct device *dev = HPRE_DEV(ctx);
struct hpre_asym_request *req;
- int err, id, done;
+ unsigned int err, done, alg;
+ int id;
#define HPRE_NO_HW_ERR 0
#define HPRE_HW_TASK_DONE 3
#define HREE_HW_ERR_MASK 0x7ff
#define HREE_SQE_DONE_MASK 0x3
+#define HREE_ALG_TYPE_MASK 0x1f
id = (int)le16_to_cpu(sqe->tag);
req = ctx->req_list[id];
hpre_rm_req_from_ctx(req);
@@ -307,7 +343,11 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe,
HREE_SQE_DONE_MASK;
if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE))
- return 0;
+ return 0;
+
+ alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK;
+ dev_err_ratelimited(dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n",
+ alg, done, err);
return -EINVAL;
}
@@ -413,7 +453,6 @@ static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
struct hpre_sqe *sqe = resp;
struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)];
-
if (unlikely(!req)) {
atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value);
return;
@@ -422,18 +461,29 @@ static void hpre_alg_cb(struct hisi_qp *qp, void *resp)
req->cb(ctx, resp);
}
-static int hpre_ctx_init(struct hpre_ctx *ctx)
+static void hpre_stop_qp_and_put(struct hisi_qp *qp)
+{
+ hisi_qm_stop_qp(qp);
+ hisi_qm_free_qps(&qp, 1);
+}
+
+static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type)
{
struct hisi_qp *qp;
+ int ret;
- qp = hpre_get_qp_and_start();
+ qp = hpre_get_qp_and_start(type);
if (IS_ERR(qp))
return PTR_ERR(qp);
qp->qp_ctx = ctx;
qp->req_cb = hpre_alg_cb;
- return hpre_ctx_set(ctx, qp, QM_Q_DEPTH);
+ ret = hpre_ctx_set(ctx, qp, QM_Q_DEPTH);
+ if (ret)
+ hpre_stop_qp_and_put(qp);
+
+ return ret;
}
static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa)
@@ -510,7 +560,6 @@ static int hpre_send(struct hpre_ctx *ctx, struct hpre_sqe *msg)
return ret;
}
-#ifdef CONFIG_CRYPTO_DH
static int hpre_dh_compute_value(struct kpp_request *req)
{
struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
@@ -674,7 +723,7 @@ static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
{
struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
- return hpre_ctx_init(ctx);
+ return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
}
static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
@@ -683,7 +732,6 @@ static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
hpre_dh_clear_ctx(ctx, true);
}
-#endif
static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len)
{
@@ -1100,7 +1148,7 @@ static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
return PTR_ERR(ctx->rsa.soft_tfm);
}
- ret = hpre_ctx_init(ctx);
+ ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE);
if (ret)
crypto_free_akcipher(ctx->rsa.soft_tfm);
@@ -1115,6 +1163,734 @@ static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
crypto_free_akcipher(ctx->rsa.soft_tfm);
}
+static void hpre_key_to_big_end(u8 *data, int len)
+{
+ int i, j;
+ u8 tmp;
+
+ for (i = 0; i < len / 2; i++) {
+ j = len - i - 1;
+ tmp = data[j];
+ data[j] = data[i];
+ data[i] = tmp;
+ }
+}
+
+static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all,
+ bool is_ecdh)
+{
+ struct device *dev = HPRE_DEV(ctx);
+ unsigned int sz = ctx->key_sz;
+ unsigned int shift = sz << 1;
+
+ if (is_clear_all)
+ hisi_qm_stop_qp(ctx->qp);
+
+ if (is_ecdh && ctx->ecdh.p) {
+ /* ecdh: p->a->k->b */
+ memzero_explicit(ctx->ecdh.p + shift, sz);
+ dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
+ ctx->ecdh.p = NULL;
+ } else if (!is_ecdh && ctx->curve25519.p) {
+ /* curve25519: p->a->k */
+ memzero_explicit(ctx->curve25519.p + shift, sz);
+ dma_free_coherent(dev, sz << 2, ctx->curve25519.p,
+ ctx->curve25519.dma_p);
+ ctx->curve25519.p = NULL;
+ }
+
+ hpre_ctx_clear(ctx, is_clear_all);
+}
+
+static unsigned int hpre_ecdh_supported_curve(unsigned short id)
+{
+ switch (id) {
+ case ECC_CURVE_NIST_P192:
+ case ECC_CURVE_NIST_P256:
+ return HPRE_ECC_HW256_KSZ_B;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static void fill_curve_param(void *addr, u64 *param, unsigned int cur_sz, u8 ndigits)
+{
+ unsigned int sz = cur_sz - (ndigits - 1) * sizeof(u64);
+ u8 i = 0;
+
+ while (i < ndigits - 1) {
+ memcpy(addr + sizeof(u64) * i, &param[i], sizeof(u64));
+ i++;
+ }
+
+ memcpy(addr + sizeof(u64) * i, &param[ndigits - 1], sz);
+ hpre_key_to_big_end((u8 *)addr, cur_sz);
+}
+
+static int hpre_ecdh_fill_curve(struct hpre_ctx *ctx, struct ecdh *params,
+ unsigned int cur_sz)
+{
+ unsigned int shifta = ctx->key_sz << 1;
+ unsigned int shiftb = ctx->key_sz << 2;
+ void *p = ctx->ecdh.p + ctx->key_sz - cur_sz;
+ void *a = ctx->ecdh.p + shifta - cur_sz;
+ void *b = ctx->ecdh.p + shiftb - cur_sz;
+ void *x = ctx->ecdh.g + ctx->key_sz - cur_sz;
+ void *y = ctx->ecdh.g + shifta - cur_sz;
+ const struct ecc_curve *curve = ecc_get_curve(ctx->curve_id);
+ char *n;
+
+ if (unlikely(!curve))
+ return -EINVAL;
+
+ n = kzalloc(ctx->key_sz, GFP_KERNEL);
+ if (!n)
+ return -ENOMEM;
+
+ fill_curve_param(p, curve->p, cur_sz, curve->g.ndigits);
+ fill_curve_param(a, curve->a, cur_sz, curve->g.ndigits);
+ fill_curve_param(b, curve->b, cur_sz, curve->g.ndigits);
+ fill_curve_param(x, curve->g.x, cur_sz, curve->g.ndigits);
+ fill_curve_param(y, curve->g.y, cur_sz, curve->g.ndigits);
+ fill_curve_param(n, curve->n, cur_sz, curve->g.ndigits);
+
+ if (params->key_size == cur_sz && memcmp(params->key, n, cur_sz) >= 0) {
+ kfree(n);
+ return -EINVAL;
+ }
+
+ kfree(n);
+ return 0;
+}
+
+static unsigned int hpre_ecdh_get_curvesz(unsigned short id)
+{
+ switch (id) {
+ case ECC_CURVE_NIST_P192:
+ return HPRE_ECC_NIST_P192_N_SIZE;
+ case ECC_CURVE_NIST_P256:
+ return HPRE_ECC_NIST_P256_N_SIZE;
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params)
+{
+ struct device *dev = HPRE_DEV(ctx);
+ unsigned int sz, shift, curve_sz;
+ int ret;
+
+ ctx->key_sz = hpre_ecdh_supported_curve(ctx->curve_id);
+ if (!ctx->key_sz)
+ return -EINVAL;
+
+ curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
+ if (!curve_sz || params->key_size > curve_sz)
+ return -EINVAL;
+
+ sz = ctx->key_sz;
+
+ if (!ctx->ecdh.p) {
+ ctx->ecdh.p = dma_alloc_coherent(dev, sz << 3, &ctx->ecdh.dma_p,
+ GFP_KERNEL);
+ if (!ctx->ecdh.p)
+ return -ENOMEM;
+ }
+
+ shift = sz << 2;
+ ctx->ecdh.g = ctx->ecdh.p + shift;
+ ctx->ecdh.dma_g = ctx->ecdh.dma_p + shift;
+
+ ret = hpre_ecdh_fill_curve(ctx, params, curve_sz);
+ if (ret) {
+ dev_err(dev, "failed to fill curve_param, ret = %d!\n", ret);
+ dma_free_coherent(dev, sz << 3, ctx->ecdh.p, ctx->ecdh.dma_p);
+ ctx->ecdh.p = NULL;
+ return ret;
+ }
+
+ return 0;
+}
+
+static bool hpre_key_is_zero(char *key, unsigned short key_sz)
+{
+ int i;
+
+ for (i = 0; i < key_sz; i++)
+ if (key[i])
+ return false;
+
+ return true;
+}
+
+static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
+ unsigned int len)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+ struct device *dev = HPRE_DEV(ctx);
+ unsigned int sz, sz_shift;
+ struct ecdh params;
+ int ret;
+
+ if (crypto_ecdh_decode_key(buf, len, &params) < 0) {
+ dev_err(dev, "failed to decode ecdh key!\n");
+ return -EINVAL;
+ }
+
+ if (hpre_key_is_zero(params.key, params.key_size)) {
+ dev_err(dev, "Invalid hpre key!\n");
+ return -EINVAL;
+ }
+
+ hpre_ecc_clear_ctx(ctx, false, true);
+
+ ret = hpre_ecdh_set_param(ctx, &params);
+ if (ret < 0) {
+ dev_err(dev, "failed to set hpre param, ret = %d!\n", ret);
+ return ret;
+ }
+
+ sz = ctx->key_sz;
+ sz_shift = (sz << 1) + sz - params.key_size;
+ memcpy(ctx->ecdh.p + sz_shift, params.key, params.key_size);
+
+ return 0;
+}
+
+static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx,
+ struct hpre_asym_request *req,
+ struct scatterlist *dst,
+ struct scatterlist *src)
+{
+ struct device *dev = HPRE_DEV(ctx);
+ struct hpre_sqe *sqe = &req->req;
+ dma_addr_t dma;
+
+ dma = le64_to_cpu(sqe->in);
+
+ if (src && req->src)
+ dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma);
+
+ dma = le64_to_cpu(sqe->out);
+
+ if (req->dst)
+ dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma);
+ if (dst)
+ dma_unmap_single(dev, dma, ctx->key_sz << 1, DMA_FROM_DEVICE);
+}
+
+static void hpre_ecdh_cb(struct hpre_ctx *ctx, void *resp)
+{
+ unsigned int curve_sz = hpre_ecdh_get_curvesz(ctx->curve_id);
+ struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
+ struct hpre_asym_request *req = NULL;
+ struct kpp_request *areq;
+ u64 overtime_thrhld;
+ char *p;
+ int ret;
+
+ ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
+ areq = req->areq.ecdh;
+ areq->dst_len = ctx->key_sz << 1;
+
+ overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
+ if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
+ atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
+
+ p = sg_virt(areq->dst);
+ memmove(p, p + ctx->key_sz - curve_sz, curve_sz);
+ memmove(p + curve_sz, p + areq->dst_len - curve_sz, curve_sz);
+
+ hpre_ecdh_hw_data_clr_all(ctx, req, areq->dst, areq->src);
+ kpp_request_complete(areq, ret);
+
+ atomic64_inc(&dfx[HPRE_RECV_CNT].value);
+}
+
+static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx,
+ struct kpp_request *req)
+{
+ struct hpre_asym_request *h_req;
+ struct hpre_sqe *msg;
+ int req_id;
+ void *tmp;
+
+ if (req->dst_len < ctx->key_sz << 1) {
+ req->dst_len = ctx->key_sz << 1;
+ return -EINVAL;
+ }
+
+ tmp = kpp_request_ctx(req);
+ h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ h_req->cb = hpre_ecdh_cb;
+ h_req->areq.ecdh = req;
+ msg = &h_req->req;
+ memset(msg, 0, sizeof(*msg));
+ msg->key = cpu_to_le64(ctx->ecdh.dma_p);
+
+ msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
+ msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
+ h_req->ctx = ctx;
+
+ req_id = hpre_add_req_to_ctx(h_req);
+ if (req_id < 0)
+ return -EBUSY;
+
+ msg->tag = cpu_to_le16((u16)req_id);
+ return 0;
+}
+
+static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req,
+ struct scatterlist *data, unsigned int len)
+{
+ struct hpre_sqe *msg = &hpre_req->req;
+ struct hpre_ctx *ctx = hpre_req->ctx;
+ struct device *dev = HPRE_DEV(ctx);
+ unsigned int tmpshift;
+ dma_addr_t dma = 0;
+ void *ptr;
+ int shift;
+
+ /* Src_data include gx and gy. */
+ shift = ctx->key_sz - (len >> 1);
+ if (unlikely(shift < 0))
+ return -EINVAL;
+
+ ptr = dma_alloc_coherent(dev, ctx->key_sz << 2, &dma, GFP_KERNEL);
+ if (unlikely(!ptr))
+ return -ENOMEM;
+
+ tmpshift = ctx->key_sz << 1;
+ scatterwalk_map_and_copy(ptr + tmpshift, data, 0, len, 0);
+ memcpy(ptr + shift, ptr + tmpshift, len >> 1);
+ memcpy(ptr + ctx->key_sz + shift, ptr + tmpshift + (len >> 1), len >> 1);
+
+ hpre_req->src = ptr;
+ msg->in = cpu_to_le64(dma);
+ return 0;
+}
+
+static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req,
+ struct scatterlist *data, unsigned int len)
+{
+ struct hpre_sqe *msg = &hpre_req->req;
+ struct hpre_ctx *ctx = hpre_req->ctx;
+ struct device *dev = HPRE_DEV(ctx);
+ dma_addr_t dma = 0;
+
+ if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) {
+ dev_err(dev, "data or data length is illegal!\n");
+ return -EINVAL;
+ }
+
+ hpre_req->dst = NULL;
+ dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(dev, dma))) {
+ dev_err(dev, "dma map data err!\n");
+ return -ENOMEM;
+ }
+
+ msg->out = cpu_to_le64(dma);
+ return 0;
+}
+
+static int hpre_ecdh_compute_value(struct kpp_request *req)
+{
+ struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+ struct device *dev = HPRE_DEV(ctx);
+ void *tmp = kpp_request_ctx(req);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_sqe *msg = &hpre_req->req;
+ int ret;
+
+ ret = hpre_ecdh_msg_request_set(ctx, req);
+ if (unlikely(ret)) {
+ dev_err(dev, "failed to set ecdh request, ret = %d!\n", ret);
+ return ret;
+ }
+
+ if (req->src) {
+ ret = hpre_ecdh_src_data_init(hpre_req, req->src, req->src_len);
+ if (unlikely(ret)) {
+ dev_err(dev, "failed to init src data, ret = %d!\n", ret);
+ goto clear_all;
+ }
+ } else {
+ msg->in = cpu_to_le64(ctx->ecdh.dma_g);
+ }
+
+ ret = hpre_ecdh_dst_data_init(hpre_req, req->dst, req->dst_len);
+ if (unlikely(ret)) {
+ dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
+ goto clear_all;
+ }
+
+ msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_ECC_MUL);
+ ret = hpre_send(ctx, msg);
+ if (likely(!ret))
+ return -EINPROGRESS;
+
+clear_all:
+ hpre_rm_req_from_ctx(hpre_req);
+ hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
+ return ret;
+}
+
+static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ /* max size is the pub_key_size, include x and y */
+ return ctx->key_sz << 1;
+}
+
+static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ ctx->curve_id = ECC_CURVE_NIST_P192;
+
+ return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
+}
+
+static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ ctx->curve_id = ECC_CURVE_NIST_P256;
+
+ return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
+}
+
+static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ hpre_ecc_clear_ctx(ctx, true, true);
+}
+
+static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf,
+ unsigned int len)
+{
+ u8 secret[CURVE25519_KEY_SIZE] = { 0 };
+ unsigned int sz = ctx->key_sz;
+ const struct ecc_curve *curve;
+ unsigned int shift = sz << 1;
+ void *p;
+
+ /*
+ * The key from 'buf' is in little-endian, we should preprocess it as
+ * the description in rfc7748: "k[0] &= 248, k[31] &= 127, k[31] |= 64",
+ * then convert it to big endian. Only in this way, the result can be
+ * the same as the software curve-25519 that exists in crypto.
+ */
+ memcpy(secret, buf, len);
+ curve25519_clamp_secret(secret);
+ hpre_key_to_big_end(secret, CURVE25519_KEY_SIZE);
+
+ p = ctx->curve25519.p + sz - len;
+
+ curve = ecc_get_curve25519();
+
+ /* fill curve parameters */
+ fill_curve_param(p, curve->p, len, curve->g.ndigits);
+ fill_curve_param(p + sz, curve->a, len, curve->g.ndigits);
+ memcpy(p + shift, secret, len);
+ fill_curve_param(p + shift + sz, curve->g.x, len, curve->g.ndigits);
+ memzero_explicit(secret, CURVE25519_KEY_SIZE);
+}
+
+static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf,
+ unsigned int len)
+{
+ struct device *dev = HPRE_DEV(ctx);
+ unsigned int sz = ctx->key_sz;
+ unsigned int shift = sz << 1;
+
+ /* p->a->k->gx */
+ if (!ctx->curve25519.p) {
+ ctx->curve25519.p = dma_alloc_coherent(dev, sz << 2,
+ &ctx->curve25519.dma_p,
+ GFP_KERNEL);
+ if (!ctx->curve25519.p)
+ return -ENOMEM;
+ }
+
+ ctx->curve25519.g = ctx->curve25519.p + shift + sz;
+ ctx->curve25519.dma_g = ctx->curve25519.dma_p + shift + sz;
+
+ hpre_curve25519_fill_curve(ctx, buf, len);
+
+ return 0;
+}
+
+static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf,
+ unsigned int len)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+ struct device *dev = HPRE_DEV(ctx);
+ int ret = -EINVAL;
+
+ if (len != CURVE25519_KEY_SIZE ||
+ !crypto_memneq(buf, curve25519_null_point, CURVE25519_KEY_SIZE)) {
+ dev_err(dev, "key is null or key len is not 32bytes!\n");
+ return ret;
+ }
+
+ /* Free old secret if any */
+ hpre_ecc_clear_ctx(ctx, false, false);
+
+ ctx->key_sz = CURVE25519_KEY_SIZE;
+ ret = hpre_curve25519_set_param(ctx, buf, CURVE25519_KEY_SIZE);
+ if (ret) {
+ dev_err(dev, "failed to set curve25519 param, ret = %d!\n", ret);
+ hpre_ecc_clear_ctx(ctx, false, false);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx,
+ struct hpre_asym_request *req,
+ struct scatterlist *dst,
+ struct scatterlist *src)
+{
+ struct device *dev = HPRE_DEV(ctx);
+ struct hpre_sqe *sqe = &req->req;
+ dma_addr_t dma;
+
+ dma = le64_to_cpu(sqe->in);
+
+ if (src && req->src)
+ dma_free_coherent(dev, ctx->key_sz, req->src, dma);
+
+ dma = le64_to_cpu(sqe->out);
+
+ if (req->dst)
+ dma_free_coherent(dev, ctx->key_sz, req->dst, dma);
+ if (dst)
+ dma_unmap_single(dev, dma, ctx->key_sz, DMA_FROM_DEVICE);
+}
+
+static void hpre_curve25519_cb(struct hpre_ctx *ctx, void *resp)
+{
+ struct hpre_dfx *dfx = ctx->hpre->debug.dfx;
+ struct hpre_asym_request *req = NULL;
+ struct kpp_request *areq;
+ u64 overtime_thrhld;
+ int ret;
+
+ ret = hpre_alg_res_post_hf(ctx, resp, (void **)&req);
+ areq = req->areq.curve25519;
+ areq->dst_len = ctx->key_sz;
+
+ overtime_thrhld = atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value);
+ if (overtime_thrhld && hpre_is_bd_timeout(req, overtime_thrhld))
+ atomic64_inc(&dfx[HPRE_OVER_THRHLD_CNT].value);
+
+ hpre_key_to_big_end(sg_virt(areq->dst), CURVE25519_KEY_SIZE);
+
+ hpre_curve25519_hw_data_clr_all(ctx, req, areq->dst, areq->src);
+ kpp_request_complete(areq, ret);
+
+ atomic64_inc(&dfx[HPRE_RECV_CNT].value);
+}
+
+static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx,
+ struct kpp_request *req)
+{
+ struct hpre_asym_request *h_req;
+ struct hpre_sqe *msg;
+ int req_id;
+ void *tmp;
+
+ if (unlikely(req->dst_len < ctx->key_sz)) {
+ req->dst_len = ctx->key_sz;
+ return -EINVAL;
+ }
+
+ tmp = kpp_request_ctx(req);
+ h_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ h_req->cb = hpre_curve25519_cb;
+ h_req->areq.curve25519 = req;
+ msg = &h_req->req;
+ memset(msg, 0, sizeof(*msg));
+ msg->key = cpu_to_le64(ctx->curve25519.dma_p);
+
+ msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT);
+ msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1;
+ h_req->ctx = ctx;
+
+ req_id = hpre_add_req_to_ctx(h_req);
+ if (req_id < 0)
+ return -EBUSY;
+
+ msg->tag = cpu_to_le16((u16)req_id);
+ return 0;
+}
+
+static void hpre_curve25519_src_modulo_p(u8 *ptr)
+{
+ int i;
+
+ for (i = 0; i < CURVE25519_KEY_SIZE - 1; i++)
+ ptr[i] = 0;
+
+ /* The modulus is ptr's last byte minus '0xed'(last byte of p) */
+ ptr[i] -= 0xed;
+}
+
+static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
+ struct scatterlist *data, unsigned int len)
+{
+ struct hpre_sqe *msg = &hpre_req->req;
+ struct hpre_ctx *ctx = hpre_req->ctx;
+ struct device *dev = HPRE_DEV(ctx);
+ u8 p[CURVE25519_KEY_SIZE] = { 0 };
+ const struct ecc_curve *curve;
+ dma_addr_t dma = 0;
+ u8 *ptr;
+
+ if (len != CURVE25519_KEY_SIZE) {
+ dev_err(dev, "sourc_data len is not 32bytes, len = %u!\n", len);
+ return -EINVAL;
+ }
+
+ ptr = dma_alloc_coherent(dev, ctx->key_sz, &dma, GFP_KERNEL);
+ if (unlikely(!ptr))
+ return -ENOMEM;
+
+ scatterwalk_map_and_copy(ptr, data, 0, len, 0);
+
+ if (!crypto_memneq(ptr, curve25519_null_point, CURVE25519_KEY_SIZE)) {
+ dev_err(dev, "gx is null!\n");
+ goto err;
+ }
+
+ /*
+ * Src_data(gx) is in little-endian order, MSB in the final byte should
+ * be masked as described in RFC7748, then transform it to big-endian
+ * form, then hisi_hpre can use the data.
+ */
+ ptr[31] &= 0x7f;
+ hpre_key_to_big_end(ptr, CURVE25519_KEY_SIZE);
+
+ curve = ecc_get_curve25519();
+
+ fill_curve_param(p, curve->p, CURVE25519_KEY_SIZE, curve->g.ndigits);
+
+ /*
+ * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p,
+ * we get its modulus to p, and then use it.
+ */
+ if (memcmp(ptr, p, ctx->key_sz) >= 0)
+ hpre_curve25519_src_modulo_p(ptr);
+
+ hpre_req->src = ptr;
+ msg->in = cpu_to_le64(dma);
+ return 0;
+
+err:
+ dma_free_coherent(dev, ctx->key_sz, ptr, dma);
+ return -EINVAL;
+}
+
+static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req,
+ struct scatterlist *data, unsigned int len)
+{
+ struct hpre_sqe *msg = &hpre_req->req;
+ struct hpre_ctx *ctx = hpre_req->ctx;
+ struct device *dev = HPRE_DEV(ctx);
+ dma_addr_t dma = 0;
+
+ if (!data || !sg_is_last(data) || len != ctx->key_sz) {
+ dev_err(dev, "data or data length is illegal!\n");
+ return -EINVAL;
+ }
+
+ hpre_req->dst = NULL;
+ dma = dma_map_single(dev, sg_virt(data), len, DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(dev, dma))) {
+ dev_err(dev, "dma map data err!\n");
+ return -ENOMEM;
+ }
+
+ msg->out = cpu_to_le64(dma);
+ return 0;
+}
+
+static int hpre_curve25519_compute_value(struct kpp_request *req)
+{
+ struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+ struct device *dev = HPRE_DEV(ctx);
+ void *tmp = kpp_request_ctx(req);
+ struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ);
+ struct hpre_sqe *msg = &hpre_req->req;
+ int ret;
+
+ ret = hpre_curve25519_msg_request_set(ctx, req);
+ if (unlikely(ret)) {
+ dev_err(dev, "failed to set curve25519 request, ret = %d!\n", ret);
+ return ret;
+ }
+
+ if (req->src) {
+ ret = hpre_curve25519_src_init(hpre_req, req->src, req->src_len);
+ if (unlikely(ret)) {
+ dev_err(dev, "failed to init src data, ret = %d!\n",
+ ret);
+ goto clear_all;
+ }
+ } else {
+ msg->in = cpu_to_le64(ctx->curve25519.dma_g);
+ }
+
+ ret = hpre_curve25519_dst_init(hpre_req, req->dst, req->dst_len);
+ if (unlikely(ret)) {
+ dev_err(dev, "failed to init dst data, ret = %d!\n", ret);
+ goto clear_all;
+ }
+
+ msg->dw0 = cpu_to_le32(le32_to_cpu(msg->dw0) | HPRE_ALG_CURVE25519_MUL);
+ ret = hpre_send(ctx, msg);
+ if (likely(!ret))
+ return -EINPROGRESS;
+
+clear_all:
+ hpre_rm_req_from_ctx(hpre_req);
+ hpre_curve25519_hw_data_clr_all(ctx, hpre_req, req->dst, req->src);
+ return ret;
+}
+
+static unsigned int hpre_curve25519_max_size(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ return ctx->key_sz;
+}
+
+static int hpre_curve25519_init_tfm(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE);
+}
+
+static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm)
+{
+ struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+ hpre_ecc_clear_ctx(ctx, true, false);
+}
+
static struct akcipher_alg rsa = {
.sign = hpre_rsa_dec,
.verify = hpre_rsa_enc,
@@ -1135,7 +1911,6 @@ static struct akcipher_alg rsa = {
},
};
-#ifdef CONFIG_CRYPTO_DH
static struct kpp_alg dh = {
.set_secret = hpre_dh_set_secret,
.generate_public_key = hpre_dh_compute_value,
@@ -1152,9 +1927,83 @@ static struct kpp_alg dh = {
.cra_module = THIS_MODULE,
},
};
-#endif
-int hpre_algs_register(void)
+static struct kpp_alg ecdh_nist_p192 = {
+ .set_secret = hpre_ecdh_set_secret,
+ .generate_public_key = hpre_ecdh_compute_value,
+ .compute_shared_secret = hpre_ecdh_compute_value,
+ .max_size = hpre_ecdh_max_size,
+ .init = hpre_ecdh_nist_p192_init_tfm,
+ .exit = hpre_ecdh_exit_tfm,
+ .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
+ .base = {
+ .cra_ctxsize = sizeof(struct hpre_ctx),
+ .cra_priority = HPRE_CRYPTO_ALG_PRI,
+ .cra_name = "ecdh-nist-p192",
+ .cra_driver_name = "hpre-ecdh",
+ .cra_module = THIS_MODULE,
+ },
+};
+
+static struct kpp_alg ecdh_nist_p256 = {
+ .set_secret = hpre_ecdh_set_secret,
+ .generate_public_key = hpre_ecdh_compute_value,
+ .compute_shared_secret = hpre_ecdh_compute_value,
+ .max_size = hpre_ecdh_max_size,
+ .init = hpre_ecdh_nist_p256_init_tfm,
+ .exit = hpre_ecdh_exit_tfm,
+ .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
+ .base = {
+ .cra_ctxsize = sizeof(struct hpre_ctx),
+ .cra_priority = HPRE_CRYPTO_ALG_PRI,
+ .cra_name = "ecdh-nist-p256",
+ .cra_driver_name = "hpre-ecdh",
+ .cra_module = THIS_MODULE,
+ },
+};
+
+static struct kpp_alg curve25519_alg = {
+ .set_secret = hpre_curve25519_set_secret,
+ .generate_public_key = hpre_curve25519_compute_value,
+ .compute_shared_secret = hpre_curve25519_compute_value,
+ .max_size = hpre_curve25519_max_size,
+ .init = hpre_curve25519_init_tfm,
+ .exit = hpre_curve25519_exit_tfm,
+ .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ,
+ .base = {
+ .cra_ctxsize = sizeof(struct hpre_ctx),
+ .cra_priority = HPRE_CRYPTO_ALG_PRI,
+ .cra_name = "curve25519",
+ .cra_driver_name = "hpre-curve25519",
+ .cra_module = THIS_MODULE,
+ },
+};
+
+
+static int hpre_register_ecdh(void)
+{
+ int ret;
+
+ ret = crypto_register_kpp(&ecdh_nist_p192);
+ if (ret)
+ return ret;
+
+ ret = crypto_register_kpp(&ecdh_nist_p256);
+ if (ret) {
+ crypto_unregister_kpp(&ecdh_nist_p192);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void hpre_unregister_ecdh(void)
+{
+ crypto_unregister_kpp(&ecdh_nist_p256);
+ crypto_unregister_kpp(&ecdh_nist_p192);
+}
+
+int hpre_algs_register(struct hisi_qm *qm)
{
int ret;
@@ -1162,19 +2011,37 @@ int hpre_algs_register(void)
ret = crypto_register_akcipher(&rsa);
if (ret)
return ret;
-#ifdef CONFIG_CRYPTO_DH
+
ret = crypto_register_kpp(&dh);
if (ret)
- crypto_unregister_akcipher(&rsa);
-#endif
+ goto unreg_rsa;
+
+ if (qm->ver >= QM_HW_V3) {
+ ret = hpre_register_ecdh();
+ if (ret)
+ goto unreg_dh;
+ ret = crypto_register_kpp(&curve25519_alg);
+ if (ret)
+ goto unreg_ecdh;
+ }
+ return 0;
+unreg_ecdh:
+ hpre_unregister_ecdh();
+unreg_dh:
+ crypto_unregister_kpp(&dh);
+unreg_rsa:
+ crypto_unregister_akcipher(&rsa);
return ret;
}
-void hpre_algs_unregister(void)
+void hpre_algs_unregister(struct hisi_qm *qm)
{
- crypto_unregister_akcipher(&rsa);
-#ifdef CONFIG_CRYPTO_DH
+ if (qm->ver >= QM_HW_V3) {
+ crypto_unregister_kpp(&curve25519_alg);
+ hpre_unregister_ecdh();
+ }
+
crypto_unregister_kpp(&dh);
-#endif
+ crypto_unregister_akcipher(&rsa);
}
diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c
index e7a2c70eb9cf..046bc962c8b2 100644
--- a/drivers/crypto/hisilicon/hpre/hpre_main.c
+++ b/drivers/crypto/hisilicon/hpre/hpre_main.c
@@ -13,7 +13,6 @@
#include <linux/uacce.h>
#include "hpre.h"
-#define HPRE_QUEUE_NUM_V2 1024
#define HPRE_QM_ABNML_INT_MASK 0x100004
#define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0)
#define HPRE_COMM_CNT_CLR_CE 0x0
@@ -119,7 +118,6 @@ static struct hisi_qm_list hpre_devices = {
};
static const char * const hpre_debug_file_name[] = {
- [HPRE_CURRENT_QM] = "current_qm",
[HPRE_CLEAR_ENABLE] = "rdclr_en",
[HPRE_CLUSTER_CTRL] = "cluster_ctrl",
};
@@ -226,41 +224,44 @@ static u32 vfs_num;
module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
-struct hisi_qp *hpre_create_qp(void)
+struct hisi_qp *hpre_create_qp(u8 type)
{
int node = cpu_to_node(smp_processor_id());
struct hisi_qp *qp = NULL;
int ret;
- ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, 0, node, &qp);
+ if (type != HPRE_V2_ALG_TYPE && type != HPRE_V3_ECC_ALG_TYPE)
+ return NULL;
+
+ /*
+ * type: 0 - RSA/DH. algorithm supported in V2,
+ * 1 - ECC algorithm in V3.
+ */
+ ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp);
if (!ret)
return qp;
return NULL;
}
-static void hpre_pasid_enable(struct hisi_qm *qm)
+static void hpre_config_pasid(struct hisi_qm *qm)
{
- u32 val;
-
- val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG);
- val |= BIT(HPRE_PASID_EN_BIT);
- writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG);
- val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG);
- val |= BIT(HPRE_PASID_EN_BIT);
- writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG);
-}
+ u32 val1, val2;
-static void hpre_pasid_disable(struct hisi_qm *qm)
-{
- u32 val;
+ if (qm->ver >= QM_HW_V3)
+ return;
- val = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG);
- val &= ~BIT(HPRE_PASID_EN_BIT);
- writel_relaxed(val, qm->io_base + HPRE_DATA_RUSER_CFG);
- val = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG);
- val &= ~BIT(HPRE_PASID_EN_BIT);
- writel_relaxed(val, qm->io_base + HPRE_DATA_WUSER_CFG);
+ val1 = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG);
+ val2 = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG);
+ if (qm->use_sva) {
+ val1 |= BIT(HPRE_PASID_EN_BIT);
+ val2 |= BIT(HPRE_PASID_EN_BIT);
+ } else {
+ val1 &= ~BIT(HPRE_PASID_EN_BIT);
+ val2 &= ~BIT(HPRE_PASID_EN_BIT);
+ }
+ writel_relaxed(val1, qm->io_base + HPRE_DATA_RUSER_CFG);
+ writel_relaxed(val2, qm->io_base + HPRE_DATA_WUSER_CFG);
}
static int hpre_cfg_by_dsm(struct hisi_qm *qm)
@@ -320,7 +321,7 @@ static int hpre_set_cluster(struct hisi_qm *qm)
}
/*
- * For Kunpeng 920, we shoul disable FLR triggered by hardware (BME/PM/SRIOV).
+ * For Kunpeng 920, we should disable FLR triggered by hardware (BME/PM/SRIOV).
* Or it may stay in D3 state when we bind and unbind hpre quickly,
* as it does FLR triggered by hardware.
*/
@@ -383,15 +384,14 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
if (qm->ver == QM_HW_V2) {
ret = hpre_cfg_by_dsm(qm);
if (ret)
- dev_err(dev, "acpi_evaluate_dsm err.\n");
+ return ret;
disable_flr_of_bme(qm);
-
- /* Enable data buffer pasid */
- if (qm->use_sva)
- hpre_pasid_enable(qm);
}
+ /* Config data buffer pasid needed by Kunpeng 920 */
+ hpre_config_pasid(qm);
+
return ret;
}
@@ -401,10 +401,6 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm)
unsigned long offset;
int i;
- /* clear current_qm */
- writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
- writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
-
/* clear clusterX/cluster_ctrl */
for (i = 0; i < clusters_num; i++) {
offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL;
@@ -456,49 +452,6 @@ static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
return &hpre->qm;
}
-static u32 hpre_current_qm_read(struct hpre_debugfs_file *file)
-{
- struct hisi_qm *qm = hpre_file_to_qm(file);
-
- return readl(qm->io_base + QM_DFX_MB_CNT_VF);
-}
-
-static int hpre_current_qm_write(struct hpre_debugfs_file *file, u32 val)
-{
- struct hisi_qm *qm = hpre_file_to_qm(file);
- u32 num_vfs = qm->vfs_num;
- u32 vfq_num, tmp;
-
- if (val > num_vfs)
- return -EINVAL;
-
- /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
- if (val == 0) {
- qm->debug.curr_qm_qp_num = qm->qp_num;
- } else {
- vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs;
- if (val == num_vfs) {
- qm->debug.curr_qm_qp_num =
- qm->ctrl_qp_num - qm->qp_num - (num_vfs - 1) * vfq_num;
- } else {
- qm->debug.curr_qm_qp_num = vfq_num;
- }
- }
-
- writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
- writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
-
- tmp = val |
- (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
- writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
-
- tmp = val |
- (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
- writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
-
- return 0;
-}
-
static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file)
{
struct hisi_qm *qm = hpre_file_to_qm(file);
@@ -519,7 +472,7 @@ static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val)
~HPRE_CTRL_CNT_CLR_CE_BIT) | val;
writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
- return 0;
+ return 0;
}
static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file)
@@ -541,7 +494,7 @@ static int hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val)
writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY);
- return 0;
+ return 0;
}
static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf,
@@ -554,9 +507,6 @@ static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf,
spin_lock_irq(&file->lock);
switch (file->type) {
- case HPRE_CURRENT_QM:
- val = hpre_current_qm_read(file);
- break;
case HPRE_CLEAR_ENABLE:
val = hpre_clear_enable_read(file);
break;
@@ -597,11 +547,6 @@ static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf,
spin_lock_irq(&file->lock);
switch (file->type) {
- case HPRE_CURRENT_QM:
- ret = hpre_current_qm_write(file, val);
- if (ret)
- goto err_input;
- break;
case HPRE_CLEAR_ENABLE:
ret = hpre_clear_enable_write(file, val);
if (ret)
@@ -740,11 +685,6 @@ static int hpre_ctrl_debug_init(struct hisi_qm *qm)
{
int ret;
- ret = hpre_create_debugfs_file(qm, NULL, HPRE_CURRENT_QM,
- HPRE_CURRENT_QM);
- if (ret)
- return ret;
-
ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE,
HPRE_CLEAR_ENABLE);
if (ret)
@@ -812,9 +752,9 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
}
if (pdev->revision >= QM_HW_V3)
- qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2\n";
+ qm->algs = "rsa\ndh\necdh\nx25519\nx448\necdsa\nsm2";
else
- qm->algs = "rsa\ndh\n";
+ qm->algs = "rsa\ndh";
qm->mode = uacce_mode;
qm->pdev = pdev;
qm->ver = pdev->revision;
@@ -867,6 +807,20 @@ static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB));
}
+static void hpre_err_info_init(struct hisi_qm *qm)
+{
+ struct hisi_qm_err_info *err_info = &qm->err_info;
+
+ err_info->ce = QM_BASE_CE;
+ err_info->fe = 0;
+ err_info->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR |
+ HPRE_OOO_ECC_2BIT_ERR;
+ err_info->dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE;
+ err_info->msi_wr_port = HPRE_WR_MSI_PORT;
+ err_info->acpi_rst = "HRST";
+ err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT;
+}
+
static const struct hisi_qm_err_ini hpre_err_ini = {
.hw_init = hpre_set_user_domain_and_cache,
.hw_err_enable = hpre_hw_error_enable,
@@ -875,16 +829,7 @@ static const struct hisi_qm_err_ini hpre_err_ini = {
.clear_dev_hw_err_status = hpre_clear_hw_err_status,
.log_dev_hw_err = hpre_log_hw_error,
.open_axi_master_ooo = hpre_open_axi_master_ooo,
- .err_info = {
- .ce = QM_BASE_CE,
- .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT,
- .fe = 0,
- .ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR |
- HPRE_OOO_ECC_2BIT_ERR,
- .dev_ce_mask = HPRE_HAC_RAS_CE_ENABLE,
- .msi_wr_port = HPRE_WR_MSI_PORT,
- .acpi_rst = "HRST",
- }
+ .err_info_init = hpre_err_info_init,
};
static int hpre_pf_probe_init(struct hpre *hpre)
@@ -892,13 +837,12 @@ static int hpre_pf_probe_init(struct hpre *hpre)
struct hisi_qm *qm = &hpre->qm;
int ret;
- qm->ctrl_qp_num = HPRE_QUEUE_NUM_V2;
-
ret = hpre_set_user_domain_and_cache(qm);
if (ret)
return ret;
qm->err_ini = &hpre_err_ini;
+ qm->err_ini->err_info_init(qm);
hisi_qm_dev_err_init(qm);
return 0;
@@ -1006,8 +950,6 @@ static void hpre_remove(struct pci_dev *pdev)
hisi_qm_stop(qm, QM_NORMAL);
if (qm->fun_type == QM_HW_PF) {
- if (qm->use_sva && qm->ver == QM_HW_V2)
- hpre_pasid_disable(qm);
hpre_cnt_regs_clear(qm);
qm->debug.curr_qm_qp_num = 0;
hisi_qm_dev_err_uninit(qm);
@@ -1016,7 +958,6 @@ static void hpre_remove(struct pci_dev *pdev)
hisi_qm_uninit(qm);
}
-
static const struct pci_error_handlers hpre_err_handler = {
.error_detected = hisi_qm_dev_err_detected,
.slot_reset = hisi_qm_dev_slot_reset,
@@ -1075,4 +1016,5 @@ module_exit(hpre_exit);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
+MODULE_AUTHOR("Meng Yu <yumeng18@huawei.com>");
MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator");
diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c
index 13cb4216561a..ce439a0c66c9 100644
--- a/drivers/crypto/hisilicon/qm.c
+++ b/drivers/crypto/hisilicon/qm.c
@@ -38,6 +38,7 @@
#define QM_MB_CMD_SQC_BT 0x4
#define QM_MB_CMD_CQC_BT 0x5
#define QM_MB_CMD_SQC_VFT_V2 0x6
+#define QM_MB_CMD_STOP_QP 0x8
#define QM_MB_CMD_SEND_BASE 0x300
#define QM_MB_EVENT_SHIFT 8
@@ -93,6 +94,12 @@
#define QM_DB_PRIORITY_SHIFT_V1 48
#define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000
#define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000
+#define QM_QUE_ISO_CFG_V 0x0030
+#define QM_QUE_ISO_EN 0x100154
+#define QM_CAPBILITY 0x100158
+#define QM_QP_NUN_MASK GENMASK(10, 0)
+#define QM_QP_DB_INTERVAL 0x10000
+#define QM_QP_MAX_NUM_SHIFT 11
#define QM_DB_CMD_SHIFT_V2 12
#define QM_DB_RAND_SHIFT_V2 16
#define QM_DB_INDEX_SHIFT_V2 32
@@ -129,9 +136,9 @@
#define QM_DFX_CNT_CLR_CE 0x100118
#define QM_ABNORMAL_INT_SOURCE 0x100000
-#define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(12, 0)
+#define QM_ABNORMAL_INT_SOURCE_CLR GENMASK(14, 0)
#define QM_ABNORMAL_INT_MASK 0x100004
-#define QM_ABNORMAL_INT_MASK_VALUE 0x1fff
+#define QM_ABNORMAL_INT_MASK_VALUE 0x7fff
#define QM_ABNORMAL_INT_STATUS 0x100008
#define QM_ABNORMAL_INT_SET 0x10000c
#define QM_ABNORMAL_INF00 0x100010
@@ -164,6 +171,14 @@
#define ACC_AM_ROB_ECC_INT_STS 0x300104
#define ACC_ROB_ECC_ERR_MULTPL BIT(1)
+#define QM_DFX_MB_CNT_VF 0x104010
+#define QM_DFX_DB_CNT_VF 0x104020
+#define QM_DFX_SQE_CNT_VF_SQN 0x104030
+#define QM_DFX_CQE_CNT_VF_CQN 0x104040
+#define QM_DFX_QN_SHIFT 16
+#define CURRENT_FUN_MASK GENMASK(5, 0)
+#define CURRENT_Q_MASK GENMASK(31, 16)
+
#define POLL_PERIOD 10
#define POLL_TIMEOUT 1000
#define WAIT_PERIOD_US_MAX 200
@@ -173,6 +188,7 @@
#define QM_CACHE_WB_DONE 0x208
#define PCI_BAR_2 2
+#define PCI_BAR_4 4
#define QM_SQE_DATA_ALIGN_MASK GENMASK(6, 0)
#define QMC_ALIGN(sz) ALIGN(sz, 32)
@@ -334,6 +350,7 @@ struct hisi_qm_hw_ops {
void (*hw_error_init)(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe);
void (*hw_error_uninit)(struct hisi_qm *qm);
enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
+ int (*stop_qp)(struct hisi_qp *qp);
};
struct qm_dfx_item {
@@ -350,6 +367,7 @@ static struct qm_dfx_item qm_dfx_files[] = {
};
static const char * const qm_debug_file_name[] = {
+ [CURRENT_QM] = "current_qm",
[CURRENT_Q] = "current_q",
[CLEAR_ENABLE] = "clear_enable",
};
@@ -373,6 +391,8 @@ static const struct hisi_qm_hw_error qm_hw_error[] = {
{ .int_msk = BIT(10), .msg = "qm_db_timeout" },
{ .int_msk = BIT(11), .msg = "qm_of_fifo_of" },
{ .int_msk = BIT(12), .msg = "qm_db_random_invalid" },
+ { .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },
+ { .int_msk = BIT(14), .msg = "qm_flr_timeout" },
{ /* sentinel */ }
};
@@ -557,21 +577,22 @@ static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
{
- u64 doorbell;
- u64 dbase;
+ void __iomem *io_base = qm->io_base;
u16 randata = 0;
+ u64 doorbell;
if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)
- dbase = QM_DOORBELL_SQ_CQ_BASE_V2;
+ io_base = qm->db_io_base + (u64)qn * qm->db_interval +
+ QM_DOORBELL_SQ_CQ_BASE_V2;
else
- dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;
+ io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;
doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |
((u64)randata << QM_DB_RAND_SHIFT_V2) |
((u64)index << QM_DB_INDEX_SHIFT_V2) |
((u64)priority << QM_DB_PRIORITY_SHIFT_V2);
- writeq(doorbell, qm->io_base + dbase);
+ writeq(doorbell, io_base);
}
static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
@@ -865,6 +886,26 @@ static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)
return 0;
}
+static int qm_get_vf_qp_num(struct hisi_qm *qm, u32 fun_num)
+{
+ u32 remain_q_num, vfq_num;
+ u32 num_vfs = qm->vfs_num;
+
+ vfq_num = (qm->ctrl_qp_num - qm->qp_num) / num_vfs;
+ if (vfq_num >= qm->max_qp_num)
+ return qm->max_qp_num;
+
+ remain_q_num = (qm->ctrl_qp_num - qm->qp_num) % num_vfs;
+ if (vfq_num + remain_q_num <= qm->max_qp_num)
+ return fun_num == num_vfs ? vfq_num + remain_q_num : vfq_num;
+
+ /*
+ * if vfq_num + remain_q_num > max_qp_num, the last VFs,
+ * each with one more queue.
+ */
+ return fun_num + remain_q_num > num_vfs ? vfq_num + 1 : vfq_num;
+}
+
static struct hisi_qm *file_to_qm(struct debugfs_file *file)
{
struct qm_debug *debug = file->debug;
@@ -918,6 +959,41 @@ static int clear_enable_write(struct debugfs_file *file, u32 rd_clr_ctrl)
return 0;
}
+static u32 current_qm_read(struct debugfs_file *file)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+
+ return readl(qm->io_base + QM_DFX_MB_CNT_VF);
+}
+
+static int current_qm_write(struct debugfs_file *file, u32 val)
+{
+ struct hisi_qm *qm = file_to_qm(file);
+ u32 tmp;
+
+ if (val > qm->vfs_num)
+ return -EINVAL;
+
+ /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
+ if (!val)
+ qm->debug.curr_qm_qp_num = qm->qp_num;
+ else
+ qm->debug.curr_qm_qp_num = qm_get_vf_qp_num(qm, val);
+
+ writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
+ writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
+
+ tmp = val |
+ (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
+ writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
+
+ tmp = val |
+ (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
+ writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
+
+ return 0;
+}
+
static ssize_t qm_debug_read(struct file *filp, char __user *buf,
size_t count, loff_t *pos)
{
@@ -929,6 +1005,9 @@ static ssize_t qm_debug_read(struct file *filp, char __user *buf,
mutex_lock(&file->lock);
switch (index) {
+ case CURRENT_QM:
+ val = current_qm_read(file);
+ break;
case CURRENT_Q:
val = current_q_read(file);
break;
@@ -971,27 +1050,24 @@ static ssize_t qm_debug_write(struct file *filp, const char __user *buf,
mutex_lock(&file->lock);
switch (index) {
+ case CURRENT_QM:
+ ret = current_qm_write(file, val);
+ break;
case CURRENT_Q:
ret = current_q_write(file, val);
- if (ret)
- goto err_input;
break;
case CLEAR_ENABLE:
ret = clear_enable_write(file, val);
- if (ret)
- goto err_input;
break;
default:
ret = -EINVAL;
- goto err_input;
}
mutex_unlock(&file->lock);
- return count;
+ if (ret)
+ return ret;
-err_input:
- mutex_unlock(&file->lock);
- return ret;
+ return count;
}
static const struct file_operations qm_debug_fops = {
@@ -1529,12 +1605,12 @@ static const struct file_operations qm_cmd_fops = {
.write = qm_cmd_write,
};
-static void qm_create_debugfs_file(struct hisi_qm *qm, enum qm_debug_file index)
+static void qm_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
+ enum qm_debug_file index)
{
- struct dentry *qm_d = qm->debug.qm_d;
struct debugfs_file *file = qm->debug.files + index;
- debugfs_create_file(qm_debug_file_name[index], 0600, qm_d, file,
+ debugfs_create_file(qm_debug_file_name[index], 0600, dir, file,
&qm_debug_fops);
file->index = index;
@@ -1628,7 +1704,7 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
if (val == (QM_DB_RANDOM_INVALID | QM_BASE_CE)) {
writel(error_status, qm->io_base +
QM_ABNORMAL_INT_SOURCE);
- writel(qm->err_ini->err_info.nfe,
+ writel(qm->err_info.nfe,
qm->io_base + QM_RAS_NFE_ENABLE);
return ACC_ERR_RECOVERED;
}
@@ -1639,6 +1715,11 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
return ACC_ERR_RECOVERED;
}
+static int qm_stop_qp(struct hisi_qp *qp)
+{
+ return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);
+}
+
static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {
.qm_db = qm_db_v1,
.get_irq_num = qm_get_irq_num_v1,
@@ -1654,6 +1735,16 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {
.hw_error_handle = qm_hw_error_handle_v2,
};
+static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {
+ .get_vft = qm_get_vft_v2,
+ .qm_db = qm_db_v2,
+ .get_irq_num = qm_get_irq_num_v2,
+ .hw_error_init = qm_hw_error_init_v2,
+ .hw_error_uninit = qm_hw_error_uninit_v2,
+ .hw_error_handle = qm_hw_error_handle_v2,
+ .stop_qp = qm_stop_qp,
+};
+
static void *qm_get_avail_sqe(struct hisi_qp *qp)
{
struct hisi_qp_status *qp_status = &qp->qp_status;
@@ -1933,6 +2024,14 @@ static int qm_drain_qp(struct hisi_qp *qp)
if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit)
return 0;
+ /* Kunpeng930 supports drain qp by device */
+ if (qm->ops->stop_qp) {
+ ret = qm->ops->stop_qp(qp);
+ if (ret)
+ dev_err(dev, "Failed to stop qp(%u)!\n", qp->qp_id);
+ return ret;
+ }
+
addr = qm_ctx_alloc(qm, size, &dma_addr);
if (IS_ERR(addr)) {
dev_err(dev, "Failed to alloc ctx for sqc and cqc!\n");
@@ -2132,6 +2231,8 @@ static int hisi_qm_uacce_mmap(struct uacce_queue *q,
{
struct hisi_qp *qp = q->priv;
struct hisi_qm *qm = qp->qm;
+ resource_size_t phys_base = qm->db_phys_base +
+ qp->qp_id * qm->db_interval;
size_t sz = vma->vm_end - vma->vm_start;
struct pci_dev *pdev = qm->pdev;
struct device *dev = &pdev->dev;
@@ -2143,16 +2244,19 @@ static int hisi_qm_uacce_mmap(struct uacce_queue *q,
if (qm->ver == QM_HW_V1) {
if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)
return -EINVAL;
- } else {
+ } else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation) {
if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +
QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))
return -EINVAL;
+ } else {
+ if (sz > qm->db_interval)
+ return -EINVAL;
}
vma->vm_flags |= VM_IO;
return remap_pfn_range(vma, vma->vm_start,
- qm->phys_base >> PAGE_SHIFT,
+ phys_base >> PAGE_SHIFT,
sz, pgprot_noncached(vma->vm_page_prot));
case UACCE_QFRT_DUS:
if (sz != qp->qdma.size)
@@ -2267,14 +2371,20 @@ static int qm_alloc_uacce(struct hisi_qm *qm)
uacce->priv = qm;
uacce->algs = qm->algs;
- if (qm->ver == QM_HW_V1) {
- mmio_page_nr = QM_DOORBELL_PAGE_NR;
+ if (qm->ver == QM_HW_V1)
uacce->api_ver = HISI_QM_API_VER_BASE;
- } else {
+ else if (qm->ver == QM_HW_V2)
+ uacce->api_ver = HISI_QM_API_VER2_BASE;
+ else
+ uacce->api_ver = HISI_QM_API_VER3_BASE;
+
+ if (qm->ver == QM_HW_V1)
+ mmio_page_nr = QM_DOORBELL_PAGE_NR;
+ else if (qm->ver == QM_HW_V2 || !qm->use_db_isolation)
mmio_page_nr = QM_DOORBELL_PAGE_NR +
QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;
- uacce->api_ver = HISI_QM_API_VER2_BASE;
- }
+ else
+ mmio_page_nr = qm->db_interval / PAGE_SIZE;
dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * QM_Q_DEPTH +
sizeof(struct qm_cqe) * QM_Q_DEPTH) >> PAGE_SHIFT;
@@ -2482,8 +2592,10 @@ static void hisi_qm_pre_init(struct hisi_qm *qm)
if (qm->ver == QM_HW_V1)
qm->ops = &qm_hw_ops_v1;
- else
+ else if (qm->ver == QM_HW_V2)
qm->ops = &qm_hw_ops_v2;
+ else
+ qm->ops = &qm_hw_ops_v3;
pci_set_drvdata(pdev, qm);
mutex_init(&qm->mailbox_lock);
@@ -2492,13 +2604,23 @@ static void hisi_qm_pre_init(struct hisi_qm *qm)
qm->misc_ctl = false;
}
-static void hisi_qm_pci_uninit(struct hisi_qm *qm)
+static void qm_put_pci_res(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
- pci_free_irq_vectors(pdev);
+ if (qm->use_db_isolation)
+ iounmap(qm->db_io_base);
+
iounmap(qm->io_base);
pci_release_mem_regions(pdev);
+}
+
+static void hisi_qm_pci_uninit(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+
+ pci_free_irq_vectors(pdev);
+ qm_put_pci_res(qm);
pci_disable_device(pdev);
}
@@ -2527,7 +2649,6 @@ void hisi_qm_uninit(struct hisi_qm *qm)
hisi_qm_cache_wb(qm);
dma_free_coherent(dev, qm->qdma.size,
qm->qdma.va, qm->qdma.dma);
- memset(&qm->qdma, 0, sizeof(qm->qdma));
}
qm_irq_unregister(qm);
@@ -2681,7 +2802,7 @@ static int __hisi_qm_start(struct hisi_qm *qm)
{
int ret;
- WARN_ON(!qm->qdma.dma);
+ WARN_ON(!qm->qdma.va);
if (qm->fun_type == QM_HW_PF) {
ret = qm_dev_mem_reset(qm);
@@ -2930,9 +3051,11 @@ void hisi_qm_debug_init(struct hisi_qm *qm)
qm->debug.qm_d = qm_d;
/* only show this in PF */
- if (qm->fun_type == QM_HW_PF)
+ if (qm->fun_type == QM_HW_PF) {
+ qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM);
for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++)
- qm_create_debugfs_file(qm, i);
+ qm_create_debugfs_file(qm, qm_d, i);
+ }
debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops);
@@ -2960,6 +3083,10 @@ void hisi_qm_debug_regs_clear(struct hisi_qm *qm)
struct qm_dfx_registers *regs;
int i;
+ /* clear current_qm */
+ writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
+ writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
+
/* clear current_q */
writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
@@ -2982,7 +3109,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear);
static void qm_hw_error_init(struct hisi_qm *qm)
{
- const struct hisi_qm_err_info *err_info = &qm->err_ini->err_info;
+ struct hisi_qm_err_info *err_info = &qm->err_info;
if (!qm->ops->hw_error_init) {
dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");
@@ -3175,30 +3302,46 @@ EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);
static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)
{
- u32 remain_q_num, q_num, i, j;
+ u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;
+ u32 max_qp_num = qm->max_qp_num;
u32 q_base = qm->qp_num;
int ret;
if (!num_vfs)
return -EINVAL;
- remain_q_num = qm->ctrl_qp_num - qm->qp_num;
+ vfs_q_num = qm->ctrl_qp_num - qm->qp_num;
- /* If remain queues not enough, return error. */
- if (qm->ctrl_qp_num < qm->qp_num || remain_q_num < num_vfs)
+ /* If vfs_q_num is less than num_vfs, return error. */
+ if (vfs_q_num < num_vfs)
return -EINVAL;
- q_num = remain_q_num / num_vfs;
- for (i = 1; i <= num_vfs; i++) {
- if (i == num_vfs)
- q_num += remain_q_num % num_vfs;
- ret = hisi_qm_set_vft(qm, i, q_base, q_num);
+ q_num = vfs_q_num / num_vfs;
+ remain_q_num = vfs_q_num % num_vfs;
+
+ for (i = num_vfs; i > 0; i--) {
+ /*
+ * if q_num + remain_q_num > max_qp_num in last vf, divide the
+ * remaining queues equally.
+ */
+ if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {
+ act_q_num = q_num + remain_q_num;
+ remain_q_num = 0;
+ } else if (remain_q_num > 0) {
+ act_q_num = q_num + 1;
+ remain_q_num--;
+ } else {
+ act_q_num = q_num;
+ }
+
+ act_q_num = min_t(int, act_q_num, max_qp_num);
+ ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);
if (ret) {
- for (j = i; j > 0; j--)
+ for (j = num_vfs; j > i; j--)
hisi_qm_set_vft(qm, j, 0, 0);
return ret;
}
- q_base += q_num;
+ q_base += act_q_num;
}
return 0;
@@ -3318,15 +3461,15 @@ static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)
/* get device hardware error status */
err_sts = qm->err_ini->get_dev_hw_err_status(qm);
if (err_sts) {
- if (err_sts & qm->err_ini->err_info.ecc_2bits_mask)
+ if (err_sts & qm->err_info.ecc_2bits_mask)
qm->err_status.is_dev_ecc_mbit = true;
if (qm->err_ini->log_dev_hw_err)
qm->err_ini->log_dev_hw_err(qm, err_sts);
/* ce error does not need to be reset */
- if ((err_sts | qm->err_ini->err_info.dev_ce_mask) ==
- qm->err_ini->err_info.dev_ce_mask) {
+ if ((err_sts | qm->err_info.dev_ce_mask) ==
+ qm->err_info.dev_ce_mask) {
if (qm->err_ini->clear_dev_hw_err_status)
qm->err_ini->clear_dev_hw_err_status(qm,
err_sts);
@@ -3639,7 +3782,7 @@ static int qm_soft_reset(struct hisi_qm *qm)
acpi_status s;
s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),
- qm->err_ini->err_info.acpi_rst,
+ qm->err_info.acpi_rst,
NULL, &value);
if (ACPI_FAILURE(s)) {
pci_err(pdev, "NO controller reset method!\n");
@@ -3707,12 +3850,11 @@ static void qm_restart_prepare(struct hisi_qm *qm)
/* temporarily close the OOO port used for PEH to write out MSI */
value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
- writel(value & ~qm->err_ini->err_info.msi_wr_port,
+ writel(value & ~qm->err_info.msi_wr_port,
qm->io_base + ACC_AM_CFG_PORT_WR_EN);
/* clear dev ecc 2bit error source if having */
- value = qm_get_dev_err_status(qm) &
- qm->err_ini->err_info.ecc_2bits_mask;
+ value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;
if (value && qm->err_ini->clear_dev_hw_err_status)
qm->err_ini->clear_dev_hw_err_status(qm, value);
@@ -3736,7 +3878,7 @@ static void qm_restart_done(struct hisi_qm *qm)
/* open the OOO port for PEH to write out MSI */
value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);
- value |= qm->err_ini->err_info.msi_wr_port;
+ value |= qm->err_info.msi_wr_port;
writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);
qm->err_status.is_qm_ecc_mbit = false;
@@ -3875,8 +4017,7 @@ static int qm_check_dev_error(struct hisi_qm *qm)
if (ret)
return ret;
- return (qm_get_dev_err_status(qm) &
- qm->err_ini->err_info.ecc_2bits_mask);
+ return (qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask);
}
void hisi_qm_reset_prepare(struct pci_dev *pdev)
@@ -4084,7 +4225,7 @@ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
mutex_unlock(&qm_list->lock);
if (flag) {
- ret = qm_list->register_to_crypto();
+ ret = qm_list->register_to_crypto(qm);
if (ret) {
mutex_lock(&qm_list->lock);
list_del(&qm->list);
@@ -4115,59 +4256,134 @@ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list)
mutex_unlock(&qm_list->lock);
if (list_empty(&qm_list->list))
- qm_list->unregister_from_crypto();
+ qm_list->unregister_from_crypto(qm);
}
EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);
-static int hisi_qm_pci_init(struct hisi_qm *qm)
+static int qm_get_qp_num(struct hisi_qm *qm)
+{
+ if (qm->ver == QM_HW_V1)
+ qm->ctrl_qp_num = QM_QNUM_V1;
+ else if (qm->ver == QM_HW_V2)
+ qm->ctrl_qp_num = QM_QNUM_V2;
+ else
+ qm->ctrl_qp_num = readl(qm->io_base + QM_CAPBILITY) &
+ QM_QP_NUN_MASK;
+
+ if (qm->use_db_isolation)
+ qm->max_qp_num = (readl(qm->io_base + QM_CAPBILITY) >>
+ QM_QP_MAX_NUM_SHIFT) & QM_QP_NUN_MASK;
+ else
+ qm->max_qp_num = qm->ctrl_qp_num;
+
+ /* check if qp number is valid */
+ if (qm->qp_num > qm->max_qp_num) {
+ dev_err(&qm->pdev->dev, "qp num(%u) is more than max qp num(%u)!\n",
+ qm->qp_num, qm->max_qp_num);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int qm_get_pci_res(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
struct device *dev = &pdev->dev;
- unsigned int num_vec;
int ret;
- ret = pci_enable_device_mem(pdev);
- if (ret < 0) {
- dev_err(dev, "Failed to enable device mem!\n");
- return ret;
- }
-
ret = pci_request_mem_regions(pdev, qm->dev_name);
if (ret < 0) {
dev_err(dev, "Failed to request mem regions!\n");
- goto err_disable_pcidev;
+ return ret;
}
qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);
- qm->phys_size = pci_resource_len(qm->pdev, PCI_BAR_2);
- qm->io_base = ioremap(qm->phys_base, qm->phys_size);
+ qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));
if (!qm->io_base) {
ret = -EIO;
- goto err_release_mem_regions;
+ goto err_request_mem_regions;
+ }
+
+ if (qm->ver > QM_HW_V2) {
+ if (qm->fun_type == QM_HW_PF)
+ qm->use_db_isolation = readl(qm->io_base +
+ QM_QUE_ISO_EN) & BIT(0);
+ else
+ qm->use_db_isolation = readl(qm->io_base +
+ QM_QUE_ISO_CFG_V) & BIT(0);
+ }
+
+ if (qm->use_db_isolation) {
+ qm->db_interval = QM_QP_DB_INTERVAL;
+ qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);
+ qm->db_io_base = ioremap(qm->db_phys_base,
+ pci_resource_len(pdev, PCI_BAR_4));
+ if (!qm->db_io_base) {
+ ret = -EIO;
+ goto err_ioremap;
+ }
+ } else {
+ qm->db_phys_base = qm->phys_base;
+ qm->db_io_base = qm->io_base;
+ qm->db_interval = 0;
}
+ if (qm->fun_type == QM_HW_PF) {
+ ret = qm_get_qp_num(qm);
+ if (ret)
+ goto err_db_ioremap;
+ }
+
+ return 0;
+
+err_db_ioremap:
+ if (qm->use_db_isolation)
+ iounmap(qm->db_io_base);
+err_ioremap:
+ iounmap(qm->io_base);
+err_request_mem_regions:
+ pci_release_mem_regions(pdev);
+ return ret;
+}
+
+static int hisi_qm_pci_init(struct hisi_qm *qm)
+{
+ struct pci_dev *pdev = qm->pdev;
+ struct device *dev = &pdev->dev;
+ unsigned int num_vec;
+ int ret;
+
+ ret = pci_enable_device_mem(pdev);
+ if (ret < 0) {
+ dev_err(dev, "Failed to enable device mem!\n");
+ return ret;
+ }
+
+ ret = qm_get_pci_res(qm);
+ if (ret)
+ goto err_disable_pcidev;
+
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
if (ret < 0)
- goto err_iounmap;
+ goto err_get_pci_res;
pci_set_master(pdev);
if (!qm->ops->get_irq_num) {
ret = -EOPNOTSUPP;
- goto err_iounmap;
+ goto err_get_pci_res;
}
num_vec = qm->ops->get_irq_num(qm);
ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);
if (ret < 0) {
dev_err(dev, "Failed to enable MSI vectors!\n");
- goto err_iounmap;
+ goto err_get_pci_res;
}
return 0;
-err_iounmap:
- iounmap(qm->io_base);
-err_release_mem_regions:
- pci_release_mem_regions(pdev);
+err_get_pci_res:
+ qm_put_pci_res(qm);
err_disable_pcidev:
pci_disable_device(pdev);
return ret;
@@ -4187,28 +4403,28 @@ int hisi_qm_init(struct hisi_qm *qm)
hisi_qm_pre_init(qm);
- ret = qm_alloc_uacce(qm);
- if (ret < 0)
- dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
-
ret = hisi_qm_pci_init(qm);
if (ret)
- goto err_remove_uacce;
+ return ret;
ret = qm_irq_register(qm);
if (ret)
- goto err_pci_uninit;
+ goto err_pci_init;
if (qm->fun_type == QM_HW_VF && qm->ver != QM_HW_V1) {
/* v2 starts to support get vft by mailbox */
ret = hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);
if (ret)
- goto err_irq_unregister;
+ goto err_irq_register;
}
+ ret = qm_alloc_uacce(qm);
+ if (ret < 0)
+ dev_warn(dev, "fail to alloc uacce (%d)\n", ret);
+
ret = hisi_qm_memory_init(qm);
if (ret)
- goto err_irq_unregister;
+ goto err_alloc_uacce;
INIT_WORK(&qm->work, qm_work_process);
if (qm->fun_type == QM_HW_PF)
@@ -4218,13 +4434,13 @@ int hisi_qm_init(struct hisi_qm *qm)
return 0;
-err_irq_unregister:
- qm_irq_unregister(qm);
-err_pci_uninit:
- hisi_qm_pci_uninit(qm);
-err_remove_uacce:
+err_alloc_uacce:
uacce_remove(qm->uacce);
qm->uacce = NULL;
+err_irq_register:
+ qm_irq_unregister(qm);
+err_pci_init:
+ hisi_qm_pci_uninit(qm);
return ret;
}
EXPORT_SYMBOL_GPL(hisi_qm_init);
diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h
index 54967c6b9c78..acefdf8b3a50 100644
--- a/drivers/crypto/hisilicon/qm.h
+++ b/drivers/crypto/hisilicon/qm.h
@@ -51,14 +51,6 @@
#define PEH_AXUSER_CFG 0x401001
#define PEH_AXUSER_CFG_ENABLE 0xffffffff
-#define QM_DFX_MB_CNT_VF 0x104010
-#define QM_DFX_DB_CNT_VF 0x104020
-#define QM_DFX_SQE_CNT_VF_SQN 0x104030
-#define QM_DFX_CQE_CNT_VF_CQN 0x104040
-#define QM_DFX_QN_SHIFT 16
-#define CURRENT_FUN_MASK GENMASK(5, 0)
-#define CURRENT_Q_MASK GENMASK(31, 16)
-
#define QM_AXI_RRESP BIT(0)
#define QM_AXI_BRESP BIT(1)
#define QM_ECC_MBIT BIT(2)
@@ -72,10 +64,13 @@
#define QM_DB_TIMEOUT BIT(10)
#define QM_OF_FIFO_OF BIT(11)
#define QM_DB_RANDOM_INVALID BIT(12)
+#define QM_MAILBOX_TIMEOUT BIT(13)
+#define QM_FLR_TIMEOUT BIT(14)
#define QM_BASE_NFE (QM_AXI_RRESP | QM_AXI_BRESP | QM_ECC_MBIT | \
QM_ACC_GET_TASK_TIMEOUT | QM_DB_TIMEOUT | \
- QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID)
+ QM_OF_FIFO_OF | QM_DB_RANDOM_INVALID | \
+ QM_MAILBOX_TIMEOUT | QM_FLR_TIMEOUT)
#define QM_BASE_CE QM_ECC_1BIT
#define QM_Q_DEPTH 1024
@@ -123,6 +118,7 @@ enum qm_fun_type {
};
enum qm_debug_file {
+ CURRENT_QM,
CURRENT_Q,
CLEAR_ENABLE,
DEBUG_FILE_NUM,
@@ -193,14 +189,14 @@ struct hisi_qm_err_ini {
void (*open_axi_master_ooo)(struct hisi_qm *qm);
void (*close_axi_master_ooo)(struct hisi_qm *qm);
void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts);
- struct hisi_qm_err_info err_info;
+ void (*err_info_init)(struct hisi_qm *qm);
};
struct hisi_qm_list {
struct mutex lock;
struct list_head list;
- int (*register_to_crypto)(void);
- void (*unregister_from_crypto)(void);
+ int (*register_to_crypto)(struct hisi_qm *qm);
+ void (*unregister_from_crypto)(struct hisi_qm *qm);
};
struct hisi_qm {
@@ -209,12 +205,15 @@ struct hisi_qm {
const char *dev_name;
struct pci_dev *pdev;
void __iomem *io_base;
+ void __iomem *db_io_base;
u32 sqe_size;
u32 qp_base;
u32 qp_num;
u32 qp_in_used;
u32 ctrl_qp_num;
+ u32 max_qp_num;
u32 vfs_num;
+ u32 db_interval;
struct list_head list;
struct hisi_qm_list *qm_list;
@@ -230,6 +229,7 @@ struct hisi_qm {
struct hisi_qm_status status;
const struct hisi_qm_err_ini *err_ini;
+ struct hisi_qm_err_info err_info;
struct hisi_qm_err_status err_status;
unsigned long misc_ctl; /* driver removing and reset sched */
@@ -252,8 +252,11 @@ struct hisi_qm {
const char *algs;
bool use_sva;
bool is_frozen;
+
+ /* doorbell isolation enable */
+ bool use_db_isolation;
resource_size_t phys_base;
- resource_size_t phys_size;
+ resource_size_t db_phys_base;
struct uacce_device *uacce;
int mode;
};
diff --git a/drivers/crypto/hisilicon/sec/sec_algs.c b/drivers/crypto/hisilicon/sec/sec_algs.c
index 8ca945ac297e..0a3c8f019b02 100644
--- a/drivers/crypto/hisilicon/sec/sec_algs.c
+++ b/drivers/crypto/hisilicon/sec/sec_algs.c
@@ -1,5 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
-/* Copyright (c) 2016-2017 Hisilicon Limited. */
+/* Copyright (c) 2016-2017 HiSilicon Limited. */
#include <linux/crypto.h>
#include <linux/dma-mapping.h>
#include <linux/dmapool.h>
diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c
index 91ee2bb575df..c8de1b51c843 100644
--- a/drivers/crypto/hisilicon/sec/sec_drv.c
+++ b/drivers/crypto/hisilicon/sec/sec_drv.c
@@ -1,8 +1,8 @@
// SPDX-License-Identifier: GPL-2.0
/*
- * Driver for the Hisilicon SEC units found on Hip06 Hip07
+ * Driver for the HiSilicon SEC units found on Hip06 Hip07
*
- * Copyright (c) 2016-2017 Hisilicon Limited.
+ * Copyright (c) 2016-2017 HiSilicon Limited.
*/
#include <linux/acpi.h>
#include <linux/atomic.h>
@@ -233,7 +233,7 @@ static int sec_queue_map_io(struct sec_queue *queue)
IORESOURCE_MEM,
2 + queue->queue_id);
if (!res) {
- dev_err(dev, "Failed to get queue %d memory resource\n",
+ dev_err(dev, "Failed to get queue %u memory resource\n",
queue->queue_id);
return -ENOMEM;
}
@@ -653,12 +653,12 @@ static int sec_queue_free(struct sec_queue *queue)
struct sec_dev_info *info = queue->dev_info;
if (queue->queue_id >= SEC_Q_NUM) {
- dev_err(info->dev, "No queue %d\n", queue->queue_id);
+ dev_err(info->dev, "No queue %u\n", queue->queue_id);
return -ENODEV;
}
if (!queue->in_use) {
- dev_err(info->dev, "Queue %d is idle\n", queue->queue_id);
+ dev_err(info->dev, "Queue %u is idle\n", queue->queue_id);
return -ENODEV;
}
@@ -834,6 +834,7 @@ int sec_queue_stop_release(struct sec_queue *queue)
/**
* sec_queue_empty() - Is this hardware queue currently empty.
+ * @queue: The queue to test
*
* We need to know if we have an empty queue for some of the chaining modes
* as if it is not empty we may need to hold the message in a software queue
@@ -1315,6 +1316,6 @@ static struct platform_driver sec_driver = {
module_platform_driver(sec_driver);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Hisilicon Security Accelerators");
+MODULE_DESCRIPTION("HiSilicon Security Accelerators");
MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com");
MODULE_AUTHOR("Jonathan Cameron <jonathan.cameron@huawei.com>");
diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h
index 4d9063a8b10b..179a8250d691 100644
--- a/drivers/crypto/hisilicon/sec/sec_drv.h
+++ b/drivers/crypto/hisilicon/sec/sec_drv.h
@@ -1,5 +1,5 @@
/* SPDX-License-Identifier: GPL-2.0 */
-/* Copyright (c) 2016-2017 Hisilicon Limited. */
+/* Copyright (c) 2016-2017 HiSilicon Limited. */
#ifndef _SEC_DRV_H_
#define _SEC_DRV_H_
diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h
index 08491912afd5..dfdce2f21e65 100644
--- a/drivers/crypto/hisilicon/sec2/sec.h
+++ b/drivers/crypto/hisilicon/sec2/sec.h
@@ -4,8 +4,6 @@
#ifndef __HISI_SEC_V2_H
#define __HISI_SEC_V2_H
-#include <linux/list.h>
-
#include "../qm.h"
#include "sec_crypto.h"
@@ -50,7 +48,7 @@ struct sec_req {
int err_type;
int req_id;
- int flag;
+ u32 flag;
/* Status of the SEC request */
bool fake_busy;
@@ -139,6 +137,7 @@ struct sec_ctx {
bool pbuf_supported;
struct sec_cipher_ctx c_ctx;
struct sec_auth_ctx a_ctx;
+ struct device *dev;
};
enum sec_endian {
@@ -148,7 +147,6 @@ enum sec_endian {
};
enum sec_debug_file_index {
- SEC_CURRENT_QM,
SEC_CLEAR_ENABLE,
SEC_DEBUG_FILE_NUM,
};
@@ -183,6 +181,6 @@ struct sec_dev {
void sec_destroy_qps(struct hisi_qp **qps, int qp_num);
struct hisi_qp **sec_create_qps(void);
-int sec_register_to_crypto(void);
-void sec_unregister_from_crypto(void);
+int sec_register_to_crypto(struct hisi_qm *qm);
+void sec_unregister_from_crypto(struct hisi_qm *qm);
#endif
diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c
index 2eaa516b3231..133aede8bf07 100644
--- a/drivers/crypto/hisilicon/sec2/sec_crypto.c
+++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c
@@ -7,6 +7,7 @@
#include <crypto/des.h>
#include <crypto/hash.h>
#include <crypto/internal/aead.h>
+#include <crypto/internal/des.h>
#include <crypto/sha1.h>
#include <crypto/sha2.h>
#include <crypto/skcipher.h>
@@ -43,7 +44,6 @@
#define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH)
#define SEC_SGL_SGE_NR 128
-#define SEC_CTX_DEV(ctx) (&(ctx)->sec->qm.pdev->dev)
#define SEC_CIPHER_AUTH 0xfe
#define SEC_AUTH_CIPHER 0x1
#define SEC_MAX_MAC_LEN 64
@@ -96,7 +96,7 @@ static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
0, QM_Q_DEPTH, GFP_ATOMIC);
mutex_unlock(&qp_ctx->req_lock);
if (unlikely(req_id < 0)) {
- dev_err(SEC_CTX_DEV(req->ctx), "alloc req id fail!\n");
+ dev_err(req->ctx->dev, "alloc req id fail!\n");
return req_id;
}
@@ -112,7 +112,7 @@ static void sec_free_req_id(struct sec_req *req)
int req_id = req->req_id;
if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) {
- dev_err(SEC_CTX_DEV(req->ctx), "free request id invalid!\n");
+ dev_err(req->ctx->dev, "free request id invalid!\n");
return;
}
@@ -138,7 +138,7 @@ static int sec_aead_verify(struct sec_req *req)
aead_req->cryptlen + aead_req->assoclen -
authsize);
if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) {
- dev_err(SEC_CTX_DEV(req->ctx), "aead verify failure!\n");
+ dev_err(req->ctx->dev, "aead verify failure!\n");
return -EBADMSG;
}
@@ -177,7 +177,7 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp)
if (unlikely(req->err_type || done != SEC_SQE_DONE ||
(ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) ||
(ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) {
- dev_err(SEC_CTX_DEV(ctx),
+ dev_err_ratelimited(ctx->dev,
"err_type[%d],done[%d],flag[%d]\n",
req->err_type, done, flag);
err = -EIO;
@@ -326,8 +326,8 @@ static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res)
static int sec_alg_resource_alloc(struct sec_ctx *ctx,
struct sec_qp_ctx *qp_ctx)
{
- struct device *dev = SEC_CTX_DEV(ctx);
struct sec_alg_res *res = qp_ctx->res;
+ struct device *dev = ctx->dev;
int ret;
ret = sec_alloc_civ_resource(dev, res);
@@ -360,7 +360,7 @@ alloc_fail:
static void sec_alg_resource_free(struct sec_ctx *ctx,
struct sec_qp_ctx *qp_ctx)
{
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
sec_free_civ_resource(dev, qp_ctx->res);
@@ -373,7 +373,7 @@ static void sec_alg_resource_free(struct sec_ctx *ctx,
static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx,
int qp_ctx_id, int alg_type)
{
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
struct sec_qp_ctx *qp_ctx;
struct hisi_qp *qp;
int ret = -ENOMEM;
@@ -428,7 +428,7 @@ err_destroy_idr:
static void sec_release_qp_ctx(struct sec_ctx *ctx,
struct sec_qp_ctx *qp_ctx)
{
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
hisi_qm_stop_qp(qp_ctx->qp);
sec_alg_resource_free(ctx, qp_ctx);
@@ -452,6 +452,7 @@ static int sec_ctx_base_init(struct sec_ctx *ctx)
sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm);
ctx->sec = sec;
+ ctx->dev = &sec->qm.pdev->dev;
ctx->hlf_q_num = sec->ctx_q_num >> 1;
ctx->pbuf_supported = ctx->sec->iommu_used;
@@ -476,11 +477,9 @@ static int sec_ctx_base_init(struct sec_ctx *ctx)
err_sec_release_qp_ctx:
for (i = i - 1; i >= 0; i--)
sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
-
kfree(ctx->qp_ctx);
err_destroy_qps:
sec_destroy_qps(ctx->qps, sec->ctx_q_num);
-
return ret;
}
@@ -499,7 +498,7 @@ static int sec_cipher_init(struct sec_ctx *ctx)
{
struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
- c_ctx->c_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
+ c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
&c_ctx->c_key_dma, GFP_KERNEL);
if (!c_ctx->c_key)
return -ENOMEM;
@@ -512,7 +511,7 @@ static void sec_cipher_uninit(struct sec_ctx *ctx)
struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
- dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
+ dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
c_ctx->c_key, c_ctx->c_key_dma);
}
@@ -520,7 +519,7 @@ static int sec_auth_init(struct sec_ctx *ctx)
{
struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
- a_ctx->a_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
+ a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
&a_ctx->a_key_dma, GFP_KERNEL);
if (!a_ctx->a_key)
return -ENOMEM;
@@ -533,7 +532,7 @@ static void sec_auth_uninit(struct sec_ctx *ctx)
struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE);
- dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
+ dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE,
a_ctx->a_key, a_ctx->a_key_dma);
}
@@ -546,7 +545,7 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
- dev_err(SEC_CTX_DEV(ctx), "get error skcipher iv size!\n");
+ pr_err("get error skcipher iv size!\n");
return -EINVAL;
}
@@ -573,10 +572,18 @@ static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
sec_ctx_base_uninit(ctx);
}
-static int sec_skcipher_3des_setkey(struct sec_cipher_ctx *c_ctx,
+static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key,
const u32 keylen,
const enum sec_cmode c_mode)
{
+ struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
+ int ret;
+
+ ret = verify_skcipher_des3_key(tfm, key);
+ if (ret)
+ return ret;
+
switch (keylen) {
case SEC_DES3_2KEY_SIZE:
c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
@@ -633,12 +640,13 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
{
struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
+ struct device *dev = ctx->dev;
int ret;
if (c_mode == SEC_CMODE_XTS) {
ret = xts_verify_key(tfm, key, keylen);
if (ret) {
- dev_err(SEC_CTX_DEV(ctx), "xts mode key err!\n");
+ dev_err(dev, "xts mode key err!\n");
return ret;
}
}
@@ -648,7 +656,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
switch (c_alg) {
case SEC_CALG_3DES:
- ret = sec_skcipher_3des_setkey(c_ctx, keylen, c_mode);
+ ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode);
break;
case SEC_CALG_AES:
case SEC_CALG_SM4:
@@ -659,7 +667,7 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
}
if (ret) {
- dev_err(SEC_CTX_DEV(ctx), "set sec key err!\n");
+ dev_err(dev, "set sec key err!\n");
return ret;
}
@@ -691,7 +699,7 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
struct aead_request *aead_req = req->aead_req.aead_req;
struct sec_cipher_req *c_req = &req->c_req;
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
int copy_size, pbuf_length;
int req_id = req->req_id;
@@ -701,21 +709,14 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
copy_size = c_req->c_len;
pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
- qp_ctx->res[req_id].pbuf,
- copy_size);
-
+ qp_ctx->res[req_id].pbuf,
+ copy_size);
if (unlikely(pbuf_length != copy_size)) {
dev_err(dev, "copy src data to pbuf error!\n");
return -EINVAL;
}
c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma;
-
- if (!c_req->c_in_dma) {
- dev_err(dev, "fail to set pbuffer address!\n");
- return -ENOMEM;
- }
-
c_req->c_out_dma = c_req->c_in_dma;
return 0;
@@ -727,7 +728,7 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
struct aead_request *aead_req = req->aead_req.aead_req;
struct sec_cipher_req *c_req = &req->c_req;
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
int copy_size, pbuf_length;
int req_id = req->req_id;
@@ -739,7 +740,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req,
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
qp_ctx->res[req_id].pbuf,
copy_size);
-
if (unlikely(pbuf_length != copy_size))
dev_err(dev, "copy pbuf data to dst error!\n");
}
@@ -751,7 +751,7 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
struct sec_aead_req *a_req = &req->aead_req;
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
struct sec_alg_res *res = &qp_ctx->res[req->req_id];
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
int ret;
if (req->use_pbuf) {
@@ -806,7 +806,7 @@ static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
struct scatterlist *src, struct scatterlist *dst)
{
struct sec_cipher_req *c_req = &req->c_req;
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
if (req->use_pbuf) {
sec_cipher_pbuf_unmap(ctx, req, dst);
@@ -891,6 +891,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
{
struct sec_ctx *ctx = crypto_aead_ctx(tfm);
struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
+ struct device *dev = ctx->dev;
struct crypto_authenc_keys keys;
int ret;
@@ -904,13 +905,13 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
ret = sec_aead_aes_set_key(c_ctx, &keys);
if (ret) {
- dev_err(SEC_CTX_DEV(ctx), "set sec cipher key err!\n");
+ dev_err(dev, "set sec cipher key err!\n");
goto bad_key;
}
ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
if (ret) {
- dev_err(SEC_CTX_DEV(ctx), "set sec auth key err!\n");
+ dev_err(dev, "set sec auth key err!\n");
goto bad_key;
}
@@ -1062,7 +1063,7 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
cryptlen - iv_size);
if (unlikely(sz != iv_size))
- dev_err(SEC_CTX_DEV(req->ctx), "copy output iv error!\n");
+ dev_err(req->ctx->dev, "copy output iv error!\n");
}
static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
@@ -1160,7 +1161,7 @@ static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
ret = sec_skcipher_bd_fill(ctx, req);
if (unlikely(ret)) {
- dev_err(SEC_CTX_DEV(ctx), "skcipher bd fill is error!\n");
+ dev_err(ctx->dev, "skcipher bd fill is error!\n");
return ret;
}
@@ -1194,7 +1195,7 @@ static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
a_req->assoclen);
if (unlikely(sz != authsize)) {
- dev_err(SEC_CTX_DEV(req->ctx), "copy out mac err!\n");
+ dev_err(c->dev, "copy out mac err!\n");
err = -EINVAL;
}
}
@@ -1259,7 +1260,7 @@ static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
ret = ctx->req_op->bd_send(ctx, req);
if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
(ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
- dev_err_ratelimited(SEC_CTX_DEV(ctx), "send sec request failed!\n");
+ dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
goto err_send_req;
}
@@ -1325,7 +1326,7 @@ static int sec_aead_init(struct crypto_aead *tfm)
ctx->alg_type = SEC_AEAD;
ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
- dev_err(SEC_CTX_DEV(ctx), "get error aead iv size!\n");
+ dev_err(ctx->dev, "get error aead iv size!\n");
return -EINVAL;
}
@@ -1374,7 +1375,7 @@ static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
if (IS_ERR(auth_ctx->hash_tfm)) {
- dev_err(SEC_CTX_DEV(ctx), "aead alloc shash error!\n");
+ dev_err(ctx->dev, "aead alloc shash error!\n");
sec_aead_exit(tfm);
return PTR_ERR(auth_ctx->hash_tfm);
}
@@ -1405,10 +1406,40 @@ static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
return sec_aead_ctx_init(tfm, "sha512");
}
+
+static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx,
+ struct sec_req *sreq)
+{
+ u32 cryptlen = sreq->c_req.sk_req->cryptlen;
+ struct device *dev = ctx->dev;
+ u8 c_mode = ctx->c_ctx.c_mode;
+ int ret = 0;
+
+ switch (c_mode) {
+ case SEC_CMODE_XTS:
+ if (unlikely(cryptlen < AES_BLOCK_SIZE)) {
+ dev_err(dev, "skcipher XTS mode input length error!\n");
+ ret = -EINVAL;
+ }
+ break;
+ case SEC_CMODE_ECB:
+ case SEC_CMODE_CBC:
+ if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) {
+ dev_err(dev, "skcipher AES input length error!\n");
+ ret = -EINVAL;
+ }
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
{
struct skcipher_request *sk_req = sreq->c_req.sk_req;
- struct device *dev = SEC_CTX_DEV(ctx);
+ struct device *dev = ctx->dev;
u8 c_alg = ctx->c_ctx.c_alg;
if (unlikely(!sk_req->src || !sk_req->dst)) {
@@ -1429,12 +1460,9 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
}
return 0;
} else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
- if (unlikely(sk_req->cryptlen & (AES_BLOCK_SIZE - 1))) {
- dev_err(dev, "skcipher aes input length error!\n");
- return -EINVAL;
- }
- return 0;
+ return sec_skcipher_cryptlen_ckeck(ctx, sreq);
}
+
dev_err(dev, "skcipher algorithm error!\n");
return -EINVAL;
@@ -1531,14 +1559,15 @@ static struct skcipher_alg sec_skciphers[] = {
static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
{
- u8 c_alg = ctx->c_ctx.c_alg;
struct aead_request *req = sreq->aead_req.aead_req;
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
size_t authsize = crypto_aead_authsize(tfm);
+ struct device *dev = ctx->dev;
+ u8 c_alg = ctx->c_ctx.c_alg;
if (unlikely(!req->src || !req->dst || !req->cryptlen ||
req->assoclen > SEC_MAX_AAD_LEN)) {
- dev_err(SEC_CTX_DEV(ctx), "aead input param error!\n");
+ dev_err(dev, "aead input param error!\n");
return -EINVAL;
}
@@ -1550,7 +1579,7 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
/* Support AES only */
if (unlikely(c_alg != SEC_CALG_AES)) {
- dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n");
+ dev_err(dev, "aead crypto alg error!\n");
return -EINVAL;
}
if (sreq->c_req.encrypt)
@@ -1559,7 +1588,7 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
sreq->c_req.c_len = req->cryptlen - authsize;
if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
- dev_err(SEC_CTX_DEV(ctx), "aead crypto length error!\n");
+ dev_err(dev, "aead crypto length error!\n");
return -EINVAL;
}
@@ -1634,7 +1663,7 @@ static struct aead_alg sec_aeads[] = {
AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
};
-int sec_register_to_crypto(void)
+int sec_register_to_crypto(struct hisi_qm *qm)
{
int ret;
@@ -1651,7 +1680,7 @@ int sec_register_to_crypto(void)
return ret;
}
-void sec_unregister_from_crypto(void)
+void sec_unregister_from_crypto(struct hisi_qm *qm)
{
crypto_unregister_skciphers(sec_skciphers,
ARRAY_SIZE(sec_skciphers));
diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h
index b2786e17d8fe..9c78edac56a4 100644
--- a/drivers/crypto/hisilicon/sec2/sec_crypto.h
+++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h
@@ -64,7 +64,6 @@ enum sec_addr_type {
};
struct sec_sqe_type2 {
-
/*
* mac_len: 0~4 bits
* a_key_len: 5~10 bits
@@ -120,7 +119,6 @@ struct sec_sqe_type2 {
/* c_pad_len_field: 0~1 bits */
__le16 c_pad_len_field;
-
__le64 long_a_data_len;
__le64 a_ivin_addr;
__le64 a_key_addr;
@@ -211,6 +209,6 @@ struct sec_sqe {
struct sec_sqe_type2 type2;
};
-int sec_register_to_crypto(void);
-void sec_unregister_from_crypto(void);
+int sec_register_to_crypto(struct hisi_qm *qm);
+void sec_unregister_from_crypto(struct hisi_qm *qm);
#endif
diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c
index dc68ba76f65e..6f0062d4408c 100644
--- a/drivers/crypto/hisilicon/sec2/sec_main.c
+++ b/drivers/crypto/hisilicon/sec2/sec_main.c
@@ -19,7 +19,6 @@
#define SEC_VF_NUM 63
#define SEC_QUEUE_NUM_V1 4096
-#define SEC_QUEUE_NUM_V2 1024
#define SEC_PF_PCI_DEVICE_ID 0xa255
#define SEC_VF_PCI_DEVICE_ID 0xa256
@@ -35,18 +34,16 @@
#define SEC_CTX_Q_NUM_MAX 32
#define SEC_CTRL_CNT_CLR_CE 0x301120
-#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0)
-#define SEC_ENGINE_PF_CFG_OFF 0x300000
-#define SEC_ACC_COMMON_REG_OFF 0x1000
+#define SEC_CTRL_CNT_CLR_CE_BIT BIT(0)
#define SEC_CORE_INT_SOURCE 0x301010
#define SEC_CORE_INT_MASK 0x301000
#define SEC_CORE_INT_STATUS 0x301008
#define SEC_CORE_SRAM_ECC_ERR_INFO 0x301C14
-#define SEC_ECC_NUM(err) (((err) >> 16) & 0xFF)
-#define SEC_ECC_ADDR(err) ((err) >> 0)
+#define SEC_ECC_NUM 16
+#define SEC_ECC_MASH 0xFF
#define SEC_CORE_INT_DISABLE 0x0
-#define SEC_CORE_INT_ENABLE 0x1ff
-#define SEC_CORE_INT_CLEAR 0x1ff
+#define SEC_CORE_INT_ENABLE 0x7c1ff
+#define SEC_CORE_INT_CLEAR 0x7c1ff
#define SEC_SAA_ENABLE 0x17f
#define SEC_RAS_CE_REG 0x301050
@@ -54,24 +51,24 @@
#define SEC_RAS_NFE_REG 0x301058
#define SEC_RAS_CE_ENB_MSK 0x88
#define SEC_RAS_FE_ENB_MSK 0x0
-#define SEC_RAS_NFE_ENB_MSK 0x177
-#define SEC_RAS_DISABLE 0x0
-#define SEC_MEM_START_INIT_REG 0x0100
-#define SEC_MEM_INIT_DONE_REG 0x0104
+#define SEC_RAS_NFE_ENB_MSK 0x7c177
+#define SEC_RAS_DISABLE 0x0
+#define SEC_MEM_START_INIT_REG 0x301100
+#define SEC_MEM_INIT_DONE_REG 0x301104
-#define SEC_CONTROL_REG 0x0200
+#define SEC_CONTROL_REG 0x301200
#define SEC_TRNG_EN_SHIFT 8
#define SEC_CLK_GATE_ENABLE BIT(3)
#define SEC_CLK_GATE_DISABLE (~BIT(3))
#define SEC_AXI_SHUTDOWN_ENABLE BIT(12)
#define SEC_AXI_SHUTDOWN_DISABLE 0xFFFFEFFF
-#define SEC_INTERFACE_USER_CTRL0_REG 0x0220
-#define SEC_INTERFACE_USER_CTRL1_REG 0x0224
-#define SEC_SAA_EN_REG 0x0270
-#define SEC_BD_ERR_CHK_EN_REG0 0x0380
-#define SEC_BD_ERR_CHK_EN_REG1 0x0384
-#define SEC_BD_ERR_CHK_EN_REG3 0x038c
+#define SEC_INTERFACE_USER_CTRL0_REG 0x301220
+#define SEC_INTERFACE_USER_CTRL1_REG 0x301224
+#define SEC_SAA_EN_REG 0x301270
+#define SEC_BD_ERR_CHK_EN_REG0 0x301380
+#define SEC_BD_ERR_CHK_EN_REG1 0x301384
+#define SEC_BD_ERR_CHK_EN_REG3 0x30138c
#define SEC_USER0_SMMU_NORMAL (BIT(23) | BIT(15))
#define SEC_USER1_SMMU_NORMAL (BIT(31) | BIT(23) | BIT(15) | BIT(7))
@@ -95,9 +92,6 @@
#define SEC_SQE_MASK_OFFSET 64
#define SEC_SQE_MASK_LEN 48
-#define SEC_ADDR(qm, offset) ((qm)->io_base + (offset) + \
- SEC_ENGINE_PF_CFG_OFF + SEC_ACC_COMMON_REG_OFF)
-
struct sec_hw_error {
u32 int_msk;
const char *msg;
@@ -117,20 +111,66 @@ static struct hisi_qm_list sec_devices = {
};
static const struct sec_hw_error sec_hw_errors[] = {
- {.int_msk = BIT(0), .msg = "sec_axi_rresp_err_rint"},
- {.int_msk = BIT(1), .msg = "sec_axi_bresp_err_rint"},
- {.int_msk = BIT(2), .msg = "sec_ecc_2bit_err_rint"},
- {.int_msk = BIT(3), .msg = "sec_ecc_1bit_err_rint"},
- {.int_msk = BIT(4), .msg = "sec_req_trng_timeout_rint"},
- {.int_msk = BIT(5), .msg = "sec_fsm_hbeat_rint"},
- {.int_msk = BIT(6), .msg = "sec_channel_req_rng_timeout_rint"},
- {.int_msk = BIT(7), .msg = "sec_bd_err_rint"},
- {.int_msk = BIT(8), .msg = "sec_chain_buff_err_rint"},
- { /* sentinel */ }
+ {
+ .int_msk = BIT(0),
+ .msg = "sec_axi_rresp_err_rint"
+ },
+ {
+ .int_msk = BIT(1),
+ .msg = "sec_axi_bresp_err_rint"
+ },
+ {
+ .int_msk = BIT(2),
+ .msg = "sec_ecc_2bit_err_rint"
+ },
+ {
+ .int_msk = BIT(3),
+ .msg = "sec_ecc_1bit_err_rint"
+ },
+ {
+ .int_msk = BIT(4),
+ .msg = "sec_req_trng_timeout_rint"
+ },
+ {
+ .int_msk = BIT(5),
+ .msg = "sec_fsm_hbeat_rint"
+ },
+ {
+ .int_msk = BIT(6),
+ .msg = "sec_channel_req_rng_timeout_rint"
+ },
+ {
+ .int_msk = BIT(7),
+ .msg = "sec_bd_err_rint"
+ },
+ {
+ .int_msk = BIT(8),
+ .msg = "sec_chain_buff_err_rint"
+ },
+ {
+ .int_msk = BIT(14),
+ .msg = "sec_no_secure_access"
+ },
+ {
+ .int_msk = BIT(15),
+ .msg = "sec_wrapping_key_auth_err"
+ },
+ {
+ .int_msk = BIT(16),
+ .msg = "sec_km_key_crc_fail"
+ },
+ {
+ .int_msk = BIT(17),
+ .msg = "sec_axi_poison_err"
+ },
+ {
+ .int_msk = BIT(18),
+ .msg = "sec_sva_err"
+ },
+ {}
};
static const char * const sec_dbg_file_name[] = {
- [SEC_CURRENT_QM] = "current_qm",
[SEC_CLEAR_ENABLE] = "clear_enable",
};
@@ -277,9 +317,7 @@ static u8 sec_get_endian(struct hisi_qm *qm)
"cannot access a register in VF!\n");
return SEC_LE;
}
- reg = readl_relaxed(qm->io_base + SEC_ENGINE_PF_CFG_OFF +
- SEC_ACC_COMMON_REG_OFF + SEC_CONTROL_REG);
-
+ reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
/* BD little endian mode */
if (!(reg & BIT(0)))
return SEC_LE;
@@ -299,13 +337,13 @@ static int sec_engine_init(struct hisi_qm *qm)
u32 reg;
/* disable clock gate control */
- reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
+ reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
reg &= SEC_CLK_GATE_DISABLE;
- writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
+ writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);
- writel_relaxed(0x1, SEC_ADDR(qm, SEC_MEM_START_INIT_REG));
+ writel_relaxed(0x1, qm->io_base + SEC_MEM_START_INIT_REG);
- ret = readl_relaxed_poll_timeout(SEC_ADDR(qm, SEC_MEM_INIT_DONE_REG),
+ ret = readl_relaxed_poll_timeout(qm->io_base + SEC_MEM_INIT_DONE_REG,
reg, reg & 0x1, SEC_DELAY_10_US,
SEC_POLL_TIMEOUT_US);
if (ret) {
@@ -313,40 +351,40 @@ static int sec_engine_init(struct hisi_qm *qm)
return ret;
}
- reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
+ reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
reg |= (0x1 << SEC_TRNG_EN_SHIFT);
- writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
+ writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);
- reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG));
+ reg = readl_relaxed(qm->io_base + SEC_INTERFACE_USER_CTRL0_REG);
reg |= SEC_USER0_SMMU_NORMAL;
- writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL0_REG));
+ writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL0_REG);
- reg = readl_relaxed(SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG));
+ reg = readl_relaxed(qm->io_base + SEC_INTERFACE_USER_CTRL1_REG);
reg &= SEC_USER1_SMMU_MASK;
if (qm->use_sva && qm->ver == QM_HW_V2)
reg |= SEC_USER1_SMMU_SVA;
else
reg |= SEC_USER1_SMMU_NORMAL;
- writel_relaxed(reg, SEC_ADDR(qm, SEC_INTERFACE_USER_CTRL1_REG));
+ writel_relaxed(reg, qm->io_base + SEC_INTERFACE_USER_CTRL1_REG);
writel(SEC_SINGLE_PORT_MAX_TRANS,
qm->io_base + AM_CFG_SINGLE_PORT_MAX_TRANS);
- writel(SEC_SAA_ENABLE, SEC_ADDR(qm, SEC_SAA_EN_REG));
+ writel(SEC_SAA_ENABLE, qm->io_base + SEC_SAA_EN_REG);
/* Enable sm4 extra mode, as ctr/ecb */
writel_relaxed(SEC_BD_ERR_CHK_EN0,
- SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG0));
+ qm->io_base + SEC_BD_ERR_CHK_EN_REG0);
/* Enable sm4 xts mode multiple iv */
writel_relaxed(SEC_BD_ERR_CHK_EN1,
- SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG1));
+ qm->io_base + SEC_BD_ERR_CHK_EN_REG1);
writel_relaxed(SEC_BD_ERR_CHK_EN3,
- SEC_ADDR(qm, SEC_BD_ERR_CHK_EN_REG3));
+ qm->io_base + SEC_BD_ERR_CHK_EN_REG3);
/* config endian */
- reg = readl_relaxed(SEC_ADDR(qm, SEC_CONTROL_REG));
+ reg = readl_relaxed(qm->io_base + SEC_CONTROL_REG);
reg |= sec_get_endian(qm);
- writel_relaxed(reg, SEC_ADDR(qm, SEC_CONTROL_REG));
+ writel_relaxed(reg, qm->io_base + SEC_CONTROL_REG);
return 0;
}
@@ -381,10 +419,6 @@ static void sec_debug_regs_clear(struct hisi_qm *qm)
{
int i;
- /* clear current_qm */
- writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
- writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
-
/* clear sec dfx regs */
writel(0x1, qm->io_base + SEC_CTRL_CNT_CLR_CE);
for (i = 0; i < ARRAY_SIZE(sec_dfx_regs); i++)
@@ -406,7 +440,7 @@ static void sec_hw_error_enable(struct hisi_qm *qm)
return;
}
- val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
+ val = readl(qm->io_base + SEC_CONTROL_REG);
/* clear SEC hw error source if having */
writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE);
@@ -422,14 +456,14 @@ static void sec_hw_error_enable(struct hisi_qm *qm)
/* enable SEC block master OOO when m-bit error occur */
val = val | SEC_AXI_SHUTDOWN_ENABLE;
- writel(val, SEC_ADDR(qm, SEC_CONTROL_REG));
+ writel(val, qm->io_base + SEC_CONTROL_REG);
}
static void sec_hw_error_disable(struct hisi_qm *qm)
{
u32 val;
- val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
+ val = readl(qm->io_base + SEC_CONTROL_REG);
/* disable RAS int */
writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG);
@@ -442,51 +476,7 @@ static void sec_hw_error_disable(struct hisi_qm *qm)
/* disable SEC block master OOO when m-bit error occur */
val = val & SEC_AXI_SHUTDOWN_DISABLE;
- writel(val, SEC_ADDR(qm, SEC_CONTROL_REG));
-}
-
-static u32 sec_current_qm_read(struct sec_debug_file *file)
-{
- struct hisi_qm *qm = file->qm;
-
- return readl(qm->io_base + QM_DFX_MB_CNT_VF);
-}
-
-static int sec_current_qm_write(struct sec_debug_file *file, u32 val)
-{
- struct hisi_qm *qm = file->qm;
- u32 vfq_num;
- u32 tmp;
-
- if (val > qm->vfs_num)
- return -EINVAL;
-
- /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
- if (!val) {
- qm->debug.curr_qm_qp_num = qm->qp_num;
- } else {
- vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num;
-
- if (val == qm->vfs_num)
- qm->debug.curr_qm_qp_num =
- qm->ctrl_qp_num - qm->qp_num -
- (qm->vfs_num - 1) * vfq_num;
- else
- qm->debug.curr_qm_qp_num = vfq_num;
- }
-
- writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
- writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
-
- tmp = val |
- (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
- writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
-
- tmp = val |
- (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
- writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
-
- return 0;
+ writel(val, qm->io_base + SEC_CONTROL_REG);
}
static u32 sec_clear_enable_read(struct sec_debug_file *file)
@@ -523,9 +513,6 @@ static ssize_t sec_debug_read(struct file *filp, char __user *buf,
spin_lock_irq(&file->lock);
switch (file->index) {
- case SEC_CURRENT_QM:
- val = sec_current_qm_read(file);
- break;
case SEC_CLEAR_ENABLE:
val = sec_clear_enable_read(file);
break;
@@ -566,11 +553,6 @@ static ssize_t sec_debug_write(struct file *filp, const char __user *buf,
spin_lock_irq(&file->lock);
switch (file->index) {
- case SEC_CURRENT_QM:
- ret = sec_current_qm_write(file, val);
- if (ret)
- goto err_input;
- break;
case SEC_CLEAR_ENABLE:
ret = sec_clear_enable_write(file, val);
if (ret)
@@ -655,7 +637,7 @@ static int sec_debug_init(struct hisi_qm *qm)
int i;
if (qm->pdev->device == SEC_PF_PCI_DEVICE_ID) {
- for (i = SEC_CURRENT_QM; i < SEC_DEBUG_FILE_NUM; i++) {
+ for (i = SEC_CLEAR_ENABLE; i < SEC_DEBUG_FILE_NUM; i++) {
spin_lock_init(&sec->debug.files[i].lock);
sec->debug.files[i].index = i;
sec->debug.files[i].qm = qm;
@@ -712,7 +694,8 @@ static void sec_log_hw_error(struct hisi_qm *qm, u32 err_sts)
err_val = readl(qm->io_base +
SEC_CORE_SRAM_ECC_ERR_INFO);
dev_err(dev, "multi ecc sram num=0x%x\n",
- SEC_ECC_NUM(err_val));
+ ((err_val) >> SEC_ECC_NUM) &
+ SEC_ECC_MASH);
}
}
errs++;
@@ -733,9 +716,23 @@ static void sec_open_axi_master_ooo(struct hisi_qm *qm)
{
u32 val;
- val = readl(SEC_ADDR(qm, SEC_CONTROL_REG));
- writel(val & SEC_AXI_SHUTDOWN_DISABLE, SEC_ADDR(qm, SEC_CONTROL_REG));
- writel(val | SEC_AXI_SHUTDOWN_ENABLE, SEC_ADDR(qm, SEC_CONTROL_REG));
+ val = readl(qm->io_base + SEC_CONTROL_REG);
+ writel(val & SEC_AXI_SHUTDOWN_DISABLE, qm->io_base + SEC_CONTROL_REG);
+ writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG);
+}
+
+static void sec_err_info_init(struct hisi_qm *qm)
+{
+ struct hisi_qm_err_info *err_info = &qm->err_info;
+
+ err_info->ce = QM_BASE_CE;
+ err_info->fe = 0;
+ err_info->ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC;
+ err_info->dev_ce_mask = SEC_RAS_CE_ENB_MSK;
+ err_info->msi_wr_port = BIT(0);
+ err_info->acpi_rst = "SRST";
+ err_info->nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT |
+ QM_ACC_WB_NOT_READY_TIMEOUT;
}
static const struct hisi_qm_err_ini sec_err_ini = {
@@ -746,16 +743,7 @@ static const struct hisi_qm_err_ini sec_err_ini = {
.clear_dev_hw_err_status = sec_clear_hw_err_status,
.log_dev_hw_err = sec_log_hw_error,
.open_axi_master_ooo = sec_open_axi_master_ooo,
- .err_info = {
- .ce = QM_BASE_CE,
- .nfe = QM_BASE_NFE | QM_ACC_DO_TASK_TIMEOUT |
- QM_ACC_WB_NOT_READY_TIMEOUT,
- .fe = 0,
- .ecc_2bits_mask = SEC_CORE_INT_STATUS_M_ECC,
- .dev_ce_mask = SEC_RAS_CE_ENB_MSK,
- .msi_wr_port = BIT(0),
- .acpi_rst = "SRST",
- }
+ .err_info_init = sec_err_info_init,
};
static int sec_pf_probe_init(struct sec_dev *sec)
@@ -763,12 +751,8 @@ static int sec_pf_probe_init(struct sec_dev *sec)
struct hisi_qm *qm = &sec->qm;
int ret;
- if (qm->ver == QM_HW_V1)
- qm->ctrl_qp_num = SEC_QUEUE_NUM_V1;
- else
- qm->ctrl_qp_num = SEC_QUEUE_NUM_V2;
-
qm->err_ini = &sec_err_ini;
+ qm->err_ini->err_info_init(qm);
ret = sec_set_user_domain_and_cache(qm);
if (ret)
@@ -786,7 +770,7 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
qm->pdev = pdev;
qm->ver = pdev->revision;
- qm->algs = "cipher\ndigest\naead\n";
+ qm->algs = "cipher\ndigest\naead";
qm->mode = uacce_mode;
qm->sqe_size = SEC_SQE_SIZE;
qm->dev_name = sec_name;
@@ -909,10 +893,15 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (ret)
pci_warn(pdev, "Failed to init debugfs!\n");
- ret = hisi_qm_alg_register(qm, &sec_devices);
- if (ret < 0) {
- pr_err("Failed to register driver to crypto.\n");
- goto err_qm_stop;
+ if (qm->qp_num >= ctx_q_num) {
+ ret = hisi_qm_alg_register(qm, &sec_devices);
+ if (ret < 0) {
+ pr_err("Failed to register driver to crypto.\n");
+ goto err_qm_stop;
+ }
+ } else {
+ pci_warn(qm->pdev,
+ "Failed to use kernel mode, qp not enough!\n");
}
if (qm->uacce) {
@@ -948,7 +937,9 @@ static void sec_remove(struct pci_dev *pdev)
struct hisi_qm *qm = pci_get_drvdata(pdev);
hisi_qm_wait_task_finish(qm, &sec_devices);
- hisi_qm_alg_unregister(qm, &sec_devices);
+ if (qm->qp_num >= ctx_q_num)
+ hisi_qm_alg_unregister(qm, &sec_devices);
+
if (qm->fun_type == QM_HW_PF && qm->vfs_num)
hisi_qm_sriov_disable(pdev, true);
diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c
index 3bff6394acaf..057273769f26 100644
--- a/drivers/crypto/hisilicon/sgl.c
+++ b/drivers/crypto/hisilicon/sgl.c
@@ -56,7 +56,7 @@ struct hisi_acc_sgl_pool {
struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
u32 count, u32 sge_nr)
{
- u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl = 0;
+ u32 sgl_size, block_size, sgl_num_per_block, block_num, remain_sgl;
struct hisi_acc_sgl_pool *pool;
struct mem_block *block;
u32 i, j;
@@ -66,6 +66,11 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
sgl_size = sizeof(struct acc_hw_sge) * sge_nr +
sizeof(struct hisi_acc_hw_sgl);
+
+ /*
+ * the pool may allocate a block of memory of size PAGE_SIZE * 2^(MAX_ORDER - 1),
+ * block size may exceed 2^31 on ia64, so the max of block size is 2^31
+ */
block_size = 1 << (PAGE_SHIFT + MAX_ORDER <= 32 ?
PAGE_SHIFT + MAX_ORDER - 1 : 31);
sgl_num_per_block = block_size / sgl_size;
@@ -85,8 +90,10 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
block[i].sgl = dma_alloc_coherent(dev, block_size,
&block[i].sgl_dma,
GFP_KERNEL);
- if (!block[i].sgl)
+ if (!block[i].sgl) {
+ dev_err(dev, "Fail to allocate hw SG buffer!\n");
goto err_free_mem;
+ }
block[i].size = block_size;
}
@@ -95,8 +102,10 @@ struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev,
block[i].sgl = dma_alloc_coherent(dev, remain_sgl * sgl_size,
&block[i].sgl_dma,
GFP_KERNEL);
- if (!block[i].sgl)
+ if (!block[i].sgl) {
+ dev_err(dev, "Fail to allocate remained hw SG buffer!\n");
goto err_free_mem;
+ }
block[i].size = remain_sgl * sgl_size;
}
@@ -167,6 +176,7 @@ static void sg_map_to_hw_sg(struct scatterlist *sgl,
{
hw_sge->buf = sg_dma_address(sgl);
hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
+ hw_sge->page_ctrl = sg_virt(sgl);
}
static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
@@ -182,6 +192,18 @@ static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
hw_sgl->entry_sum_in_chain = cpu_to_le16(sum);
}
+static void clear_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
+{
+ struct acc_hw_sge *hw_sge = hw_sgl->sge_entries;
+ int i;
+
+ for (i = 0; i < le16_to_cpu(hw_sgl->entry_sum_in_sgl); i++) {
+ hw_sge[i].page_ctrl = NULL;
+ hw_sge[i].buf = 0;
+ hw_sge[i].len = 0;
+ }
+}
+
/**
* hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
* @dev: The device which hw sgl belongs to.
@@ -211,16 +233,19 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
sg_n = sg_nents(sgl);
sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
- if (!sg_n_mapped)
+ if (!sg_n_mapped) {
+ dev_err(dev, "DMA mapping for SG error!\n");
return ERR_PTR(-EINVAL);
+ }
if (sg_n_mapped > pool->sge_nr) {
- dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
+ dev_err(dev, "the number of entries in input scatterlist is bigger than SGL pool setting.\n");
return ERR_PTR(-EINVAL);
}
curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
if (IS_ERR(curr_hw_sgl)) {
+ dev_err(dev, "Get SGL error!\n");
dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
return ERR_PTR(-ENOMEM);
@@ -256,7 +281,7 @@ void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
return;
dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
-
+ clear_hw_sgl_sge(hw_sgl);
hw_sgl->entry_sum_in_chain = 0;
hw_sgl->entry_sum_in_sgl = 0;
hw_sgl->entry_length_in_sgl = 0;
diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c
index 29712685498a..829f2caf0f67 100644
--- a/drivers/crypto/hisilicon/trng/trng.c
+++ b/drivers/crypto/hisilicon/trng/trng.c
@@ -18,6 +18,8 @@
#define HISI_TRNG_REG 0x00F0
#define HISI_TRNG_BYTES 4
#define HISI_TRNG_QUALITY 512
+#define HISI_TRNG_VERSION 0x01B8
+#define HISI_TRNG_VER_V1 GENMASK(31, 0)
#define SLEEP_US 10
#define TIMEOUT_US 10000
#define SW_DRBG_NUM_SHIFT 2
@@ -50,6 +52,7 @@ struct hisi_trng {
struct hisi_trng_list *trng_list;
struct list_head list;
struct hwrng rng;
+ u32 ver;
bool is_used;
struct mutex mutex;
};
@@ -260,6 +263,7 @@ static int hisi_trng_probe(struct platform_device *pdev)
return PTR_ERR(trng->base);
trng->is_used = false;
+ trng->ver = readl(trng->base + HISI_TRNG_VERSION);
if (!trng_devices.is_init) {
INIT_LIST_HEAD(&trng_devices.list);
mutex_init(&trng_devices.lock);
@@ -267,7 +271,8 @@ static int hisi_trng_probe(struct platform_device *pdev)
}
hisi_trng_add_to_list(trng);
- if (atomic_inc_return(&trng_active_devs) == 1) {
+ if (trng->ver != HISI_TRNG_VER_V1 &&
+ atomic_inc_return(&trng_active_devs) == 1) {
ret = crypto_register_rng(&hisi_trng_alg);
if (ret) {
dev_err(&pdev->dev,
@@ -289,7 +294,8 @@ static int hisi_trng_probe(struct platform_device *pdev)
return ret;
err_crypto_unregister:
- if (atomic_dec_return(&trng_active_devs) == 0)
+ if (trng->ver != HISI_TRNG_VER_V1 &&
+ atomic_dec_return(&trng_active_devs) == 0)
crypto_unregister_rng(&hisi_trng_alg);
err_remove_from_list:
@@ -305,7 +311,8 @@ static int hisi_trng_remove(struct platform_device *pdev)
while (hisi_trng_del_from_list(trng))
;
- if (atomic_dec_return(&trng_active_devs) == 0)
+ if (trng->ver != HISI_TRNG_VER_V1 &&
+ atomic_dec_return(&trng_active_devs) == 0)
crypto_unregister_rng(&hisi_trng_alg);
return 0;
diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h
index 92397f993e23..517fdbdff3ea 100644
--- a/drivers/crypto/hisilicon/zip/zip.h
+++ b/drivers/crypto/hisilicon/zip/zip.h
@@ -33,35 +33,55 @@ struct hisi_zip_sqe {
u32 consumed;
u32 produced;
u32 comp_data_length;
+ /*
+ * status: 0~7 bits
+ * rsvd: 8~31 bits
+ */
u32 dw3;
u32 input_data_length;
- u32 lba_l;
- u32 lba_h;
+ u32 dw5;
+ u32 dw6;
+ /*
+ * in_sge_data_offset: 0~23 bits
+ * rsvd: 24~27 bits
+ * sqe_type: 29~31 bits
+ */
u32 dw7;
+ /*
+ * out_sge_data_offset: 0~23 bits
+ * rsvd: 24~31 bits
+ */
u32 dw8;
+ /*
+ * request_type: 0~7 bits
+ * buffer_type: 8~11 bits
+ * rsvd: 13~31 bits
+ */
u32 dw9;
u32 dw10;
- u32 priv_info;
+ u32 dw11;
u32 dw12;
- u32 tag;
+ /* tag: in sqe type 0 */
+ u32 dw13;
u32 dest_avail_out;
- u32 rsvd0;
- u32 comp_head_addr_l;
- u32 comp_head_addr_h;
+ u32 dw15;
+ u32 dw16;
+ u32 dw17;
u32 source_addr_l;
u32 source_addr_h;
u32 dest_addr_l;
u32 dest_addr_h;
- u32 stream_ctx_addr_l;
- u32 stream_ctx_addr_h;
- u32 cipher_key1_addr_l;
- u32 cipher_key1_addr_h;
- u32 cipher_key2_addr_l;
- u32 cipher_key2_addr_h;
+ u32 dw22;
+ u32 dw23;
+ u32 dw24;
+ u32 dw25;
+ /* tag: in sqe type 3 */
+ u32 dw26;
+ u32 dw27;
u32 rsvd1[4];
};
int zip_create_qps(struct hisi_qp **qps, int ctx_num, int node);
-int hisi_zip_register_to_crypto(void);
-void hisi_zip_unregister_from_crypto(void);
+int hisi_zip_register_to_crypto(struct hisi_qm *qm);
+void hisi_zip_unregister_from_crypto(struct hisi_qm *qm);
#endif
diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c
index 08b4660b014c..9520a4113c81 100644
--- a/drivers/crypto/hisilicon/zip/zip_crypto.c
+++ b/drivers/crypto/hisilicon/zip/zip_crypto.c
@@ -10,6 +10,7 @@
#define HZIP_BD_STATUS_M GENMASK(7, 0)
/* hisi_zip_sqe dw7 */
#define HZIP_IN_SGE_DATA_OFFSET_M GENMASK(23, 0)
+#define HZIP_SQE_TYPE_M GENMASK(31, 28)
/* hisi_zip_sqe dw8 */
#define HZIP_OUT_SGE_DATA_OFFSET_M GENMASK(23, 0)
/* hisi_zip_sqe dw9 */
@@ -91,8 +92,22 @@ struct hisi_zip_qp_ctx {
struct hisi_zip_ctx *ctx;
};
+struct hisi_zip_sqe_ops {
+ u8 sqe_type;
+ void (*fill_addr)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req);
+ void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req);
+ void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type);
+ void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type);
+ void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req);
+ void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type);
+ u32 (*get_tag)(struct hisi_zip_sqe *sqe);
+ u32 (*get_status)(struct hisi_zip_sqe *sqe);
+ u32 (*get_dstlen)(struct hisi_zip_sqe *sqe);
+};
+
struct hisi_zip_ctx {
struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM];
+ const struct hisi_zip_sqe_ops *ops;
};
static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp)
@@ -119,35 +134,367 @@ static u16 sgl_sge_nr = HZIP_SGL_SGE_NR;
module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444);
MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)");
-static void hisi_zip_config_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
+static u16 get_extra_field_size(const u8 *start)
+{
+ return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
+}
+
+static u32 get_name_field_size(const u8 *start)
+{
+ return strlen(start) + 1;
+}
+
+static u32 get_comment_field_size(const u8 *start)
+{
+ return strlen(start) + 1;
+}
+
+static u32 __get_gzip_head_size(const u8 *src)
+{
+ u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
+ u32 size = GZIP_HEAD_FEXTRA_SHIFT;
+
+ if (head_flg & GZIP_HEAD_FEXTRA_BIT)
+ size += get_extra_field_size(src + size);
+ if (head_flg & GZIP_HEAD_FNAME_BIT)
+ size += get_name_field_size(src + size);
+ if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
+ size += get_comment_field_size(src + size);
+ if (head_flg & GZIP_HEAD_FHCRC_BIT)
+ size += GZIP_HEAD_FHCRC_SIZE;
+
+ return size;
+}
+
+static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl)
+{
+ char buf[HZIP_GZIP_HEAD_BUF];
+
+ sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
+
+ return __get_gzip_head_size(buf);
+}
+
+static int add_comp_head(struct scatterlist *dst, u8 req_type)
+{
+ int head_size = TO_HEAD_SIZE(req_type);
+ const u8 *head = TO_HEAD(req_type);
+ int ret;
+
+ ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
+ if (ret != head_size) {
+ pr_err("the head size of buffer is wrong (%d)!\n", ret);
+ return -ENOMEM;
+ }
+
+ return head_size;
+}
+
+static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type)
+{
+ if (!acomp_req->src || !acomp_req->slen)
+ return -EINVAL;
+
+ if (req_type == HZIP_ALG_TYPE_GZIP &&
+ acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT)
+ return -EINVAL;
+
+ switch (req_type) {
+ case HZIP_ALG_TYPE_ZLIB:
+ return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
+ case HZIP_ALG_TYPE_GZIP:
+ return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP);
+ default:
+ pr_err("request type does not support!\n");
+ return -EINVAL;
+ }
+}
+
+static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
+ struct hisi_zip_qp_ctx *qp_ctx,
+ size_t head_size, bool is_comp)
+{
+ struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+ struct hisi_zip_req *q = req_q->q;
+ struct hisi_zip_req *req_cache;
+ int req_id;
+
+ write_lock(&req_q->req_lock);
+
+ req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
+ if (req_id >= req_q->size) {
+ write_unlock(&req_q->req_lock);
+ dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
+ return ERR_PTR(-EAGAIN);
+ }
+ set_bit(req_id, req_q->req_bitmap);
+
+ req_cache = q + req_id;
+ req_cache->req_id = req_id;
+ req_cache->req = req;
+
+ if (is_comp) {
+ req_cache->sskip = 0;
+ req_cache->dskip = head_size;
+ } else {
+ req_cache->sskip = head_size;
+ req_cache->dskip = 0;
+ }
+
+ write_unlock(&req_q->req_lock);
+
+ return req_cache;
+}
+
+static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
+ struct hisi_zip_req *req)
+{
+ struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+
+ write_lock(&req_q->req_lock);
+ clear_bit(req->req_id, req_q->req_bitmap);
+ memset(req, 0, sizeof(struct hisi_zip_req));
+ write_unlock(&req_q->req_lock);
+}
+
+static void hisi_zip_fill_addr(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
+{
+ sqe->source_addr_l = lower_32_bits(req->dma_src);
+ sqe->source_addr_h = upper_32_bits(req->dma_src);
+ sqe->dest_addr_l = lower_32_bits(req->dma_dst);
+ sqe->dest_addr_h = upper_32_bits(req->dma_dst);
+}
+
+static void hisi_zip_fill_buf_size(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
+{
+ struct acomp_req *a_req = req->req;
+
+ sqe->input_data_length = a_req->slen - req->sskip;
+ sqe->dest_avail_out = a_req->dlen - req->dskip;
+ sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, req->sskip);
+ sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, req->dskip);
+}
+
+static void hisi_zip_fill_buf_type(struct hisi_zip_sqe *sqe, u8 buf_type)
{
u32 val;
- val = (sqe->dw9) & ~HZIP_BUF_TYPE_M;
+ val = sqe->dw9 & ~HZIP_BUF_TYPE_M;
val |= FIELD_PREP(HZIP_BUF_TYPE_M, buf_type);
sqe->dw9 = val;
}
-static void hisi_zip_config_tag(struct hisi_zip_sqe *sqe, u32 tag)
+static void hisi_zip_fill_req_type(struct hisi_zip_sqe *sqe, u8 req_type)
{
- sqe->tag = tag;
+ u32 val;
+
+ val = sqe->dw9 & ~HZIP_REQ_TYPE_M;
+ val |= FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
+ sqe->dw9 = val;
}
-static void hisi_zip_fill_sqe(struct hisi_zip_sqe *sqe, u8 req_type,
- dma_addr_t s_addr, dma_addr_t d_addr, u32 slen,
- u32 dlen, u32 sskip, u32 dskip)
+static void hisi_zip_fill_tag_v1(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
{
+ sqe->dw13 = req->req_id;
+}
+
+static void hisi_zip_fill_tag_v2(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req)
+{
+ sqe->dw26 = req->req_id;
+}
+
+static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type)
+{
+ u32 val;
+
+ val = sqe->dw7 & ~HZIP_SQE_TYPE_M;
+ val |= FIELD_PREP(HZIP_SQE_TYPE_M, sqe_type);
+ sqe->dw7 = val;
+}
+
+static void hisi_zip_fill_sqe(struct hisi_zip_ctx *ctx, struct hisi_zip_sqe *sqe,
+ u8 req_type, struct hisi_zip_req *req)
+{
+ const struct hisi_zip_sqe_ops *ops = ctx->ops;
+
memset(sqe, 0, sizeof(struct hisi_zip_sqe));
- sqe->input_data_length = slen - sskip;
- sqe->dw7 = FIELD_PREP(HZIP_IN_SGE_DATA_OFFSET_M, sskip);
- sqe->dw8 = FIELD_PREP(HZIP_OUT_SGE_DATA_OFFSET_M, dskip);
- sqe->dw9 = FIELD_PREP(HZIP_REQ_TYPE_M, req_type);
- sqe->dest_avail_out = dlen - dskip;
- sqe->source_addr_l = lower_32_bits(s_addr);
- sqe->source_addr_h = upper_32_bits(s_addr);
- sqe->dest_addr_l = lower_32_bits(d_addr);
- sqe->dest_addr_h = upper_32_bits(d_addr);
+ ops->fill_addr(sqe, req);
+ ops->fill_buf_size(sqe, req);
+ ops->fill_buf_type(sqe, HZIP_SGL);
+ ops->fill_req_type(sqe, req_type);
+ ops->fill_tag(sqe, req);
+ ops->fill_sqe_type(sqe, ops->sqe_type);
+}
+
+static int hisi_zip_do_work(struct hisi_zip_req *req,
+ struct hisi_zip_qp_ctx *qp_ctx)
+{
+ struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
+ struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
+ struct acomp_req *a_req = req->req;
+ struct hisi_qp *qp = qp_ctx->qp;
+ struct device *dev = &qp->qm->pdev->dev;
+ struct hisi_zip_sqe zip_sqe;
+ int ret;
+
+ if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)
+ return -EINVAL;
+
+ req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
+ req->req_id << 1, &req->dma_src);
+ if (IS_ERR(req->hw_src)) {
+ dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n",
+ PTR_ERR(req->hw_src));
+ return PTR_ERR(req->hw_src);
+ }
+
+ req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool,
+ (req->req_id << 1) + 1,
+ &req->dma_dst);
+ if (IS_ERR(req->hw_dst)) {
+ ret = PTR_ERR(req->hw_dst);
+ dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n",
+ ret);
+ goto err_unmap_input;
+ }
+
+ hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req);
+
+ /* send command to start a task */
+ atomic64_inc(&dfx->send_cnt);
+ ret = hisi_qp_send(qp, &zip_sqe);
+ if (ret < 0) {
+ atomic64_inc(&dfx->send_busy_cnt);
+ ret = -EAGAIN;
+ dev_dbg_ratelimited(dev, "failed to send request!\n");
+ goto err_unmap_output;
+ }
+
+ return -EINPROGRESS;
+
+err_unmap_output:
+ hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst);
+err_unmap_input:
+ hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src);
+ return ret;
+}
+
+static u32 hisi_zip_get_tag_v1(struct hisi_zip_sqe *sqe)
+{
+ return sqe->dw13;
+}
+
+static u32 hisi_zip_get_tag_v2(struct hisi_zip_sqe *sqe)
+{
+ return sqe->dw26;
+}
+
+static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe)
+{
+ return sqe->dw3 & HZIP_BD_STATUS_M;
+}
+
+static u32 hisi_zip_get_dstlen(struct hisi_zip_sqe *sqe)
+{
+ return sqe->produced;
+}
+
+static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
+{
+ struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
+ const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops;
+ struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
+ struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
+ struct device *dev = &qp->qm->pdev->dev;
+ struct hisi_zip_sqe *sqe = data;
+ u32 tag = ops->get_tag(sqe);
+ struct hisi_zip_req *req = req_q->q + tag;
+ struct acomp_req *acomp_req = req->req;
+ u32 status, dlen, head_size;
+ int err = 0;
+
+ atomic64_inc(&dfx->recv_cnt);
+ status = ops->get_status(sqe);
+ if (status != 0 && status != HZIP_NC_ERR) {
+ dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
+ (qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
+ sqe->produced);
+ atomic64_inc(&dfx->err_bd_cnt);
+ err = -EIO;
+ }
+
+ dlen = ops->get_dstlen(sqe);
+
+ hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src);
+ hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst);
+
+ head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
+ acomp_req->dlen = dlen + head_size;
+
+ if (acomp_req->base.complete)
+ acomp_request_complete(acomp_req, err);
+
+ hisi_zip_remove_req(qp_ctx, req);
+}
+
+static int hisi_zip_acompress(struct acomp_req *acomp_req)
+{
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
+ struct hisi_zip_req *req;
+ int head_size;
+ int ret;
+
+ /* let's output compression head now */
+ head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
+ if (head_size < 0) {
+ dev_err_ratelimited(dev, "failed to add comp head (%d)!\n",
+ head_size);
+ return head_size;
+ }
+
+ req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+
+ ret = hisi_zip_do_work(req, qp_ctx);
+ if (ret != -EINPROGRESS) {
+ dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
+ hisi_zip_remove_req(qp_ctx, req);
+ }
+
+ return ret;
+}
+
+static int hisi_zip_adecompress(struct acomp_req *acomp_req)
+{
+ struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
+ struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
+ struct device *dev = &qp_ctx->qp->qm->pdev->dev;
+ struct hisi_zip_req *req;
+ int head_size, ret;
+
+ head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type);
+ if (head_size < 0) {
+ dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n",
+ head_size);
+ return head_size;
+ }
+
+ req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
+ if (IS_ERR(req))
+ return PTR_ERR(req);
+
+ ret = hisi_zip_do_work(req, qp_ctx);
+ if (ret != -EINPROGRESS) {
+ dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
+ ret);
+ hisi_zip_remove_req(qp_ctx, req);
+ }
+
+ return ret;
}
static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *ctx,
@@ -177,9 +524,36 @@ static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
hisi_qm_release_qp(ctx->qp);
}
+static const struct hisi_zip_sqe_ops hisi_zip_ops_v1 = {
+ .sqe_type = 0,
+ .fill_addr = hisi_zip_fill_addr,
+ .fill_buf_size = hisi_zip_fill_buf_size,
+ .fill_buf_type = hisi_zip_fill_buf_type,
+ .fill_req_type = hisi_zip_fill_req_type,
+ .fill_tag = hisi_zip_fill_tag_v1,
+ .fill_sqe_type = hisi_zip_fill_sqe_type,
+ .get_tag = hisi_zip_get_tag_v1,
+ .get_status = hisi_zip_get_status,
+ .get_dstlen = hisi_zip_get_dstlen,
+};
+
+static const struct hisi_zip_sqe_ops hisi_zip_ops_v2 = {
+ .sqe_type = 0x3,
+ .fill_addr = hisi_zip_fill_addr,
+ .fill_buf_size = hisi_zip_fill_buf_size,
+ .fill_buf_type = hisi_zip_fill_buf_type,
+ .fill_req_type = hisi_zip_fill_req_type,
+ .fill_tag = hisi_zip_fill_tag_v2,
+ .fill_sqe_type = hisi_zip_fill_sqe_type,
+ .get_tag = hisi_zip_get_tag_v2,
+ .get_status = hisi_zip_get_status,
+ .get_dstlen = hisi_zip_get_dstlen,
+};
+
static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int node)
{
struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL };
+ struct hisi_zip_qp_ctx *qp_ctx;
struct hisi_zip *hisi_zip;
int ret, i, j;
@@ -193,8 +567,9 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int
for (i = 0; i < HZIP_CTX_Q_NUM; i++) {
/* alg_type = 0 for compress, 1 for decompress in hw sqe */
- ret = hisi_zip_start_qp(qps[i], &hisi_zip_ctx->qp_ctx[i], i,
- req_type);
+ qp_ctx = &hisi_zip_ctx->qp_ctx[i];
+ qp_ctx->ctx = hisi_zip_ctx;
+ ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type);
if (ret) {
for (j = i - 1; j >= 0; j--)
hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp);
@@ -203,9 +578,14 @@ static int hisi_zip_ctx_init(struct hisi_zip_ctx *hisi_zip_ctx, u8 req_type, int
return ret;
}
- hisi_zip_ctx->qp_ctx[i].zip_dev = hisi_zip;
+ qp_ctx->zip_dev = hisi_zip;
}
+ if (hisi_zip->qm.ver < QM_HW_V3)
+ hisi_zip_ctx->ops = &hisi_zip_ops_v1;
+ else
+ hisi_zip_ctx->ops = &hisi_zip_ops_v2;
+
return 0;
}
@@ -217,38 +597,6 @@ static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx)
hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
}
-static u16 get_extra_field_size(const u8 *start)
-{
- return *((u16 *)start) + GZIP_HEAD_FEXTRA_XLEN;
-}
-
-static u32 get_name_field_size(const u8 *start)
-{
- return strlen(start) + 1;
-}
-
-static u32 get_comment_field_size(const u8 *start)
-{
- return strlen(start) + 1;
-}
-
-static u32 __get_gzip_head_size(const u8 *src)
-{
- u8 head_flg = *(src + GZIP_HEAD_FLG_SHIFT);
- u32 size = GZIP_HEAD_FEXTRA_SHIFT;
-
- if (head_flg & GZIP_HEAD_FEXTRA_BIT)
- size += get_extra_field_size(src + size);
- if (head_flg & GZIP_HEAD_FNAME_BIT)
- size += get_name_field_size(src + size);
- if (head_flg & GZIP_HEAD_FCOMMENT_BIT)
- size += get_comment_field_size(src + size);
- if (head_flg & GZIP_HEAD_FHCRC_BIT)
- size += GZIP_HEAD_FHCRC_SIZE;
-
- return size;
-}
-
static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx)
{
struct hisi_zip_req_q *req_q;
@@ -336,52 +684,6 @@ static void hisi_zip_release_sgl_pool(struct hisi_zip_ctx *ctx)
ctx->qp_ctx[i].sgl_pool);
}
-static void hisi_zip_remove_req(struct hisi_zip_qp_ctx *qp_ctx,
- struct hisi_zip_req *req)
-{
- struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
-
- write_lock(&req_q->req_lock);
- clear_bit(req->req_id, req_q->req_bitmap);
- memset(req, 0, sizeof(struct hisi_zip_req));
- write_unlock(&req_q->req_lock);
-}
-
-static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data)
-{
- struct hisi_zip_sqe *sqe = data;
- struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx;
- struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
- struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
- struct hisi_zip_req *req = req_q->q + sqe->tag;
- struct acomp_req *acomp_req = req->req;
- struct device *dev = &qp->qm->pdev->dev;
- u32 status, dlen, head_size;
- int err = 0;
-
- atomic64_inc(&dfx->recv_cnt);
- status = sqe->dw3 & HZIP_BD_STATUS_M;
- if (status != 0 && status != HZIP_NC_ERR) {
- dev_err(dev, "%scompress fail in qp%u: %u, output: %u\n",
- (qp->alg_type == 0) ? "" : "de", qp->qp_id, status,
- sqe->produced);
- atomic64_inc(&dfx->err_bd_cnt);
- err = -EIO;
- }
- dlen = sqe->produced;
-
- hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src);
- hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst);
-
- head_size = (qp->alg_type == 0) ? TO_HEAD_SIZE(qp->req_type) : 0;
- acomp_req->dlen = dlen + head_size;
-
- if (acomp_req->base.complete)
- acomp_request_complete(acomp_req, err);
-
- hisi_zip_remove_req(qp_ctx, req);
-}
-
static void hisi_zip_set_acomp_cb(struct hisi_zip_ctx *ctx,
void (*fn)(struct hisi_qp *, void *))
{
@@ -439,204 +741,6 @@ static void hisi_zip_acomp_exit(struct crypto_acomp *tfm)
hisi_zip_ctx_exit(ctx);
}
-static int add_comp_head(struct scatterlist *dst, u8 req_type)
-{
- int head_size = TO_HEAD_SIZE(req_type);
- const u8 *head = TO_HEAD(req_type);
- int ret;
-
- ret = sg_copy_from_buffer(dst, sg_nents(dst), head, head_size);
- if (ret != head_size) {
- pr_err("the head size of buffer is wrong (%d)!\n", ret);
- return -ENOMEM;
- }
-
- return head_size;
-}
-
-static size_t __maybe_unused get_gzip_head_size(struct scatterlist *sgl)
-{
- char buf[HZIP_GZIP_HEAD_BUF];
-
- sg_copy_to_buffer(sgl, sg_nents(sgl), buf, sizeof(buf));
-
- return __get_gzip_head_size(buf);
-}
-
-static int get_comp_head_size(struct acomp_req *acomp_req, u8 req_type)
-{
- if (!acomp_req->src || !acomp_req->slen)
- return -EINVAL;
-
- if ((req_type == HZIP_ALG_TYPE_GZIP) &&
- (acomp_req->slen < GZIP_HEAD_FEXTRA_SHIFT))
- return -EINVAL;
-
- switch (req_type) {
- case HZIP_ALG_TYPE_ZLIB:
- return TO_HEAD_SIZE(HZIP_ALG_TYPE_ZLIB);
- case HZIP_ALG_TYPE_GZIP:
- return TO_HEAD_SIZE(HZIP_ALG_TYPE_GZIP);
- default:
- pr_err("request type does not support!\n");
- return -EINVAL;
- }
-}
-
-static struct hisi_zip_req *hisi_zip_create_req(struct acomp_req *req,
- struct hisi_zip_qp_ctx *qp_ctx,
- size_t head_size, bool is_comp)
-{
- struct hisi_zip_req_q *req_q = &qp_ctx->req_q;
- struct hisi_zip_req *q = req_q->q;
- struct hisi_zip_req *req_cache;
- int req_id;
-
- write_lock(&req_q->req_lock);
-
- req_id = find_first_zero_bit(req_q->req_bitmap, req_q->size);
- if (req_id >= req_q->size) {
- write_unlock(&req_q->req_lock);
- dev_dbg(&qp_ctx->qp->qm->pdev->dev, "req cache is full!\n");
- return ERR_PTR(-EAGAIN);
- }
- set_bit(req_id, req_q->req_bitmap);
-
- req_cache = q + req_id;
- req_cache->req_id = req_id;
- req_cache->req = req;
-
- if (is_comp) {
- req_cache->sskip = 0;
- req_cache->dskip = head_size;
- } else {
- req_cache->sskip = head_size;
- req_cache->dskip = 0;
- }
-
- write_unlock(&req_q->req_lock);
-
- return req_cache;
-}
-
-static int hisi_zip_do_work(struct hisi_zip_req *req,
- struct hisi_zip_qp_ctx *qp_ctx)
-{
- struct acomp_req *a_req = req->req;
- struct hisi_qp *qp = qp_ctx->qp;
- struct device *dev = &qp->qm->pdev->dev;
- struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool;
- struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx;
- struct hisi_zip_sqe zip_sqe;
- dma_addr_t input, output;
- int ret;
-
- if (!a_req->src || !a_req->slen || !a_req->dst || !a_req->dlen)
- return -EINVAL;
-
- req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool,
- req->req_id << 1, &input);
- if (IS_ERR(req->hw_src)) {
- dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n",
- PTR_ERR(req->hw_src));
- return PTR_ERR(req->hw_src);
- }
- req->dma_src = input;
-
- req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool,
- (req->req_id << 1) + 1,
- &output);
- if (IS_ERR(req->hw_dst)) {
- ret = PTR_ERR(req->hw_dst);
- dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n",
- ret);
- goto err_unmap_input;
- }
- req->dma_dst = output;
-
- hisi_zip_fill_sqe(&zip_sqe, qp->req_type, input, output, a_req->slen,
- a_req->dlen, req->sskip, req->dskip);
- hisi_zip_config_buf_type(&zip_sqe, HZIP_SGL);
- hisi_zip_config_tag(&zip_sqe, req->req_id);
-
- /* send command to start a task */
- atomic64_inc(&dfx->send_cnt);
- ret = hisi_qp_send(qp, &zip_sqe);
- if (ret < 0) {
- atomic64_inc(&dfx->send_busy_cnt);
- ret = -EAGAIN;
- dev_dbg_ratelimited(dev, "failed to send request!\n");
- goto err_unmap_output;
- }
-
- return -EINPROGRESS;
-
-err_unmap_output:
- hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst);
-err_unmap_input:
- hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src);
- return ret;
-}
-
-static int hisi_zip_acompress(struct acomp_req *acomp_req)
-{
- struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP];
- struct device *dev = &qp_ctx->qp->qm->pdev->dev;
- struct hisi_zip_req *req;
- int head_size;
- int ret;
-
- /* let's output compression head now */
- head_size = add_comp_head(acomp_req->dst, qp_ctx->qp->req_type);
- if (head_size < 0) {
- dev_err_ratelimited(dev, "failed to add comp head (%d)!\n",
- head_size);
- return head_size;
- }
-
- req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, true);
- if (IS_ERR(req))
- return PTR_ERR(req);
-
- ret = hisi_zip_do_work(req, qp_ctx);
- if (ret != -EINPROGRESS) {
- dev_info_ratelimited(dev, "failed to do compress (%d)!\n", ret);
- hisi_zip_remove_req(qp_ctx, req);
- }
-
- return ret;
-}
-
-static int hisi_zip_adecompress(struct acomp_req *acomp_req)
-{
- struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm);
- struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP];
- struct device *dev = &qp_ctx->qp->qm->pdev->dev;
- struct hisi_zip_req *req;
- int head_size, ret;
-
- head_size = get_comp_head_size(acomp_req, qp_ctx->qp->req_type);
- if (head_size < 0) {
- dev_err_ratelimited(dev, "failed to get comp head size (%d)!\n",
- head_size);
- return head_size;
- }
-
- req = hisi_zip_create_req(acomp_req, qp_ctx, head_size, false);
- if (IS_ERR(req))
- return PTR_ERR(req);
-
- ret = hisi_zip_do_work(req, qp_ctx);
- if (ret != -EINPROGRESS) {
- dev_info_ratelimited(dev, "failed to do decompress (%d)!\n",
- ret);
- hisi_zip_remove_req(qp_ctx, req);
- }
-
- return ret;
-}
-
static struct acomp_alg hisi_zip_acomp_zlib = {
.init = hisi_zip_acomp_init,
.exit = hisi_zip_acomp_exit,
@@ -665,7 +769,7 @@ static struct acomp_alg hisi_zip_acomp_gzip = {
}
};
-int hisi_zip_register_to_crypto(void)
+int hisi_zip_register_to_crypto(struct hisi_qm *qm)
{
int ret;
@@ -684,7 +788,7 @@ int hisi_zip_register_to_crypto(void)
return ret;
}
-void hisi_zip_unregister_from_crypto(void)
+void hisi_zip_unregister_from_crypto(struct hisi_qm *qm)
{
crypto_unregister_acomp(&hisi_zip_acomp_gzip);
crypto_unregister_acomp(&hisi_zip_acomp_zlib);
diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c
index 02c445722445..2178b40e9f82 100644
--- a/drivers/crypto/hisilicon/zip/zip_main.c
+++ b/drivers/crypto/hisilicon/zip/zip_main.c
@@ -18,7 +18,6 @@
#define PCI_DEVICE_ID_ZIP_VF 0xa251
#define HZIP_QUEUE_NUM_V1 4096
-#define HZIP_QUEUE_NUM_V2 1024
#define HZIP_CLOCK_GATE_CTRL 0x301004
#define COMP0_ENABLE BIT(0)
@@ -69,10 +68,10 @@
#define HZIP_CORE_INT_RAS_CE_ENABLE 0x1
#define HZIP_CORE_INT_RAS_NFE_ENB 0x301164
#define HZIP_CORE_INT_RAS_FE_ENB 0x301168
-#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x7FE
+#define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE
#define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16
#define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24
-#define HZIP_CORE_INT_MASK_ALL GENMASK(10, 0)
+#define HZIP_CORE_INT_MASK_ALL GENMASK(12, 0)
#define HZIP_COMP_CORE_NUM 2
#define HZIP_DECOMP_CORE_NUM 6
#define HZIP_CORE_NUM (HZIP_COMP_CORE_NUM + \
@@ -134,17 +133,17 @@ static const struct hisi_zip_hw_error zip_hw_error[] = {
{ .int_msk = BIT(8), .msg = "zip_com_inf_err" },
{ .int_msk = BIT(9), .msg = "zip_enc_inf_err" },
{ .int_msk = BIT(10), .msg = "zip_pre_out_err" },
+ { .int_msk = BIT(11), .msg = "zip_axi_poison_err" },
+ { .int_msk = BIT(12), .msg = "zip_sva_err" },
{ /* sentinel */ }
};
enum ctrl_debug_file_index {
- HZIP_CURRENT_QM,
HZIP_CLEAR_ENABLE,
HZIP_DEBUG_FILE_NUM,
};
static const char * const ctrl_debug_file_name[] = {
- [HZIP_CURRENT_QM] = "current_qm",
[HZIP_CLEAR_ENABLE] = "clear_enable",
};
@@ -363,48 +362,6 @@ static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file)
return &hisi_zip->qm;
}
-static u32 current_qm_read(struct ctrl_debug_file *file)
-{
- struct hisi_qm *qm = file_to_qm(file);
-
- return readl(qm->io_base + QM_DFX_MB_CNT_VF);
-}
-
-static int current_qm_write(struct ctrl_debug_file *file, u32 val)
-{
- struct hisi_qm *qm = file_to_qm(file);
- u32 vfq_num;
- u32 tmp;
-
- if (val > qm->vfs_num)
- return -EINVAL;
-
- /* According PF or VF Dev ID to calculation curr_qm_qp_num and store */
- if (val == 0) {
- qm->debug.curr_qm_qp_num = qm->qp_num;
- } else {
- vfq_num = (qm->ctrl_qp_num - qm->qp_num) / qm->vfs_num;
- if (val == qm->vfs_num)
- qm->debug.curr_qm_qp_num = qm->ctrl_qp_num -
- qm->qp_num - (qm->vfs_num - 1) * vfq_num;
- else
- qm->debug.curr_qm_qp_num = vfq_num;
- }
-
- writel(val, qm->io_base + QM_DFX_MB_CNT_VF);
- writel(val, qm->io_base + QM_DFX_DB_CNT_VF);
-
- tmp = val |
- (readl(qm->io_base + QM_DFX_SQE_CNT_VF_SQN) & CURRENT_Q_MASK);
- writel(tmp, qm->io_base + QM_DFX_SQE_CNT_VF_SQN);
-
- tmp = val |
- (readl(qm->io_base + QM_DFX_CQE_CNT_VF_CQN) & CURRENT_Q_MASK);
- writel(tmp, qm->io_base + QM_DFX_CQE_CNT_VF_CQN);
-
- return 0;
-}
-
static u32 clear_enable_read(struct ctrl_debug_file *file)
{
struct hisi_qm *qm = file_to_qm(file);
@@ -438,9 +395,6 @@ static ssize_t hisi_zip_ctrl_debug_read(struct file *filp, char __user *buf,
spin_lock_irq(&file->lock);
switch (file->index) {
- case HZIP_CURRENT_QM:
- val = current_qm_read(file);
- break;
case HZIP_CLEAR_ENABLE:
val = clear_enable_read(file);
break;
@@ -478,11 +432,6 @@ static ssize_t hisi_zip_ctrl_debug_write(struct file *filp,
spin_lock_irq(&file->lock);
switch (file->index) {
- case HZIP_CURRENT_QM:
- ret = current_qm_write(file, val);
- if (ret)
- goto err_input;
- break;
case HZIP_CLEAR_ENABLE:
ret = clear_enable_write(file, val);
if (ret)
@@ -580,7 +529,7 @@ static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm)
struct hisi_zip *zip = container_of(qm, struct hisi_zip, qm);
int i;
- for (i = HZIP_CURRENT_QM; i < HZIP_DEBUG_FILE_NUM; i++) {
+ for (i = HZIP_CLEAR_ENABLE; i < HZIP_DEBUG_FILE_NUM; i++) {
spin_lock_init(&zip->ctrl->files[i].lock);
zip->ctrl->files[i].ctrl = zip->ctrl;
zip->ctrl->files[i].index = i;
@@ -627,10 +576,6 @@ static void hisi_zip_debug_regs_clear(struct hisi_qm *qm)
{
int i, j;
- /* clear current_qm */
- writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF);
- writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF);
-
/* enable register read_clear bit */
writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE);
for (i = 0; i < ARRAY_SIZE(core_offsets); i++)
@@ -714,6 +659,22 @@ static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm)
qm->io_base + HZIP_CORE_INT_SET);
}
+static void hisi_zip_err_info_init(struct hisi_qm *qm)
+{
+ struct hisi_qm_err_info *err_info = &qm->err_info;
+
+ err_info->ce = QM_BASE_CE;
+ err_info->fe = 0;
+ err_info->ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC;
+ err_info->dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE;
+ err_info->msi_wr_port = HZIP_WR_PORT;
+ err_info->acpi_rst = "ZRST";
+ err_info->nfe = QM_BASE_NFE | QM_ACC_WB_NOT_READY_TIMEOUT;
+
+ if (qm->ver >= QM_HW_V3)
+ err_info->nfe |= QM_ACC_DO_TASK_TIMEOUT;
+}
+
static const struct hisi_qm_err_ini hisi_zip_err_ini = {
.hw_init = hisi_zip_set_user_domain_and_cache,
.hw_err_enable = hisi_zip_hw_error_enable,
@@ -723,16 +684,7 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = {
.log_dev_hw_err = hisi_zip_log_hw_error,
.open_axi_master_ooo = hisi_zip_open_axi_master_ooo,
.close_axi_master_ooo = hisi_zip_close_axi_master_ooo,
- .err_info = {
- .ce = QM_BASE_CE,
- .nfe = QM_BASE_NFE |
- QM_ACC_WB_NOT_READY_TIMEOUT,
- .fe = 0,
- .ecc_2bits_mask = HZIP_CORE_INT_STATUS_M_ECC,
- .dev_ce_mask = HZIP_CORE_INT_RAS_CE_ENABLE,
- .msi_wr_port = HZIP_WR_PORT,
- .acpi_rst = "ZRST",
- }
+ .err_info_init = hisi_zip_err_info_init,
};
static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
@@ -746,13 +698,8 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip)
hisi_zip->ctrl = ctrl;
ctrl->hisi_zip = hisi_zip;
-
- if (qm->ver == QM_HW_V1)
- qm->ctrl_qp_num = HZIP_QUEUE_NUM_V1;
- else
- qm->ctrl_qp_num = HZIP_QUEUE_NUM_V2;
-
qm->err_ini = &hisi_zip_err_ini;
+ qm->err_ini->err_info_init(qm);
hisi_zip_set_user_domain_and_cache(qm);
hisi_qm_dev_err_init(qm);
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-01 12:07:55 +0000